From abf0d45784dcf9060d2bcc38d02a0621b848c481 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sat, 22 Oct 2022 00:54:52 -0400 Subject: [PATCH] improvement: Refactor mdgw/ftx to use SecurityCriteria So we can subscribe to depth for "FTX Options" or "FTX BTC Futures", rather than having to list each security explicitly. This commit also unhooks binance from MarketDataGateway. A new paradigm is emerging with the FTX impl, and it'll be easier to redo Binance than try to make the existing impl fit the new mold. Unfortunately, ccapi doesn't provide all the security metadata we need, such as the security type and expiration date/time, so we get the list of FTX securities directly from the exchange. --- CMakeLists.txt | 1 + src/lib/CMakeLists.txt | 3 +- src/lib/api/md/BinanceMarketDataConfig.proto | 10 + src/lib/api/md/DepthStreamConfig.proto | 11 - src/lib/api/md/FTXMarketDataConfig.proto | 13 + src/lib/api/md/MarketDataConfig.proto | 12 +- src/lib/collections/DiscreteValuesFilter.h | 98 ++++ src/lib/collections/Find.h | 6 + src/lib/collections/Remove.h | 58 +++ src/lib/config/ConfigurationState.cpp | 23 +- src/lib/config/defaults/MarketData.conf | 14 +- src/lib/config/defaults/MarketDataConfig.conf | 14 + src/lib/config/protos/market_data.proto | 4 +- src/lib/core/BoolName.h | 39 ++ src/lib/core/NaN.h | 24 + src/lib/core/StringContains.h | 70 +++ src/lib/core/StringFinder.h | 32 ++ src/lib/core/StringJoiner.h | 178 ++++++++ src/lib/io/JsonReader.cpp | 420 ++++++++++++++++++ src/lib/io/JsonReader.h | 123 +++++ src/lib/io/JsonType.h | 47 ++ src/lib/md/CCApiCorrelationID.h | 284 ++++++++++-- src/lib/md/CCApiEventDemuxer.cpp | 166 +++++++ src/lib/md/CCApiEventDemuxer.h | 82 ++++ src/lib/md/CCApiListenerAdaptor.cpp | 165 ------- src/lib/md/CCApiListenerAdaptor.h | 37 -- src/lib/md/CCApiMarketDepthAdaptor.cpp | 78 ++++ src/lib/md/CCApiMarketDepthAdaptor.h | 44 ++ src/lib/md/CCApiMessageHandler.h | 46 ++ src/lib/md/CCApiMessageLogger.cpp | 33 ++ src/lib/md/CCApiMessageLogger.h | 34 ++ src/lib/md/CCApiReader.h | 155 +++++++ src/lib/md/CCApiSecurityParser.h | 107 ----- src/lib/md/CCApiSession.cpp | 186 +++++--- src/lib/md/CCApiSession.h | 81 +++- src/lib/md/CCApiTradeAdaptor.cpp | 70 +++ src/lib/md/CCApiTradeAdaptor.h | 39 ++ src/lib/md/CCApiTradeRecord.cpp | 93 ++++ src/lib/md/CCApiTradeRecord.h | 50 +++ src/lib/md/ICCApiMessageHandler.cpp | 10 + src/lib/md/ICCApiMessageHandler.h | 32 ++ src/lib/md/MarketDataGateway.cpp | 102 +++-- src/lib/md/MarketDataListenerData.h | 12 + src/lib/md/MarketDataSubscription.cpp | 27 ++ src/lib/md/MarketDataSubscription.h | 32 ++ src/lib/md/ccApi/CCMain.cpp | 14 +- src/lib/md/ccApi/CCMarketDataListener.cpp | 16 - src/lib/md/ftx/FTXFuture.cpp | 181 ++++++++ src/lib/md/ftx/FTXFuture.h | 73 +++ src/lib/md/ftx/FTXHistoricalPriceAdaptor.cpp | 54 +++ src/lib/md/ftx/FTXHistoricalPriceAdaptor.h | 31 ++ src/lib/md/ftx/FTXHistoricalPriceRecord.cpp | 90 ++++ src/lib/md/ftx/FTXHistoricalPriceRecord.h | 41 ++ src/lib/md/ftx/FTXMarket.cpp | 217 +++++++++ src/lib/md/ftx/FTXMarket.h | 61 +++ src/lib/md/ftx/FTXMessageType.h | 35 ++ src/lib/md/ftx/FTXSecurityRequestor.cpp | 275 ++++++++++++ src/lib/md/ftx/FTXSecurityRequestor.h | 79 ++++ src/lib/md/ftx/FTXSecurityType.h | 65 +++ src/lib/md/ftx/FTXSubscription.cpp | 243 ++++++++++ src/lib/md/ftx/FTXSubscription.h | 50 +++ src/lib/utils/Delegate.h | 11 +- src/lib/utils/datetime.cpp | 9 +- src/server/sparkd/cmds/md/cmds/Run.cpp | 6 +- 64 files changed, 4180 insertions(+), 536 deletions(-) create mode 100644 src/lib/api/md/BinanceMarketDataConfig.proto delete mode 100644 src/lib/api/md/DepthStreamConfig.proto create mode 100644 src/lib/api/md/FTXMarketDataConfig.proto create mode 100644 src/lib/collections/DiscreteValuesFilter.h create mode 100644 src/lib/collections/Find.h create mode 100644 src/lib/collections/Remove.h create mode 100644 src/lib/config/defaults/MarketDataConfig.conf create mode 100644 src/lib/core/BoolName.h create mode 100644 src/lib/core/NaN.h create mode 100644 src/lib/core/StringContains.h create mode 100644 src/lib/core/StringFinder.h create mode 100644 src/lib/core/StringJoiner.h create mode 100644 src/lib/io/JsonReader.cpp create mode 100644 src/lib/io/JsonReader.h create mode 100644 src/lib/io/JsonType.h create mode 100644 src/lib/md/CCApiEventDemuxer.cpp create mode 100644 src/lib/md/CCApiEventDemuxer.h delete mode 100644 src/lib/md/CCApiListenerAdaptor.cpp delete mode 100644 src/lib/md/CCApiListenerAdaptor.h create mode 100644 src/lib/md/CCApiMarketDepthAdaptor.cpp create mode 100644 src/lib/md/CCApiMarketDepthAdaptor.h create mode 100644 src/lib/md/CCApiMessageHandler.h create mode 100644 src/lib/md/CCApiMessageLogger.cpp create mode 100644 src/lib/md/CCApiMessageLogger.h create mode 100644 src/lib/md/CCApiReader.h delete mode 100644 src/lib/md/CCApiSecurityParser.h create mode 100644 src/lib/md/CCApiTradeAdaptor.cpp create mode 100644 src/lib/md/CCApiTradeAdaptor.h create mode 100644 src/lib/md/CCApiTradeRecord.cpp create mode 100644 src/lib/md/CCApiTradeRecord.h create mode 100644 src/lib/md/ICCApiMessageHandler.cpp create mode 100644 src/lib/md/ICCApiMessageHandler.h create mode 100644 src/lib/md/MarketDataSubscription.cpp create mode 100644 src/lib/md/MarketDataSubscription.h create mode 100644 src/lib/md/ftx/FTXFuture.cpp create mode 100644 src/lib/md/ftx/FTXFuture.h create mode 100644 src/lib/md/ftx/FTXHistoricalPriceAdaptor.cpp create mode 100644 src/lib/md/ftx/FTXHistoricalPriceAdaptor.h create mode 100644 src/lib/md/ftx/FTXHistoricalPriceRecord.cpp create mode 100644 src/lib/md/ftx/FTXHistoricalPriceRecord.h create mode 100644 src/lib/md/ftx/FTXMarket.cpp create mode 100644 src/lib/md/ftx/FTXMarket.h create mode 100644 src/lib/md/ftx/FTXMessageType.h create mode 100644 src/lib/md/ftx/FTXSecurityRequestor.cpp create mode 100644 src/lib/md/ftx/FTXSecurityRequestor.h create mode 100644 src/lib/md/ftx/FTXSecurityType.h create mode 100644 src/lib/md/ftx/FTXSubscription.cpp create mode 100644 src/lib/md/ftx/FTXSubscription.h diff --git a/CMakeLists.txt b/CMakeLists.txt index b00d05f22..442bfd067 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ if(WIN32) else() add_compile_definitions(SPARK_ENABLE_CCAPI) add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA) + add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT) add_compile_definitions(CCAPI_ENABLE_EXCHANGE_FTX) add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE) add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US) diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index 6a67abf2d..76e7dca43 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -9,7 +9,8 @@ file( GLOB_RECURSE Config_Protos RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} config/protos/*.proto - api/domain/SecurityMetadataCollection.proto) + api/domain/SecurityMetadataCollection.proto + api/md/MarketDataConfig.proto) file( GLOB_RECURSE Service_Protos RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/src/lib/api/md/BinanceMarketDataConfig.proto b/src/lib/api/md/BinanceMarketDataConfig.proto new file mode 100644 index 000000000..e323518aa --- /dev/null +++ b/src/lib/api/md/BinanceMarketDataConfig.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package bts.api.md; + +option csharp_namespace = "bts.api.md"; +option cc_enable_arenas = true; + +message BinanceMarketDataConfig { + // TODO +} diff --git a/src/lib/api/md/DepthStreamConfig.proto b/src/lib/api/md/DepthStreamConfig.proto deleted file mode 100644 index 31f296d8f..000000000 --- a/src/lib/api/md/DepthStreamConfig.proto +++ /dev/null @@ -1,11 +0,0 @@ -syntax = "proto3"; - -import "lib/api/domain/Venue.proto"; - -package bts.api.md; - -message DepthStreamConfig { - bts.api.domain.Venue venue = 1; - repeated string symbols = 2; - uint32 num_levels = 3; -} diff --git a/src/lib/api/md/FTXMarketDataConfig.proto b/src/lib/api/md/FTXMarketDataConfig.proto new file mode 100644 index 000000000..32536f1d2 --- /dev/null +++ b/src/lib/api/md/FTXMarketDataConfig.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +import "lib/api/domain/SecurityCriteria.proto"; + +package bts.api.md; + +option csharp_namespace = "bts.api.md"; +option cc_enable_arenas = true; + +message FTXMarketDataConfig { + uint32 num_book_levels = 1; + bts.api.domain.SecurityCriteria security_criteria = 2; +} diff --git a/src/lib/api/md/MarketDataConfig.proto b/src/lib/api/md/MarketDataConfig.proto index 6fd886490..59a82e1ff 100644 --- a/src/lib/api/md/MarketDataConfig.proto +++ b/src/lib/api/md/MarketDataConfig.proto @@ -1,10 +1,16 @@ syntax = "proto3"; -import "lib/api/md/DepthStreamConfig.proto"; +import "lib/api/md/FTXMarketDataConfig.proto"; +import "lib/api/md/BinanceMarketDataConfig.proto"; + +option csharp_namespace = "bts.api.md"; +option cc_enable_arenas = true; package bts.api.md; message MarketDataConfig { - repeated bts.api.md.DepthStreamConfig depth_streams = 1; - uint32 num_dispatcher_threads = 2; + oneof venue { + bts.api.md.FTXMarketDataConfig ftx = 1; + bts.api.md.BinanceMarketDataConfig binance = 2; + } } diff --git a/src/lib/collections/DiscreteValuesFilter.h b/src/lib/collections/DiscreteValuesFilter.h new file mode 100644 index 000000000..a4294c040 --- /dev/null +++ b/src/lib/collections/DiscreteValuesFilter.h @@ -0,0 +1,98 @@ +#pragma once +#include +#include + +namespace bts::collections { + +template +class DiscreteValuesFilter { + public: + using Values = std::vector; + + bool is_active() const; + bool is_included(Value value) const; + void include(Value value); + bool include(const Values &values); + void exclude(Value value); + bool exclude(const Values &values); + + void reset(Values values, bool is_included); + + private: + bool are_values_included_ = false; + Values values_; + + // Adds a value to the list if it isn't there already + void add_value_(Value value); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +template +bool DiscreteValuesFilter::is_active() const { + return are_values_included_ || !values_.empty(); +} + +template +bool DiscreteValuesFilter::is_included(Value value) const { + return are_values_included_ == + (std ::find(values_.begin(), values_.end(), value) != values_.end()); +} + +template +void DiscreteValuesFilter::include(Value value) { + if (are_values_included_) { + add_value_(value); + return; + } + values_.erase(std::find(values_.begin(), values_.end(), value)); +} + +template +bool DiscreteValuesFilter::include(const Values &values) { + auto orig_sz = this->values_.size(); + auto orig_is_included = are_values_included_; + for (auto t : values) { + include(t); + } + return this->values_.size() != orig_sz || + orig_is_included != are_values_included_; +} + +template +void DiscreteValuesFilter::exclude(Value value) { + if (are_values_included_) { + values_.erase(std::find(values_.begin(), values_.end(), value)); + return; + } + add_value_(value); +} + +template +bool DiscreteValuesFilter::exclude(const Values &values) { + auto orig_sz = this->values_.size(); + auto orig_is_included = are_values_included_; + for (auto t : values) { + exclude(t); + } + return this->values_.size() != orig_sz || + orig_is_included != are_values_included_; +} + +template +void DiscreteValuesFilter::reset(Values values, bool is_included) { + this->values_ = std::move(values); + are_values_included_ = is_included; +} + +template +void DiscreteValuesFilter::add_value_(Value value) { + auto it = std::find(values_.begin(), values_.end(), value); + if (it == values_.end()) { + values_.push_back(value); + } +} + +} // namespace bts::collections diff --git a/src/lib/collections/Find.h b/src/lib/collections/Find.h new file mode 100644 index 000000000..c91e4c1b7 --- /dev/null +++ b/src/lib/collections/Find.h @@ -0,0 +1,6 @@ +#pragma once + +#define FIND(range, value) std::find(range.begin(), range.end(), value) + +#define FIND_IF(range, predicate) \ + std::find_if(range.begin(), range.end(), predicate) diff --git a/src/lib/collections/Remove.h b/src/lib/collections/Remove.h new file mode 100644 index 000000000..bfa7ad587 --- /dev/null +++ b/src/lib/collections/Remove.h @@ -0,0 +1,58 @@ +#pragma once +#include +#include +#include + +// Removes all occurrences of value +#define REMOVE(container, value) \ + container.erase( \ + std::remove(container.begin(), container.end(), value), \ + container.end()) + +// Removes all values that satisfy the predicate +#define REMOVE_IF(container, predicate) \ + container.erase( \ + std::remove_if(container.begin(), container.end(), predicate), \ + container.end()) + +// Removes first occurrence of value +#define REMOVE_FIRST(range, value) \ + range.erase(std::find(range.begin(), range.end(), value)) + +// Removes first value that satisfies the predicate +#define REMOVE_FIRST_IF(range, predicate) \ + range.erase(std::find_if(range.begin(), range.end(), predicate)) + +namespace bts::collections { + +class Remove { + public: + template + using TCount = typename std::result_of::type; + + template + static TCount remove(Range &range, Value value); + + template + static TCount remove_if(Range &range, Predicate predicate); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +template +Remove::TCount Remove::remove(Range &range, Value value) { + auto sz_before = range.size(); + REMOVE_RANGE(range, value); + return sz_before - range.size(); +} + +template +Remove::TCount Remove::remove_if(Range &range, Predicate predicate) { + auto sz_before = range.size(); + REMOVE_RANGE_IF(range, predicate); + return sz_before - range.size(); +} + +} // namespace bts::collections diff --git a/src/lib/config/ConfigurationState.cpp b/src/lib/config/ConfigurationState.cpp index f30d882a8..7fa2a2140 100644 --- a/src/lib/config/ConfigurationState.cpp +++ b/src/lib/config/ConfigurationState.cpp @@ -103,8 +103,7 @@ string ConfigurationState::settings_path() { void ConfigurationState::initialize_defaults(IConfigLog &log) { initialize_defaults( - log, - {std::string("lib/config/protos/"), std::string("lib/api/domain/")}); + log, {"lib/config/protos/", "lib/api/domain/", "lib/api/md/"}); } void ConfigurationState::initialize_defaults( @@ -545,10 +544,12 @@ void ConfigurationState::generate_all_config_proto( // add a meta type and field for each generated message proto::DescriptorPool::generated_pool(); - auto find_proto_file = [&proto_dirs](const std::string &name) -> const proto::FileDescriptor * { + auto find_proto_file = + [&proto_dirs](const std::string &name) -> const proto::FileDescriptor * { for (const auto &proto_dir : proto_dirs) { - const auto * res = proto::DescriptorPool::generated_pool()->FindFileByName( - proto_dir + name); + const auto *res = + proto::DescriptorPool::generated_pool()->FindFileByName( + proto_dir + name); if (res != nullptr) { return res; } @@ -615,8 +616,8 @@ bool is_valid_key_descriptor(const proto::Descriptor *property_desc) { return false; } -const proto::Message * -ConfigurationState::try_get_message(const std::string &property_name) { +const proto::Message *ConfigurationState::try_get_message( + const std::string &property_name) { initialize_settings(); const proto::Message *message; @@ -655,8 +656,8 @@ const proto::Message *ConfigurationState::try_get_message_or_default( return nullptr; } -const proto::Message & -ConfigurationState::get_message(const proto::Descriptor *property_desc) { +const proto::Message &ConfigurationState::get_message( + const proto::Descriptor *property_desc) { if (!is_valid_key_descriptor(property_desc)) { throw InvalidKeyException(property_desc->name().c_str()); } @@ -711,8 +712,8 @@ const proto::Message *ConfigurationState::try_get_value( return nullptr; } -const google::protobuf::Descriptor * -ConfigurationState::get_descriptor_by_name(const std::string &name) { +const google::protobuf::Descriptor *ConfigurationState::get_descriptor_by_name( + const std::string &name) { const proto::Message *pmsg = _defaults.get(); if (pmsg) { // Get the metadata descriptor (Meta = rule+value fields) diff --git a/src/lib/config/defaults/MarketData.conf b/src/lib/config/defaults/MarketData.conf index 5ff4b5df8..c79576013 100644 --- a/src/lib/config/defaults/MarketData.conf +++ b/src/lib/config/defaults/MarketData.conf @@ -2,7 +2,7 @@ # Common Market data options # -MarketDataConfig { +HistoricMarketDataConfig { rule: "DeploymentType.type == 'PROD'" value { use_sim: false @@ -45,12 +45,12 @@ CMEOptions { } } -ICEPublisherOptions { - value { - network_interface: "eth1" - } -} - +ICEPublisherOptions { + value { + network_interface: "eth1" + } +} + #to make this work in dev clone the operations repo #go here: c:\\code\\bts\\operations\\ice\\ICE_iMpactMulticastFeed_GettingStarted-1_1_36\\bin\\easyTunnel> #modify the easyTunnelProxy.properties file to replace diff --git a/src/lib/config/defaults/MarketDataConfig.conf b/src/lib/config/defaults/MarketDataConfig.conf new file mode 100644 index 000000000..4d6ebe0bb --- /dev/null +++ b/src/lib/config/defaults/MarketDataConfig.conf @@ -0,0 +1,14 @@ +MarketDataConfig { + value { + ftx { + security_criteria { + clauses { + security_types: FUTURE + security_types: SPOT + symbols: "BTC" + symbols: "ETH" + } + } + } + } +} diff --git a/src/lib/config/protos/market_data.proto b/src/lib/config/protos/market_data.proto index e84c36e93..b3f6c663f 100644 --- a/src/lib/config/protos/market_data.proto +++ b/src/lib/config/protos/market_data.proto @@ -1,7 +1,7 @@ syntax = 'proto2'; package bts.md; -message MarketDataConfig { +message HistoricMarketDataConfig { // Allows you to turn off in-proc market data entirely optional bool use_md = 1 [ default = true ]; optional bool use_onixs = 2 [ default = true ]; @@ -251,4 +251,4 @@ message FeedHandlerConfig { repeated string channel = 1; optional string network_interface_a = 3 [ default = "0.0.0.0" ]; optional string network_interface_b = 4 [ default = "" ]; -} \ No newline at end of file +} diff --git a/src/lib/core/BoolName.h b/src/lib/core/BoolName.h new file mode 100644 index 000000000..c21c6e5d0 --- /dev/null +++ b/src/lib/core/BoolName.h @@ -0,0 +1,39 @@ +#pragma once + +namespace bts::core { + +class BoolName { + public: + explicit constexpr BoolName(bool value); + + [[nodiscard]] static constexpr const char *true_false(bool value); + [[nodiscard]] static constexpr const char *yes_no(bool value); + [[nodiscard]] static constexpr const char *y_n(bool value); + [[nodiscard]] static constexpr const char *on_off(bool value); + + private: + bool value_; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +constexpr const char *BoolName::true_false(bool value) { + return value ? "true" : "false"; +} + +constexpr const char *BoolName::yes_no(bool value) { + return value ? "yes" : "no"; +} + +constexpr const char *BoolName::y_n(bool value) { + // NOLINTNEXTLINE + return value ? "y" : "n"; +} + +constexpr const char *BoolName::on_off(bool value) { + return value ? "on" : "off"; +} + +} // namespace bts::core diff --git a/src/lib/core/NaN.h b/src/lib/core/NaN.h new file mode 100644 index 000000000..e9a46102e --- /dev/null +++ b/src/lib/core/NaN.h @@ -0,0 +1,24 @@ +#pragma once +#include + +namespace bts::core { + +class NaN { + public: + static constexpr double value(); + static constexpr float valuef(); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +constexpr double NaN::value() { + return std::numeric_limits::quiet_NaN(); +} + +constexpr float NaN::valuef() { + return std::numeric_limits::quiet_NaN(); +} + +} // namespace bts::core diff --git a/src/lib/core/StringContains.h b/src/lib/core/StringContains.h new file mode 100644 index 000000000..30fae9f7e --- /dev/null +++ b/src/lib/core/StringContains.h @@ -0,0 +1,70 @@ +#pragma once +#include +#include +#include + +namespace bts::core { + +class StringContains { + public: + template + using AllSame = std::enable_if_t...>>; + + template + using AllString = AllSame; + + constexpr bool contains(const std::string &str, const std::string &substr); + constexpr bool contains(const std::string &str, const char *substr); + constexpr bool contains(const char *str, const char *substr); + + template + constexpr bool contains( + const std::string &str, const std::string &substr, Ts... substrs); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +constexpr bool StringContains::contains( + const std::string &str, const std::string &substr) { + return str.find(substr) != std::string::npos; +} + +constexpr bool StringContains::contains( + const std::string &str, const char *substr) { + return str.find(substr) != std::string::npos; +} + +constexpr bool StringContains::contains(const char *str, const char *substr) { + if (str[0] == '\0') { + return false; + } + if (substr[0] == '\0') { + return false; + } + + for (std::size_t i = 0; str[i] != '\0'; i++) { + bool is_match = true; + for (std::size_t j = 0; substr[j] < '\0'; j++) { + if (str[i + j] != substr[j]) { + is_match = false; + break; + } + } + if (is_match) { + return true; + } + } + return false; +} + +template +constexpr bool StringContains::contains( + const std::string &str, const std::string &substr, Ts... substrs) { + return contains(str, substr) || contains(str, substrs...); +} + +#define STRING_CONTAINS(x, ...) (bts::StringContains(x, __VA_ARGS__)) + +} // namespace bts::core diff --git a/src/lib/core/StringFinder.h b/src/lib/core/StringFinder.h new file mode 100644 index 000000000..7a709dc11 --- /dev/null +++ b/src/lib/core/StringFinder.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include + +namespace bts::core { + +class StringFinder { + public: + static std::size_t find(std::string str, char ch, int occurrence = 1); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::size_t StringFinder::find( + std::string str, char ch, int occurrence) { + int occur = 0; + + // Loop to find the Nth + // occurrence of the character + for (std::size_t i = 0; i < str.length(); i++) { + if (str[i] == ch) { + occur += 1; + } + if (occur == occurrence) + return i; + } + return std::string::npos; +} + +} // namespace bts::core diff --git a/src/lib/core/StringJoiner.h b/src/lib/core/StringJoiner.h new file mode 100644 index 000000000..76a8a1d85 --- /dev/null +++ b/src/lib/core/StringJoiner.h @@ -0,0 +1,178 @@ +#pragma once +#include +#include +#include +#include + +namespace bts::core { + +class StringJoiner { + public: + template + static std::ostream &join( + const std::string &delimiter, std::ostream &out, Arg arg); + + template + static std::string join(const std::string &delimiter, Arg arg); + + template + static std::ostream &join( + const std::string &delimiter, std::ostream &out, Arg arg, Args... args); + + template + static std::string join(const std::string &delimiter, Arg arg, Args... args); + + template + static std::ostream &join( + Iter first, Iter last, const std::string &delimiter, std::ostream &out); + + template + static std::string join(Iter first, Iter last, const std::string &delimiter); + + /** + * Converts the args to strings (using the standard output operator), then + * joins the non-empty ones with the specified delimiter. + */ + template + static std::string join_non_empties(const std::string &delimiter, Arg arg); + + /** + * Converts the args to strings (using the standard output operator), then + * joins the non-empty ones with the specified delimiter. + */ + template + static std::string join_non_empties( + const std::string &delimiter, Arg arg, Args... args); + + /** + * Converts the args to strings (using the standard output operator), then + * joins the non-empty ones with the specified delimiter. + */ + template + static std::ostream &join_non_empties( + const std::string &delimiter, std::ostream &out, Arg arg); + + /** + * Converts the args to strings (using the standard output operator), then + * joins the non-empty ones with the specified delimiter. + */ + template + static std::ostream &join_non_empties( + const std::string &delimiter, std::ostream &out, Arg arg, Args... args); + + private: + template + static std::ostream &join_non_empties_( + const std::string &delimiter, std::ostream &out, Arg arg, Args... args); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +template +std::ostream &StringJoiner::join( + const std::string & /*delimiter*/, std::ostream &out, Arg arg) { + // special case for single item (don't use delimiter) + out << arg; + return out; +} + +template +std::string StringJoiner::join(const std::string & /*delimiter*/, Arg arg) { + // special case for single item (don't use delimiter) + std::stringstream out; + out << arg; + return out.str(); +} + +template +std::ostream &StringJoiner::join( + const std::string &delimiter, std::ostream &out, Arg arg, Args... args) { + out << arg << delimiter; + return join(delimiter, out, args...); +} + +template +std::string StringJoiner::join( + const std::string &delimiter, Arg arg, Args... args) { + std::stringstream out; + out << arg << delimiter; + join(delimiter, out, args...); + return out.str(); +} + +template +std::ostream &StringJoiner::join( + Iter it, Iter last, const std::string &delimiter, std::ostream &out) { + if (it == last) { + return out; + } + out << *it; + it++; + for (; it != last; it++) { + out << delimiter << *it; + } + return out; +} + +template +std::string StringJoiner::join( + Iter it, Iter last, const std::string &delimiter) { + std::stringstream out; + join(it, last, delimiter, out); + return out.str(); +} + +template +std::string StringJoiner::join_non_empties( + const std::string & /*delimiter*/, Arg arg) { + std::stringstream ss; + ss << arg; + return ss.str(); +} + +template +std::string StringJoiner::join_non_empties( + const std::string &delimiter, Arg arg, Args... args) { + std::stringstream ss; + std::ostream &out = ss; + join_non_empties(delimiter, out, arg, args...); + return ss.str(); +} + +template +std::ostream &StringJoiner::join_non_empties( + const std::string & /*delimiter*/, std::ostream &out, Arg arg) { + out << arg; + return out; +} + +template +std::ostream &StringJoiner::join_non_empties( + const std::string &delimiter, std::ostream &out, Arg arg, Args... args) { + std::stringstream ss; + ss << arg; + auto arg_str = ss.str(); + if (arg_str.empty()) { + return join_non_empties(delimiter, out, args...); + } + out << arg; + return join_non_empties_(delimiter, out, args...); +} + +template +std::ostream &StringJoiner::join_non_empties_( + const std::string &delimiter, std::ostream &out, Arg arg, Args... args) { + std::stringstream ss; + ss << arg; + auto arg_str = ss.str(); + if (arg_str.empty()) { + return join_non_empties_(delimiter, out, args...); + } + out << delimiter << arg_str; + join(delimiter, out, args...); + return out; +} + +} // namespace bts::core diff --git a/src/lib/io/JsonReader.cpp b/src/lib/io/JsonReader.cpp new file mode 100644 index 000000000..d2ce6ad85 --- /dev/null +++ b/src/lib/io/JsonReader.cpp @@ -0,0 +1,420 @@ +#include "JsonReader.h" +#include "lib/core/TimeStamp.h" +#include "lib/logging/logging.h" +#include "lib/utils/TimeStamp.h" +#include "lib/utils/datetime.h" +#include +#include +#include + +namespace bts::io { + +using core::TimeStamp; +using std::int32_t; +using std::int64_t; +using std::numeric_limits; +using std::stoull; +using std::string; +using std::uint32_t; +using std::uint64_t; + +// 'read' fn bodies should be wrapped between BEGIN/END_READ_JSON tags. +// These macros abstract out repetative exception handling code (and a nullptr check). + +#define BEGIN_READ_JSON \ + if (!json_) { \ + return false; \ + } \ + try + +#define END_READ_JSON(type_name) \ + catch (const std::exception &e) { \ + LOG(Gen, Diag, "Failed to parse json to '{}'. Exception: {}", \ + std::string{type_name}, std::string{e.what()}); \ + } \ + catch (...) { \ + bts_assert_msgf( \ + false, "Failed to parse json to '{}'. Exception: unknown", \ + std::string{type_name}); \ + } \ + return false; + +// The idea here is we parse numbers to wide types (either int64, uint64, or double), +// then cast that value to the (potentially narrower) output type. This macro evaluates +// to true if 'wide_val' is within the bounds of the result type (otherwise false). +#define NUM_WITHIN_BOUNDS(wide_val, result_t) \ + (wide_val >= \ + static_cast(numeric_limits::min()) && \ + wide_val <= \ + static_cast(numeric_limits::max())) + +static const nlohmann::json *get_field_json( + const nlohmann::json *json, const std::string &name) { + if (!json || !json->is_object()) { + return nullptr; + } + auto it = json->find(name); + if (it == json->end()) { + return nullptr; + } + return &it.value(); +} + +JsonReader::JsonReader(const nlohmann::json *json) : json_(json) {} + +JsonReader::JsonReader(const nlohmann::json *json, const std::string &name) + : json_(get_field_json(json, name)) {} + +bool JsonReader::read(const string &name, string &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(const string &name, int32_t &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(const string &name, uint32_t &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(const string &name, int64_t &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(const string &name, uint64_t &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(const string &name, double &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(const string &name, TimeStamp &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read_epoch_nanos(const string &name, TimeStamp &value) const { + return JsonReader{json_, name}.read_epoch_nanos(value); +} + +bool JsonReader::read_epoch_micros(const string &name, TimeStamp &value) const { + return JsonReader{json_, name}.read_epoch_micros(value); +} + +bool JsonReader::read_epoch_millis(const string &name, TimeStamp &value) const { + return JsonReader{json_, name}.read_epoch_millis(value); +} + +bool JsonReader::read(const string &name, nlohmann::json &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(int32_t &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::number_float: { + const auto d = json_->get(); + result = static_cast(d); + if (result != d) { + LOG(Gen, Warn, + "Potential loss of data converting field of type " + "'number_float' ({}) to int32 ({})", + d, result); + } + return true; + } + case nlohmann::json::value_t::number_integer: { + const auto il = json_->get(); + if (!NUM_WITHIN_BOUNDS(il, int32_t)) { + LOG(Gen, Error, + "Failed to convert field of type 'number_integer' ({}) to " + "uint32: value is negative.", + il); + } + result = static_cast(il); + return true; + } + case nlohmann::json::value_t::number_unsigned: { + const auto ul = json_->get(); + if (ul > static_cast(numeric_limits::max())) { + LOG(Gen, Error, + "Failed to convert field of type 'number_unsigned' ({}) to " + "int32: value is too large.", + ul); + return false; + } + result = static_cast(ul); + return true; + } + case nlohmann::json::value_t::string: { + const auto str = json_->get(); + result = std::stoi(str); + return true; + } + default: + return false; + } + } + END_READ_JSON("int32") +} + +bool JsonReader::read(uint32_t &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::number_float: { + const auto d = json_->get(); + result = static_cast(d); + if (result != d) { + LOG(Gen, Warn, + "Potential loss of data converting field of type " + "'number_float' ({}) to uint32 ({})", + d, result); + } + return true; + } + case nlohmann::json::value_t::number_integer: { + const auto i = json_->get(); + if (!NUM_WITHIN_BOUNDS(i, uint32_t)) { + LOG(Gen, Error, + "Failed to convert field of type 'number_integer' ({}) to " + "uint32: value is negative.", + i); + return false; + } + result = static_cast(i); + return true; + } + case nlohmann::json::value_t::number_unsigned: { + const auto ul = json_->get(); + if (ul > static_cast(numeric_limits::max())) { + LOG(Gen, Error, + "Failed to convert field of type 'number_unsigned' ({}) to " + "int32: value is too large.", + ul); + return false; + } + result = static_cast(ul); + return true; + } + case nlohmann::json::value_t::string: { + const auto str = json_->get(); + result = std::stoul(str); + return true; + } + default: + return false; + } + } + END_READ_JSON("uint32") +} + +bool JsonReader::read(int64_t &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::number_float: { + const auto d = json_->get(); + if (result != d) { + LOG(Gen, Warn, + "Potential loss of data converting field of type " + "'number_float' ({}) to int64 ({})", + d, result); + } + result = static_cast(d); + return true; + } + case nlohmann::json::value_t::number_integer: { + result = json_->get(); + return true; + } + case nlohmann::json::value_t::number_unsigned: { + const auto ul = json_->get(); + if (NUM_WITHIN_BOUNDS(ul, int64_t)) { + LOG(Gen, Error, + "Potential loss of data converting field of type " + "'number_unsigned' ({}) to int64 ({})", + ul, result); + return false; + } + result = static_cast(ul); + return true; + } + case nlohmann::json::value_t::string: { + const auto str = json_->get(); + result = std::stoul(str); + return true; + } + default: + return false; + } + } + END_READ_JSON("int64") +} + +bool JsonReader::read(uint64_t &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::number_float: { + const auto d = json_->get(); + if (result != d) { + LOG(Gen, Warn, + "Potential loss of data converting field of type " + "'number_float' ({}) to uint64 ({})", + d, result); + } + result = static_cast(d); + return true; + } + case nlohmann::json::value_t::number_integer: { + const auto il = json_->get(); + if (NUM_WITHIN_BOUNDS(il, uint64_t)) { + LOG(Gen, Error, + "Potential loss of data converting field of type " + "'number_integer' ({}) to uint64 ({})", + il, result); + return false; + } + result = static_cast(il); + return true; + } + case nlohmann::json::value_t::number_unsigned: { + result = json_->get(); + return true; + } + case nlohmann::json::value_t::string: { + const auto str = json_->get(); + result = std::stoul(str); + return true; + } + default: + return false; + } + } + END_READ_JSON("uint64") +} + +bool JsonReader::read(double &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::number_float: + case nlohmann::json::value_t::number_integer: + case nlohmann::json::value_t::number_unsigned: + json_->get_to(result); + return true; + case nlohmann::json::value_t::string: { + const auto str = json_->get(); + result = std::stod(str); + return true; + } + default: + return false; + } + } + END_READ_JSON("double") +} + +bool JsonReader::read(string &result) const { + BEGIN_READ_JSON { + json_->get_to(result); + return true; + } + END_READ_JSON("string") +} + +bool JsonReader::read(nlohmann::json &result) const { + BEGIN_READ_JSON { + json_->get_to(result); + return true; + } + END_READ_JSON("json-object") +} + +bool JsonReader::parse_time_stamp_(const string &str, TimeStamp &value) const { + return parse_iso_time_stamp_(str, value) || + parse_epoch_nanos_time_stamp_(str, value); +} + +bool JsonReader::parse_iso_time_stamp_( + const string &str, TimeStamp &value) const { + value = TimeStamp::from_iso_string(str); + return true; +} + +bool JsonReader::parse_epoch_nanos_time_stamp_( + const string &str, TimeStamp &value) const { + uint64_t nanos = stoull(str); + value = TimeStamp{nanos}; + return true; +} + +bool JsonReader::read(TimeStamp &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::null: + case nlohmann::json::value_t::object: + case nlohmann::json::value_t::array: + case nlohmann::json::value_t::boolean: + case nlohmann::json::value_t::binary: + case nlohmann::json::value_t::discarded: + return false; + case nlohmann::json::value_t::string: { + const auto str = json_->get(); + return parse_iso_time_stamp_(str, result) || + parse_epoch_nanos_time_stamp_(str, result); + } + case nlohmann::json::value_t::number_integer: + case nlohmann::json::value_t::number_unsigned: + case nlohmann::json::value_t::number_float: + // Maybe we could assume seconds here, idk though...when in doubt, prob + // better to be cautious than cavalier + return false; + } + } + END_READ_JSON("TimeStamp") +} + +bool JsonReader::read_iso( + const std::string &name, core::TimeStamp &result) const { + return JsonReader{json_, name}.read_iso(result); +} + +bool JsonReader::read_iso(core::TimeStamp &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::string: { + return parse_iso_time_stamp_(json_->get(), result); + } + default: + return false; + } + } + END_READ_JSON("TimeStamp") +} + +bool JsonReader::read_epoch_nanos(TimeStamp &value) const { + int64_t nanos; + if (!read(nanos)) { + return false; + } + value = TimeStamp::from_nanos(nanos); + return true; +} + +bool JsonReader::read_epoch_micros(TimeStamp &value) const { + int64_t micros; + if (!read(micros)) { + return false; + } + value = TimeStamp::from_micros(micros); + return true; +} + +bool JsonReader::read_epoch_millis(TimeStamp &value) const { + int64_t millis; + if (!read(millis)) { + return false; + } + value = TimeStamp::from_millis(millis); + return true; +} + +} // namespace bts::io diff --git a/src/lib/io/JsonReader.h b/src/lib/io/JsonReader.h new file mode 100644 index 000000000..022fb906b --- /dev/null +++ b/src/lib/io/JsonReader.h @@ -0,0 +1,123 @@ +#pragma once +#include "lib/core/TimeStamp.h" +#include +#include +#include +#include + +namespace bts::io { + +class JsonReader { + public: + explicit JsonReader(const nlohmann::json *json); + + JsonReader(const nlohmann::json *json, const std::string &name); + + bool read(const std::string &name, std::string &result) const; + bool read(const std::string &name, std::int32_t &result) const; + bool read(const std::string &name, std::uint32_t &result) const; + bool read(const std::string &name, std::int64_t &result) const; + bool read(const std::string &name, std::uint64_t &result) const; + bool read(const std::string &name, double &result) const; + bool read(const std::string &name, nlohmann::json &result) const; + bool read(const std::string &name, core::TimeStamp &result) const; + bool read_iso(const std::string &name, core::TimeStamp &result) const; + bool read_epoch_nanos( + const std::string &name, core::TimeStamp &result) const; + bool read_epoch_micros( + const std::string &name, core::TimeStamp &result) const; + bool read_epoch_millis( + const std::string &name, core::TimeStamp &result) const; + + template + bool read(const std::string &name, T &result) const; + + template + T read(const std::string &name) const; + + bool read(std::string &result) const; + bool read(std::int32_t &result) const; + bool read(std::uint32_t &result) const; + bool read(std::int64_t &result) const; + bool read(std::uint64_t &result) const; + bool read(double &result) const; + bool read(nlohmann::json &result) const; + bool read(core::TimeStamp &result) const; + bool read_iso(core::TimeStamp &result) const; + bool read_epoch_nanos(core::TimeStamp &result) const; + bool read_epoch_micros(core::TimeStamp &result) const; + bool read_epoch_millis(core::TimeStamp &result) const; + + template + bool read(T &result) const; + + /** + * @brief Reads a json field into \p result + * + * @details If the read fails, assigns \p default_value to \p result + * @return True if the read succeeds, otherwise false + */ + template + bool read_or(const std::string &name, T &result, T default_value) const; + + private: + const nlohmann::json *json_; + + bool parse_time_stamp_( + const std::string &str, core::TimeStamp &result) const; + bool parse_iso_time_stamp_( + const std::string &str, core::TimeStamp &result) const; + bool parse_epoch_nanos_time_stamp_( + const std::string &str, core::TimeStamp &result) const; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +template +inline bool JsonReader::read(const std::string &name, T &result) const { + if (!json_ || !json_->is_object()) { + return false; + } + auto it = json_->find(name); + if (it == json_->end()) { + return false; + } + it->get_to(result); + return true; +} + +template +inline bool JsonReader::read(T &result) const { + try { + json_->get_to(result); + return true; + } catch (...) { + const auto j_str = json_->dump(2); + LOG(md, Error, "Exception parsing json:\n{}", j_str); + return false; + } +} + +template +inline T JsonReader::read(const std::string &name) const { + bts_assert(json_ && json_->is_object()); + auto it = json_->find(name); + bts_assert_msgf(it != json_->end(), "json doesn't contain field '{}'", name); + T result; + it->get_to(result); + return result; +} + +template +inline bool JsonReader::read_or( + const std::string &name, T &result, T default_value) const { + if (read(name, result)) { + return true; + } + result = default_value; + return false; +} + +} // namespace bts::io diff --git a/src/lib/io/JsonType.h b/src/lib/io/JsonType.h new file mode 100644 index 000000000..41964334a --- /dev/null +++ b/src/lib/io/JsonType.h @@ -0,0 +1,47 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/core/EnumMacros.h" +#include +#include +#include + +namespace bts::io { + +// clang-format off +#define FOR_ALL_VALUES_T(per_value, t) \ + per_value(t, null) \ + per_value(t, object) \ + per_value(t, array) \ + per_value(t, string) \ + per_value(t, boolean) \ + per_value(t, number_integer) \ + per_value(t, number_unsigned) \ + per_value(t, number_float) \ + per_value(t, binary) \ + per_value(t, discarded) + +#define FOR_ALL_VALUES(per_value) FOR_ALL_VALUES_T(per_value, nlohmann::detail::value_t) +// clang-format on + +/** + * @brief A wrapper class around nlohmann::detail::value_t that provides + * basic functionality, such as name/parse fns, output operator, etc. + */ +class JsonType { + public: + using Value = nlohmann::detail::value_t; + ENUM_BASIC_MEMBERS(JsonType) +}; + +ENUM_OUTPUT_OPS(JsonType) +ENUM_NAME_DEF(JsonType, FOR_ALL_VALUES) +ENUM_PARSE_DEF(JsonType, FOR_ALL_VALUES) +ENUM_FROM_JSON(JsonType) +ENUM_EQUALITY_OPS(JsonType) + +// undefine macros +#undef FOR_ALL_VALUES +#undef FOR_ALL_VALUES_T + +} // namespace bts::io +#endif diff --git a/src/lib/md/CCApiCorrelationID.h b/src/lib/md/CCApiCorrelationID.h index 42b7a87aa..22b1b537b 100644 --- a/src/lib/md/CCApiCorrelationID.h +++ b/src/lib/md/CCApiCorrelationID.h @@ -1,57 +1,275 @@ #pragma once #ifdef SPARK_ENABLE_CCAPI -#include "lib/api/domain/Venue.pb.h" -#include "lib/md/MarketDataListenerID.h" - +#include "lib/core/FMT_IMPL.h" +#include "lib/core/StringFinder.h" +#include "lib/core/StringJoiner.h" +#include "lib/utils/bts_assert.h" #include -#include -#include +#include #include -#include +#include namespace bts::md { class CCApiCorrelationID { public: - static std::string fmt_market_depth( - api::domain::Venue venue, - std::string desc, - MarketDataListenerID listener_id); + static constexpr const char *delimiter(); - static std::optional< - std::tuple> - parse_market_depth(const std::string &corr_id_str); + static CCApiCorrelationID from_string(const std::string &str); + + template + static CCApiCorrelationID from_parts(Parts... parts); + + /** + * @brief Gets the number of parts + * + * @return constexpr std::size_t The number of parts + */ + constexpr std::size_t size() const; + + constexpr bool empty() const; + /** + * Gets a vector of the next \p len parts (or however many are available, if + * fewer than \p len). The cursor remains unchanged. + * + * @param len The number of parts to get + * @param require_len Whether or not to assert at least \p len parts are + * available + * + * @return A vector of pointers to the next \p len parts (or fewer if less + * than \p len are available) + */ + std::vector peek_parts( + std::size_t len = 1, bool require_len = true) const; + + std::vector peek_parts_at( + std::size_t index, std::size_t len = 1, bool require_len = true) const; + + /** + * Gets a vector of the next \p len parts (or however many are available, if + * fewer than \p len). The cursor is incremented by the size of the returned + * vector. + * + * @param len The number of parts to get + * @param require_len Whether or not to assert at least \p len parts are + * available to consume + * + * @return A vector of pointers to the next \p len parts (or fewer if less + * than \p len are available to consume) + */ + std::vector consume_parts( + std::size_t len, bool require_len = true); + + template + bool consume_part(T &value, TParse parse = {}); + + template + bool consume_parts(std::size_t len, T &value, TParse parse = {}); + + constexpr const std::string &to_string() const; + std::string peek_string(std::size_t len = 1, bool require_len = true) const; + std::string peek_string_at( + std::size_t index, std::size_t len = 1, bool require_len = true) const; + std::string consume_string(std::size_t len, bool require_len = true); + CCApiCorrelationID trim_front() const; + + private: + std::vector parts_; + std::size_t cursor_ = 0; + std::string str_; + + /** + * @brief Advances 'cursor_' by the specified amount. If 'str_' is non-null, + * it is trimmed the same number of parts (i.e., so 'cursor_' and 'str_' + * remain in sync). + * + * @details Don't mutate 'cursor_' outside of this method (or risk + * desynchronizing it with 'str_'). + * + * @param len + */ + void advance_(std::size_t len); }; +constexpr bool operator==( + const CCApiCorrelationID &lhs, const CCApiCorrelationID &rhs); + +constexpr bool operator!=( + const CCApiCorrelationID &lhs, const CCApiCorrelationID &rhs); + +std::ostream &operator<<( + std::ostream &out, const CCApiCorrelationID &corr_id) noexcept; + ////////////////////////////////////////////////// // Definition ////////////////////////////////////////////////// -inline std::string CCApiCorrelationID::fmt_market_depth( - api::domain::Venue venue, - std::string desc, - MarketDataListenerID listener_id) { - return fmt::format( - "{}.{}.{}", api::domain::Venue_Name(venue), desc, listener_id); +constexpr const char *CCApiCorrelationID::delimiter() { return "."; } + +inline CCApiCorrelationID CCApiCorrelationID::from_string( + const std::string &str) { + std::vector parts; + boost::algorithm::split(parts, str, [](auto c) { return c == '.'; }); + CCApiCorrelationID rv; + rv.parts_ = std::move(parts); + rv.cursor_ = 0; + rv.str_ = str; + return rv; } -inline std::optional< - std::tuple> -CCApiCorrelationID::parse_market_depth(const std::string &corr_id_str) { - std::vector parts; - boost::algorithm::split(parts, corr_id_str, [](auto c) { return c == '.'; }); - if (parts.size() != 3) { - return std::nullopt; +template +inline CCApiCorrelationID CCApiCorrelationID::from_parts(Parts... parts) { + // TODO: create the parts vector/string simultaneously (more efficient) + auto parts_str = core::StringJoiner::join(delimiter(), parts...); + return from_string(parts_str); +} + +inline std::vector CCApiCorrelationID::peek_parts( + std::size_t len, bool require_len) const { + return peek_parts_at(0, len, require_len); +} + +inline std::vector CCApiCorrelationID::peek_parts_at( + std::size_t index, std::size_t len, bool require_len) const { + std::vector rv; + auto adj_cursor = cursor_ + index; + std::size_t rem_len = + parts_.size() <= adj_cursor ? 0 : parts_.size() - adj_cursor; + if (len > rem_len) { + bts_assert_msgf( + !require_len, + "Expected at least {} more parts in correlation id ({} remain). " + "cursor: {}; total_parts_len: {}", + len, rem_len, cursor_, parts_.size()); + len = rem_len; } - api::domain::Venue venue; - if (!api::domain::Venue_Parse(parts.at(0), &venue)) { - return std::nullopt; + for (std::size_t i = 0; i < len; i++) { + rv.push_back(&parts_.at(adj_cursor + i)); } - auto sec_desc = parts.at(1); - auto listener_id = MarketDataListenerID{ - static_cast(std::stoul(parts.at(2)))}; - return std::make_tuple(venue, sec_desc, listener_id); + return rv; +} + +inline std::vector CCApiCorrelationID::consume_parts( + std::size_t len, bool require_len) { + auto rv = peek_parts(len, require_len); + advance_(rv.size()); + return rv; +} + +constexpr const std::string &CCApiCorrelationID::to_string() const { + return str_; +} + +inline std::string CCApiCorrelationID::peek_string( + std::size_t len, bool require_len) const { + return peek_string_at(0, len, require_len); +} + +inline std::string CCApiCorrelationID::peek_string_at( + std::size_t index, std::size_t len, bool require_len) const { + auto sub_parts = peek_parts_at(index, len, require_len); + std::string rv; + bool first = true; + for (const auto *p : sub_parts) { + rv.append(*p); + if (first) { + first = false; + } else { + rv.append(delimiter()); + } + } + return rv; +} + +inline std::string CCApiCorrelationID::consume_string( + std::size_t len, bool require_len) { + auto sub_parts = consume_parts(len, require_len); + std::string rv; + bool first = true; + for (const auto *p : sub_parts) { + rv.append(*p); + if (first) { + first = false; + } else { + rv.append(delimiter()); + } + } + return rv; +} + +template +inline bool CCApiCorrelationID::consume_part(T &value, TParse parse) { + return consume_parts(value, parse, 1); +} + +template +inline bool CCApiCorrelationID::consume_parts( + std::size_t len, T &value, TParse parse) { + bts_assert(len > 0); + if (size() < len) { + return false; + } + auto str = consume_string(len); + return parse(str, &value); +} + +inline CCApiCorrelationID CCApiCorrelationID::trim_front() const { + if (cursor_ >= parts_.size()) { + return CCApiCorrelationID{}; + } + std::vector sub_parts{parts_.begin() + cursor_, parts_.end()}; + auto parts_str = fmt::to_string( + fmt::join(parts_.begin() + cursor_, parts_.end(), delimiter())); + CCApiCorrelationID rv; + rv.parts_ = std::move(sub_parts); + rv.cursor_ = 0; + rv.str_ = std::move(parts_str); + return rv; +} + +inline void CCApiCorrelationID::advance_(std::size_t len) { + if (len == 0) { + return; + } + cursor_ += len; + auto idx = core::StringFinder::find(str_, '.', len); + bts_assert(idx >= 0); + str_ = str_.substr(idx); +} + +constexpr std::size_t CCApiCorrelationID::size() const { + if (cursor_ > parts_.size()) { + return 0; + } + return parts_.size() - cursor_; +} + +constexpr bool CCApiCorrelationID::empty() const { return size() == 0; } + +constexpr bool operator==( + const CCApiCorrelationID &lhs, const CCApiCorrelationID &rhs) { + return lhs.to_string() == rhs.to_string(); +} + +constexpr bool operator!=( + const CCApiCorrelationID &lhs, const CCApiCorrelationID &rhs) { + return !(lhs.to_string() == rhs.to_string()); +} + +inline std::ostream &operator<<( + std::ostream &out, const CCApiCorrelationID &corr_id) noexcept { + out << corr_id.to_string(); + return out; } } // namespace bts::md + +template <> +struct std::hash { + std::size_t operator()(const bts::md::CCApiCorrelationID &corr_id) const { + return std::hash()(corr_id.to_string()); + } +}; + +FMT_IMPL(bts::md::CCApiCorrelationID, cid, "{}", cid.to_string()); #endif diff --git a/src/lib/md/CCApiEventDemuxer.cpp b/src/lib/md/CCApiEventDemuxer.cpp new file mode 100644 index 000000000..e7cc1a34a --- /dev/null +++ b/src/lib/md/CCApiEventDemuxer.cpp @@ -0,0 +1,166 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "CCApiEventDemuxer.h" +#include "lib/collections/DiscreteValuesFilter.h" +#include "lib/collections/Find.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/ICCApiMessageHandler.h" + +#include +#include +#include +#include +#include + +namespace bts::md { + +using bts::collections::DiscreteValuesFilter; +using std::string; +using std::unordered_map; +using std::vector; + +struct CCApiEventDemuxer::HandlerData { + ICCApiMessageHandler *handler; + DiscreteValuesFilter msg_type_filter; + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const vector &msgs, + const CCApiCorrelationID &corr_id) { + if (!msg_type_filter.is_active()) { + // Handler is listening to msgs of all types + handler->on_messages(event, session, msgs, corr_id); + return; + } + + std::vector filtered_msgs; + for (const auto *msg : msgs) { + if (msg_type_filter.is_included(msg->getType())) { + filtered_msgs.push_back(msg); + } + } + if (!filtered_msgs.empty()) { + handler->on_messages(event, session, filtered_msgs, corr_id); + } + } +}; + +struct CCApiEventDemuxer::PImpl { + std::size_t corr_id_len; + unordered_map> correlated_handlers; +}; + +CCApiEventDemuxer::CCApiEventDemuxer(std::size_t corr_id_len) + : me_(std::make_unique()) { + me_->corr_id_len = corr_id_len; +} + +CCApiEventDemuxer::~CCApiEventDemuxer() = default; + +bool CCApiEventDemuxer::processEvent( + const ccapi::Event &event, ccapi::Session *session) { + unordered_map> + demuxed_msgs; + for (const auto &msg : event.getMessageList()) { + switch (msg.getType()) { + case ccapi::Message::Type::SESSION_CONNECTION_UP: + LOG(md, Info, "Session connection up"); + continue; + case ccapi::Message::Type::SESSION_CONNECTION_DOWN: + LOG(md, Warn, "Session connection down"); + continue; + default: + break; + } + for (const auto &corr_id_str : msg.getCorrelationIdList()) { + auto child_corr_id = CCApiCorrelationID::from_string(corr_id_str); + // trim the front parts off (they're all the same) + child_corr_id.consume_parts(me_->corr_id_len); + auto [it, _] = demuxed_msgs.try_emplace(child_corr_id); + it->second.push_back(&msg); + } + } + + for (const auto &[hnd_corr_id, msgs] : demuxed_msgs) { + auto hnds_it = me_->correlated_handlers.find(hnd_corr_id); + if (hnds_it == me_->correlated_handlers.end()) { + continue; + } + for (auto &hnd : hnds_it->second) { + hnd.on_messages(event, session, msgs, hnd_corr_id); + } + } + return true; +} + +void CCApiEventDemuxer::on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID & /*corr_id*/) { + unordered_map> + demuxed_msgs; + for (const auto *msg : msgs) { + for (const auto &corr_id_str : msg->getCorrelationIdList()) { + auto handler_corr_id = CCApiCorrelationID::from_string(corr_id_str); + // trim the front parts off (they're all the same) + handler_corr_id.consume_parts(me_->corr_id_len); + auto [it, _] = demuxed_msgs.try_emplace(handler_corr_id); + it->second.push_back(msg); + } + } + + for (const auto &[handler_corr_id, dmsgs] : demuxed_msgs) { + auto hnds_it = me_->correlated_handlers.find(handler_corr_id); + if (hnds_it == me_->correlated_handlers.end()) { + LOG(md, Error, + "No handler registered with correlation id '{}' (msgs will be " + "ignored)", + handler_corr_id); + continue; + } + for (auto &hnd : hnds_it->second) { + hnd.on_messages(event, session, dmsgs, handler_corr_id); + } + } +} + +bool CCApiEventDemuxer::has_handler(const CCApiCorrelationID &corr_id) const { + return me_->correlated_handlers.find(corr_id) != + me_->correlated_handlers.end(); +} + +bool CCApiEventDemuxer::add_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + vector msg_types) { + auto &hnds = me_->correlated_handlers.try_emplace(corr_id).first->second; + auto it = FIND_IF( + hnds, [handler](const auto &hnd) { return hnd.handler == handler; }); + if (it != hnds.end()) { + return it->msg_type_filter.include(msg_types); + } + auto &hnd = hnds.emplace_back(); + hnd.handler = handler; + if (!msg_types.empty()) { + hnd.msg_type_filter.reset(std::move(msg_types), true); + } + return true; +} + +bool CCApiEventDemuxer::remove_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + vector types) { + auto corr_hnd_it = me_->correlated_handlers.find(corr_id); + if (corr_hnd_it == me_->correlated_handlers.end()) { + return false; + } + auto &hnds = corr_hnd_it->second; + auto it = FIND_IF( + hnds, [handler](const auto &hnd) { return hnd.handler == handler; }); + return it != hnds.end() && it->msg_type_filter.exclude(types); +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiEventDemuxer.h b/src/lib/md/CCApiEventDemuxer.h new file mode 100644 index 000000000..02e208eda --- /dev/null +++ b/src/lib/md/CCApiEventDemuxer.h @@ -0,0 +1,82 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/md/ICCApiMessageHandler.h" +#include +#include +#include +#include +#include + +namespace bts::md { + +/** + * Demuxes ccapi::Events by CorrelationID to ICCApiMessageHandlers + */ +class CCApiEventDemuxer : public ccapi::EventHandler, + public ICCApiMessageHandler { + public: + explicit CCApiEventDemuxer(std::size_t corr_id_len = 0); + ~CCApiEventDemuxer() override; + + bool processEvent( + const ccapi::Event &event, ccapi::Session *session) override; + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override; + + /** + * @brief Determines whether or not a handler has been added to a particular + * correlation id + * + * @param corr_id The correlation id to check + * + * @return True if a handler has been added for the specified correlation id, + * otherwise false + */ + bool has_handler(const CCApiCorrelationID &corr_id) const; + + /** + * @brief Connects a handler to ccapi callbacks with the specified + * correlation id + * + * @param corr_id The correlation id of the callbacks the handler will + * receive + * @param handler The handler to connect + * + * @return True if the handler is added, otherwise False (i.e., the handler + * had already been added to the specified correlation id) + */ + bool add_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + std::vector types = {}); + + /** + * @brief Disconnects a handler from ccapi callbacks with the specified + * correlation id + * + * @param corr_id The correlation id of the callbacks the handler will no + * longer receive + * @param handler The handler to disconnect + * + * @return True if the handler is removed, otherwise False (either handler + * was already disconnected, or it wasn't added for the specified correlation + * id in the first place) + */ + bool remove_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + std::vector types = {}); + + private: + struct PImpl; + struct HandlerData; + + std::unique_ptr me_; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiListenerAdaptor.cpp b/src/lib/md/CCApiListenerAdaptor.cpp deleted file mode 100644 index 58142748f..000000000 --- a/src/lib/md/CCApiListenerAdaptor.cpp +++ /dev/null @@ -1,165 +0,0 @@ -#ifdef SPARK_ENABLE_CCAPI -#include "CCApiListenerAdaptor.h" -#include "lib/api/md/MarketDataConfig.pb.h" -#include "lib/collections/Contains.h" -#include "lib/domain/BUY_SELL.h" -#include "lib/domain/SecurityID.h" -#include "lib/logging/logging.h" -#include "lib/md/CCApiCorrelationID.h" -#include "lib/md/CCApiDepthWalker.h" -#include "lib/md/CCApiSecurityParser.h" -#include "lib/md/IMarketDataListener.h" -#include "lib/md/MarketDataListenerData.h" -#include "lib/persistence/IDGenerator.h" - -#include -#include -#include -#include - -namespace bts::md { - -using namespace bts::domain; -using std::string; -using std::unique_ptr; -using std::unordered_map; -using std::vector; - -struct CCApiListenerAdaptor::PImpl { - PImpl( - IMarketDataListener *const listener, - persistence::IDGenerator *const sec_id_store) - : listener(listener), sec_id_store(sec_id_store) {} - - IMarketDataListener *const listener; - - persistence::IDGenerator *const sec_id_store; - unordered_map> securities; - - void on_instruments(const ccapi::Message &msg, api::domain::Venue venue) { - vector all_secs; - auto new_secs = CCApiSecurityParser::parse(msg.getElementList()); - for (const auto &sec : new_secs) { - const auto &desc = sec.security_common.desc; - bts_assert(!desc.empty()); - auto sec_id = SecurityID{ - sec_id_store->get_or_add(api::domain::Venue_Name(venue), desc)}; - if (CONTAINS_KEY(securities, sec_id)) { - // don't replace existing secs (in case code downstream relies on - // ptr equality) - continue; - } - auto sec_ptr = std::make_unique(sec); - sec_ptr->security_common.id.value() = sec_id_store->get_or_add( - api::domain::Venue_Name(venue), sec_ptr->security_common.desc); - all_secs.push_back(sec_ptr.get()); - securities[sec_id] = std::move(sec_ptr); - } - listener->on_securities(all_secs); - } - - void on_instruments(const ccapi::Message &msg, const string &corr_id) { - // Correlation ID is the Venue - api::domain::Venue venue; - if (!api::domain::Venue_Parse(corr_id, &venue)) { - LOG(md, Error, "Failed to parse venue from ccapi correlation id '{}'", - corr_id); - return; - } - on_instruments(msg, venue); - } - - void on_book(const ccapi::Message &msg, SecurityID sec_id) { - if (msg.getType() != - ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH) { - return; - } - - Book book; - book.set_security_id(sec_id); - // The market data arrives in the following order: - // * bid levels (by price descending) - // * ask levels (by price ascending) - CCApiDepthWalker bid_walker{"BID_PRICE", "BID_SIZE", &book[BUY]}; - CCApiDepthWalker ask_walker{"ASK_PRICE", "ASK_SIZE", &book[SELL]}; - const auto &levels = msg.getElementList(); - for (const auto &level : levels) { - // Each level has 2 entries, either: - // ["BID_PRICE", "BID_SIZE"], or - // ["ASK_PRICE", "ASK_SIZE"] - const auto &props = level.getNameValueMap(); - for (const auto &[name, value] : props) { - bid_walker.parse_field(name, value) || - ask_walker.parse_field(name, value); - } - } - - // validate the book - bts_assert(bid_walker.price_idx == bid_walker.qty_idx); - bts_assert(ask_walker.price_idx == ask_walker.qty_idx); - bts_assert(book.is_sorted()); - bts_assert(book[BUY].count_populated() == bid_walker.price_idx); - bts_assert(book[SELL].count_populated() == ask_walker.price_idx); - listener->on_book(book); - } - - void on_book(const ccapi::Message &msg, const string &corr_id) { - // Correlation ID is a formatted string: - // ".." - const auto corr_id_res = CCApiCorrelationID::parse_market_depth(corr_id); - bts_assert(corr_id_res.has_value()); - const auto [venue, sec_desc, listener_id] = *corr_id_res; - LOG(md, Diag, "listener_id: {}", listener_id); // to build gcc - const auto sec_id = - sec_id_store->get_or_add(api::domain::Venue_Name(venue), sec_desc); - on_book(msg, SecurityID{sec_id}); - } - - SecurityID get_sec_id( - const api::domain::Venue venue, const string &sec_desc) { - const auto sec_id = - sec_id_store->get_or_add(api::domain::Venue_Name(venue), sec_desc); - return SecurityID{sec_id}; - } -}; - -CCApiListenerAdaptor::CCApiListenerAdaptor( - IMarketDataListener *const listener, - persistence::IDGenerator *const sec_id_store) - : me_(std::make_unique(listener, sec_id_store)) {} - -CCApiListenerAdaptor::~CCApiListenerAdaptor() = default; - -bool CCApiListenerAdaptor::processEvent( - const ccapi::Event &event, ccapi::Session *) { - LOG(md, Diag, "Received ccapi event: {}", event.toString()); - const auto &msgs = event.getMessageList(); - for (const auto &msg : msgs) { - const auto &corr_ids = msg.getCorrelationIdList(); - for (const auto &corr_id : corr_ids) { - switch (msg.getType()) { - case ccapi::Message::Type::GET_INSTRUMENTS: { - // This message should only arrive with a RESPONSE event - bts_assert(event.getType() == ccapi::Event::Type::RESPONSE); - me_->on_instruments(msg, corr_id); - break; - } - case ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH: { - // This message should only arrive with a SUBSCRIPTION_DATA event - bts_assert( - event.getType() == ccapi::Event::Type::SUBSCRIPTION_DATA); - me_->on_book(msg, corr_id); - break; - } - default: - LOG(md, Warn, "Unexpected ccapi message type '{}'", - ccapi::Message::typeToString(msg.getType())); - break; - } - } - } - return true; -} - -} // namespace bts::md -#endif diff --git a/src/lib/md/CCApiListenerAdaptor.h b/src/lib/md/CCApiListenerAdaptor.h deleted file mode 100644 index 3f67d04fd..000000000 --- a/src/lib/md/CCApiListenerAdaptor.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once -#ifdef SPARK_ENABLE_CCAPI -#include "lib/api/domain/Venue.pb.h" - -#include -#include - -namespace bts::api::md { -class MarketDataConfig; -} - -namespace bts::persistence { -class IDGenerator; -} - -namespace bts::md { - -class IMarketDataListener; - -class CCApiListenerAdaptor : public ccapi::EventHandler { - public: - CCApiListenerAdaptor( - IMarketDataListener *const listener, - persistence::IDGenerator *const sec_id_store); - - ~CCApiListenerAdaptor(); - - bool processEvent( - const ccapi::Event &event, ccapi::Session *session) override; - - private: - struct PImpl; - std::unique_ptr me_; -}; - -} // namespace bts::md -#endif diff --git a/src/lib/md/CCApiMarketDepthAdaptor.cpp b/src/lib/md/CCApiMarketDepthAdaptor.cpp new file mode 100644 index 000000000..5ee549d06 --- /dev/null +++ b/src/lib/md/CCApiMarketDepthAdaptor.cpp @@ -0,0 +1,78 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "CCApiMarketDepthAdaptor.h" +#include "lib/domain/BUY_SELL.h" +#include "lib/domain/Book.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/CCApiDepthWalker.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/MarketDataListenerData.h" + +#include +#include +#include +#include +#include + +namespace bts::md { + +static bool parse(const ccapi::Message &msg, domain::Book &book) { + if (msg.getType() != ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH) { + return false; + } + + // The market data arrives in the following order: + // * bid levels (by price descending) + // * ask levels (by price ascending) + CCApiDepthWalker bid_walker{ + CCAPI_BEST_BID_N_PRICE, CCAPI_BEST_BID_N_SIZE, &book[BUY]}; + CCApiDepthWalker ask_walker{ + CCAPI_BEST_ASK_N_PRICE, CCAPI_BEST_ASK_N_SIZE, &book[SELL]}; + const auto &levels = msg.getElementList(); + for (const auto &level : levels) { + // Each level has 2 entries, either: + // ["BID_PRICE", "BID_SIZE"], or + // ["ASK_PRICE", "ASK_SIZE"] + const auto &props = level.getNameValueMap(); + for (const auto &[name, value] : props) { + bid_walker.parse_field(name, value) || + ask_walker.parse_field(name, value); + } + } + + // validate the book + bts_assert(bid_walker.price_idx == bid_walker.qty_idx); + bts_assert(ask_walker.price_idx == ask_walker.qty_idx); + bts_assert(book.is_sorted()); + bts_assert(book[BUY].count_populated() == bid_walker.price_idx); + bts_assert(book[SELL].count_populated() == ask_walker.price_idx); + return true; +} + +static domain::Book parse(const ccapi::Message &msg) { + domain::Book book; + parse(msg, book); + return book; +} + +CCApiMarketDepthAdaptor::CCApiMarketDepthAdaptor( + domain::SecurityID security_id, IMarketDataListener *listener) + : security_id_(security_id), listener_(listener) {} + +const domain::Book &CCApiMarketDepthAdaptor::last_book() const { + return last_book_; +} + +void CCApiMarketDepthAdaptor::on_messages( + const ccapi::Event & /*event*/, + ccapi::Session * /*session*/, + const std::vector &msgs, + const CCApiCorrelationID & /*corr_id*/) { + for (const auto *msg : msgs) { + auto book = parse(*msg); + book.set_security_id(security_id_); + listener_->on_book(book); + } +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiMarketDepthAdaptor.h b/src/lib/md/CCApiMarketDepthAdaptor.h new file mode 100644 index 000000000..c25bb9269 --- /dev/null +++ b/src/lib/md/CCApiMarketDepthAdaptor.h @@ -0,0 +1,44 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/domain/Book.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/IMarketDataListener.h" +#include + +namespace ccapi { +class Event; +class Message; +class Session; +} // namespace ccapi + +namespace bts::md { + +class CCApiCorrelationID; + +/** + * @brief A bts::md::ICCApiMessageHandler that translates + * market depth messages into books for an bts::md::IMarketDataListener + */ +class CCApiMarketDepthAdaptor : public ICCApiMessageHandler { + public: + CCApiMarketDepthAdaptor() = default; + + CCApiMarketDepthAdaptor( + domain::SecurityID security_id, IMarketDataListener *listener); + + const domain::Book &last_book() const; + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override; + + private: + domain::SecurityID security_id_; + IMarketDataListener *listener_; + domain::Book last_book_; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiMessageHandler.h b/src/lib/md/CCApiMessageHandler.h new file mode 100644 index 000000000..6f3cd0962 --- /dev/null +++ b/src/lib/md/CCApiMessageHandler.h @@ -0,0 +1,46 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/utils/Delegate.h" +#include +#include +#include + +namespace bts::md { + +/** + * @brief A proxy handler that emits messages to a bindable callback delegate. + * It is designed to be used with a CCApiSession. + * + * @details This class is useful for creating handlers without the overhead of + * defining a new class. It's especially convenient for cases where a 'master' + * class handles callbacks for multiple message types. If a proxy handler is + * used for each message type, rather than a single handler for all message + * types, one avoids the need to parse the message-type from the correlation id + * and demux the callback with switch cases. + */ +class CCApiMessageHandler : public ICCApiMessageHandler { + public: + using Callback = core::Delegate &, + const CCApiCorrelationID &)>; + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override { + callback_(event, session, msgs, corr_id); + } + + Callback &callback() { return callback_; } + + private: + Callback callback_; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiMessageLogger.cpp b/src/lib/md/CCApiMessageLogger.cpp new file mode 100644 index 000000000..b2146ca57 --- /dev/null +++ b/src/lib/md/CCApiMessageLogger.cpp @@ -0,0 +1,33 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "CCApiMessageLogger.h" +#include "lib/core/StringJoiner.h" +#include "lib/core/TimeStamp.h" +#include +#include +#include +#include + +namespace bts::md { + +using bts::core::StringJoiner; +using std::string; + +CCApiMessageLogger::CCApiMessageLogger(std::string name) + : name_(std::move(name)) {} + +void CCApiMessageLogger::on_messages( + const ccapi::Event & /*event*/, + ccapi::Session * /*session*/, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) { + + auto msg_strs = msgs | ranges::views::transform([](const ccapi::Message *m) { + return m->toStringPretty(); + }); + auto msgs_str = StringJoiner::join(msg_strs.begin(), msg_strs.end(), "\n"); + LOG(md, Info, "Received CCApi messages. name: {}; corr_id: {}; msgs:\n{}", + name_, corr_id, msgs_str); +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiMessageLogger.h b/src/lib/md/CCApiMessageLogger.h new file mode 100644 index 000000000..6d16d3576 --- /dev/null +++ b/src/lib/md/CCApiMessageLogger.h @@ -0,0 +1,34 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/domain/Book.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/IMarketDataListener.h" +#include + +namespace ccapi { +class Event; +class Message; +class Session; +} // namespace ccapi + +namespace bts::md { + +class CCApiCorrelationID; + +class CCApiMessageLogger : public ICCApiMessageHandler { + public: + CCApiMessageLogger() = default; + explicit CCApiMessageLogger(std::string name); + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override; + + private: + std::string name_; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiReader.h b/src/lib/md/CCApiReader.h new file mode 100644 index 000000000..4874aa66f --- /dev/null +++ b/src/lib/md/CCApiReader.h @@ -0,0 +1,155 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/utils/bts_assert.h" +#include +#include +#include +#include + +namespace bts::md { + +class CCApiReader { + public: + using NameValues = std::map; + + explicit CCApiReader(const NameValues *name_values); + + bool read(const std::string &key, std::string &value) const; + bool read(const std::string &key, nlohmann::json &value) const; + + template + bool read(const std::string &key, Value &value) const; + + template + bool read(const std::string &key, Value &value, Parse parse = {}) const; + + private: + const NameValues *name_values_ptr_; + + const std::string *get_(const std::string &key) const; + + bool parse_(const std::string &str, double &value) const noexcept; + bool parse_(const std::string &str, std::int32_t &value) const noexcept; + bool parse_(const std::string &str, std::int64_t &value) const noexcept; + bool parse_(const std::string &str, std::uint32_t &value) const noexcept; + bool parse_(const std::string &str, std::uint64_t &value) const noexcept; + + template + bool parse_(const std::string &str, Value &value) const; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline CCApiReader::CCApiReader(const NameValues *name_values) + : name_values_ptr_(name_values) {} + +inline bool CCApiReader::read( + const std::string &key, std::string &value) const { + const auto *value_str = get_(key); + if (value_str == nullptr) { + return false; + } + value = *value_str; + return true; +} + +inline bool CCApiReader::read( + const std::string &key, nlohmann::json &value) const { + const auto *value_str = get_(key); + if (value_str == nullptr) { + return false; + } + auto js = nlohmann::json::parse(*value_str); + value = std::move(js); + return true; +} + +template +inline bool CCApiReader::read(const std::string &key, Value &value) const { + const auto *value_str = get_(key); + if (value_str == nullptr) { + return false; + } + return parse_(*value_str, value); +} + +template +inline bool CCApiReader::read( + const std::string &key, Value &value, Parse parse) const { + const auto *value_str = get_(key); + return value_str != nullptr && parse(*value_str, value); +} + +inline const std::string *CCApiReader::get_(const std::string &key) const { + bts_assert(name_values_ptr_ != nullptr); + auto it = name_values_ptr_->find(key); + return it == name_values_ptr_->end() ? nullptr : &it->second; +} + +template +inline bool CCApiReader::parse_(const std::string &str, Value &value) const { + std::stringstream ss; + ss << str; + ss >> value; + return true; +} + +inline bool CCApiReader::parse_( + const std::string &str, double &value) const noexcept { + try { + value = std::stod(str); + return true; + } catch (...) { + return false; + } +} + +inline bool CCApiReader::parse_( + const std::string &str, std::int32_t &value) const noexcept { + try { + value = std::stoi(str); + return true; + } catch (...) { + return false; + } +} + +inline bool CCApiReader::parse_( + const std::string &str, std::int64_t &value) const noexcept { + try { + value = std::stol(str); + return true; + } catch (...) { + return false; + } +} + +inline bool CCApiReader::parse_( + const std::string &str, std::uint32_t &value) const noexcept { + try { + const auto ul = std::stoul(str); + // the only error stoul won't throw + if (ul > std::numeric_limits::max()) { + return false; + } + value = static_cast(value); + return true; + } catch (...) { + return false; + } +} + +inline bool CCApiReader::parse_( + const std::string &str, std::uint64_t &value) const noexcept { + try { + value = std::stoul(str); + return true; + } catch (...) { + return false; + } +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiSecurityParser.h b/src/lib/md/CCApiSecurityParser.h deleted file mode 100644 index 1110f5f45..000000000 --- a/src/lib/md/CCApiSecurityParser.h +++ /dev/null @@ -1,107 +0,0 @@ -#pragma once -#ifdef SPARK_ENABLE_CCAPI -#include "lib/core/StringFiller.h" -#include "lib/domain/Security.h" - -#include -#include -#include - -namespace bts::md { - -class CCApiSecurityParser { - public: - template > - static domain::Security parse(const TMap &name_values); - - template > - static void parse(domain::Security &security, const TMap &name_values); - - static std::vector parse( - const std::vector &securities_in); - - static void parse( - const std::vector &securities_in, - std::vector &securities_out); -}; - -////////////////////////////////////////////////// -// Definition -////////////////////////////////////////////////// - -template -inline domain::Security CCApiSecurityParser::parse(const TMap &name_values) { - domain::Security sec; - parse(sec, name_values); - return sec; -} - -template -inline void CCApiSecurityParser::parse( - domain::Security &security, const TMap &props) { - auto &scomm = security.security_common; - for (const auto &[name, value] : props) { - if (name == CCAPI_INSTRUMENT) { - scomm.desc = value; - continue; - } - if (name == CCAPI_ORDER_PRICE_INCREMENT) { - double min_price_inc = std::stod(value); - scomm.min_price_increment = domain::Price::from_double(min_price_inc); - continue; - } - if (name == CCAPI_ORDER_QUANTITY_INCREMENT) { - scomm.min_quantity_increment = std::stod(value); - continue; - } - if (name == CCAPI_UNDERLYING_SYMBOL) { - scomm.symbol = value; - // TODO: make sure these name-mappings are appropriate - core::StringFiller::fill_if_empty(scomm.asset, value); - core::StringFiller::fill_if_empty(scomm.group, value); - continue; - } - if (name == CCAPI_BASE_ASSET) { - scomm.asset = value; - // TODO: make sure these name-mappings are appropriate - core::StringFiller::fill_if_empty(scomm.symbol, value); - core::StringFiller::fill_if_empty(scomm.group, value); - continue; - } - if (name == CCAPI_QUOTE_ASSET) { - if (!api::domain::Currency_Parse(value, &scomm.currency)) { - const auto default_currency = api::domain::Currency::USD; - scomm.currency = default_currency; - LOG(md, Error, - "Failed to parse currency type from string '{}'. Falling back " - "to default value '{}'", - value, default_currency); - } - } - if (name == CCAPI_ORDER_QUANTITY_MIN) { - scomm.min_quantity = std::stod(value); - continue; - } - } -} - -inline void CCApiSecurityParser::parse( - const std::vector &securities_in, - std::vector &securities_out) { - for (const auto &el : securities_in) { - const std::map &name_vals = - el.getNameValueMap(); - auto sec = CCApiSecurityParser::parse(name_vals); - securities_out.push_back(sec); - } -} - -inline std::vector CCApiSecurityParser::parse( - const std::vector &securities_in) { - std::vector secs; - parse(securities_in, secs); - return secs; -} - -} // namespace bts::md -#endif diff --git a/src/lib/md/CCApiSession.cpp b/src/lib/md/CCApiSession.cpp index 249b7effc..e70659789 100644 --- a/src/lib/md/CCApiSession.cpp +++ b/src/lib/md/CCApiSession.cpp @@ -1,91 +1,145 @@ #ifdef SPARK_ENABLE_CCAPI #include "CCApiSession.h" -#include "lib/api/md/DepthStreamConfig.pb.h" +#include "lib/logging/logging.h" #include "lib/md/CCApiCorrelationID.h" -#include "lib/md/CCApiListenerAdaptor.h" -#include "lib/md/MarketDataListenerData.h" -#include "lib/md/MarketDataRegistrationFactory.h" -#include "lib/persistence/IDGenerator.h" +#include "lib/md/CCApiEventDemuxer.h" +#include "lib/md/ICCApiMessageHandler.h" -#include +#include #include -#include +#include #include #include #include -#include +#include namespace bts::md { +using ccapi::EventDispatcher; +using ccapi::Request; +using ccapi::Session; +using ccapi::SessionConfigs; +using ccapi::SessionOptions; +using ccapi::Subscription; +using std::vector; + struct CCApiSession::PImpl { - PImpl( - const MarketDataListenerData listener_data, - bts::persistence::IDGenerator *const sec_id_store) - : listener_data(listener_data), sec_id_store(sec_id_store), - ccapi_dispatcher( - static_cast(listener_data.cfg.num_dispatcher_threads())), - ccapi_listener(listener_data.listener, sec_id_store), - ccapi_session( - ccapi_opts, ccapi_cfgs, &ccapi_listener, &ccapi_dispatcher) {} + PImpl(EventDispatcher *dispatcher, SessionConfigs cfgs, SessionOptions opts) + : dispatcher(dispatcher), demuxer(), cfgs(std::move(cfgs)), + opts(std::move(opts)), + session(this->opts, this->cfgs, &this->demuxer, dispatcher) {} - MarketDataListenerData listener_data; - bts::persistence::IDGenerator *sec_id_store; - - ccapi::SessionOptions ccapi_opts; - ccapi::SessionConfigs ccapi_cfgs; - - // Dispatches ccapi callbacks - ccapi::EventDispatcher ccapi_dispatcher; - - // Handles ccapi sec/md callbacks - CCApiListenerAdaptor ccapi_listener; + EventDispatcher *dispatcher; + CCApiEventDemuxer demuxer; + SessionConfigs cfgs; + SessionOptions opts; // Manages ccapi connection to exchanges - ccapi::Session ccapi_session; + Session session; - void request_securities(api::domain::Venue venue) { - auto venue_name = api::domain::Venue_Name(venue); - auto venue_name_lower = boost::algorithm::to_lower_copy(venue_name); - // The 'venue' in the request must be lower-case (ccapi expects this). The - // correlation id, however, should remain unchanged (in the callback, we - // parse the venue with 'api::domain::Venue_Parse') - ccapi::Request req( - ccapi::Request::Operation::GET_INSTRUMENTS, venue_name_lower); - req.setCorrelationId(venue_name); - ccapi_session.sendRequest(req); - } - - void subscribe_md(const api::md::DepthStreamConfig &ds) { - const auto opts = - fmt::format("{}={}", CCAPI_MARKET_DEPTH_MAX, ds.num_levels()); - auto venue_name = bts::api::domain::Venue_Name(ds.venue()); - boost::algorithm::to_lower(venue_name); - for (const auto &symbol : ds.symbols()) { - const auto corr_id = CCApiCorrelationID::fmt_market_depth( - ds.venue(), symbol, listener_data.id); - ccapi::Subscription sub( - venue_name, symbol, CCAPI_MARKET_DEPTH, opts, corr_id); - ccapi_session.subscribe(sub); - } + void check_handler(const CCApiCorrelationID &corr_id) { + bts_assert_msg(!corr_id.empty(), "Must specify non-empty correlation id"); + bts_assert_msgf( + demuxer.has_handler(corr_id), + "No handler is associated with the request's correlation id '{}'", + corr_id); } }; -CCApiSession::CCApiSession( - MarketDataListenerData listener_data, - bts::persistence::IDGenerator *sec_id_store) - : me_(std::make_unique(std::move(listener_data), sec_id_store)) { +CCApiSession::CCApiSession(EventDispatcher *dispatcher) + : CCApiSession(dispatcher, SessionConfigs{}, SessionOptions{}) {} - std::unordered_set venues; - for (auto ds : me_->listener_data.cfg.depth_streams()) { - // Limit one sec request per Venue - if (venues.insert(ds.venue()).second) { - me_->request_securities(ds.venue()); - } - me_->subscribe_md(ds); - } +CCApiSession::CCApiSession( + EventDispatcher *dispatcher, SessionConfigs cfgs, SessionOptions opts) + : me_(std::make_unique( + dispatcher, std::move(cfgs), std::move(opts))) {} + +CCApiSession::~CCApiSession() { me_->session.stop(); } + +EventDispatcher *CCApiSession::dispatcher() { return me_->dispatcher; } + +bool CCApiSession::add_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + vector msg_types) { + return me_->demuxer.add_handler(corr_id, handler, msg_types); } -CCApiSession::~CCApiSession() { me_->ccapi_session.stop(); } +bool CCApiSession::remove_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + vector msg_types) { + return me_->demuxer.remove_handler(corr_id, handler, msg_types); +} + +void CCApiSession::request(Request req) { + LOG(md, Info, "Requesting {}", req.toString()); + auto corr_id_str = req.getCorrelationId(); + auto corr_id = CCApiCorrelationID::from_string(corr_id_str); + me_->check_handler(corr_id); + me_->session.sendRequest(req); +} + +void CCApiSession::request(Request req, ICCApiMessageHandler *handler) { + auto corr_id_str = req.getCorrelationId(); + auto corr_id = CCApiCorrelationID::from_string(corr_id_str); + add_handler(corr_id, handler); + request(req); +} + +void CCApiSession::request( + BatchRequest batch_req, vector msg_types) { + if (batch_req.empty()) { + return; + } + vector reqs; + for (auto &[tmp_sub, handler] : batch_req) { + auto &sub = reqs.emplace_back(std::move(tmp_sub)); + auto corr_id_str = sub.getCorrelationId(); + auto corr_id = CCApiCorrelationID::from_string(corr_id_str); + add_handler(corr_id, handler, msg_types); + } + auto req_strs = + ranges::transform_view(reqs, [](const auto &r) { return r.toString(); }); + LOG(md, Info, "Requesting batch:\n{}", + fmt::to_string(fmt::join(req_strs, "\n"))); + me_->session.sendRequest(reqs); +} + +void CCApiSession::subscribe(Subscription sub) { + LOG(md, Info, "Subscribing {}", sub.toString()); + auto corr_id_str = sub.getCorrelationId(); + auto corr_id = CCApiCorrelationID::from_string(corr_id_str); + me_->check_handler(corr_id); + me_->session.subscribe(sub); +} + +void CCApiSession::subscribe(Subscription sub, ICCApiMessageHandler *handler) { + auto corr_id_str = sub.getCorrelationId(); + auto corr_id = CCApiCorrelationID::from_string(corr_id_str); + add_handler(corr_id, handler); + subscribe(sub); +} + +void CCApiSession::subscribe( + CCApiSession::BatchSubscription batch_sub, + vector msg_types) { + if (batch_sub.empty()) { + return; + } + vector subs; + for (auto &[tmp_sub, handler] : batch_sub) { + auto &sub = subs.emplace_back(std::move(tmp_sub)); + auto corr_id_str = sub.getCorrelationId(); + auto corr_id = CCApiCorrelationID::from_string(corr_id_str); + add_handler(corr_id, handler, msg_types); + } + auto sub_strs = + ranges::transform_view(subs, [](const auto &s) { return s.toString(); }); + LOG(md, Info, "Subscribing batch:\n{}", + fmt::to_string(fmt::join(sub_strs, "\n"))); + me_->session.subscribe(subs); +} } // namespace bts::md #endif diff --git a/src/lib/md/CCApiSession.h b/src/lib/md/CCApiSession.h index 3edc9fd53..b77fb8bc0 100644 --- a/src/lib/md/CCApiSession.h +++ b/src/lib/md/CCApiSession.h @@ -1,33 +1,84 @@ #pragma once #ifdef SPARK_ENABLE_CCAPI -#include "lib/api/domain/Venue.pb.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/ICCApiMessageHandler.h" +#include +#include #include +#include +#include namespace ccapi { -class Session; -} - -namespace bts::persistence { -class IDGenerator; -} +class EventDispatcher; +class EventHandler; +class SessionConfigs; +class SessionOptions; +class Request; +class Subscription; +} // namespace ccapi namespace bts::md { -struct MarketDataListenerData; +class CCApiSubscription; +class ICCApiMessageHandler; +/** + * @brief Provides facilities to request/subscribe data from exchanges + * + * @details The basic workflow to request/subscribe data is as follows: + * First, create a CCApiCorrelationID to associate with the req/sub. Next, add a + * handler with that particular CCApiCorrelationID. Then, create the req/sub + * object with that same CCApiCorrelationID and pass it to the req/sub method. + * The handler should receive all callbacks associated with that + * CCApiCorrelationId. For convenience, there are request/subscribe overloads + * with an ICCApiMessageHandler* param, so you may set the handler and + * and make the request/subscription in one call. + */ class CCApiSession { public: - explicit CCApiSession( - MarketDataListenerData listener_data, - bts::persistence::IDGenerator *sec_id_store); + using BatchRequest = + std::vector>; - CCApiSession(const CCApiSession &) = delete; - CCApiSession(CCApiSession &&) = default; - CCApiSession &operator=(const CCApiSession &) = delete; - CCApiSession &operator=(CCApiSession &&) = default; + using BatchSubscription = + std::vector>; + + explicit CCApiSession(ccapi::EventDispatcher *dispatcher); + + CCApiSession( + ccapi::EventDispatcher *dispatcher, + ccapi::SessionConfigs cfgs, + ccapi::SessionOptions opts); ~CCApiSession(); + ccapi::EventDispatcher *dispatcher(); + + bool add_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + std::vector msg_types = {}); + + bool remove_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + std::vector msg_types = {}); + + void request(ccapi::Request req); + + void request(ccapi::Request req, ICCApiMessageHandler *handler); + + void request( + BatchRequest batch_req, + std::vector msg_types = {}); + + void subscribe(ccapi::Subscription sub); + + void subscribe(ccapi::Subscription sub, ICCApiMessageHandler *handler); + + void subscribe( + BatchSubscription batch_sub, + std::vector msg_types = {}); + private: struct PImpl; std::unique_ptr me_; diff --git a/src/lib/md/CCApiTradeAdaptor.cpp b/src/lib/md/CCApiTradeAdaptor.cpp new file mode 100644 index 000000000..e65bc8790 --- /dev/null +++ b/src/lib/md/CCApiTradeAdaptor.cpp @@ -0,0 +1,70 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "CCApiTradeAdaptor.h" +#include "lib/core/TimeStamp.h" +#include "lib/domain/SecurityID.h" +#include "lib/domain/TradeRecord.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/CCApiTradeRecord.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/MarketDataListenerData.h" + +#include +#include +#include +#include + +namespace bts::md { + +using bts::core::TimeStamp; +using bts::domain::SecurityID; +using std::vector; + +CCApiTradeAdaptor::CCApiTradeAdaptor( + SecurityID sec_id, IMarketDataListener *listener) + : sec_id_(sec_id), listener_(listener) {} + +void CCApiTradeAdaptor::on_messages( + const ccapi::Event &event, + ccapi::Session * /*session*/, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) { + std::vector trades; + for (const auto *msg : msgs) { + switch (msg->getType()) { + case ccapi::Message::Type::MARKET_DATA_EVENTS_TRADE: + case ccapi::Message::Type::MARKET_DATA_EVENTS_AGG_TRADE: + break; + default: + bts_assert_msgf( + false, "Received CCApi msg of unexpected type '{}'", + ccapi::Message::typeToString(msg->getType())); + return; + } + LOG(md, Diag, "Received CCApi trade msg:\n{}", msg->toStringPretty()); + auto ts = TimeStamp::from_int64( + msg->getTimeReceived().time_since_epoch().count()); + auto sec_desc = corr_id.peek_string(); + auto ccapi_trades = CCApiTradeRecord::parse(*msg); + trades.reserve(trades.size() + ccapi_trades.size()); + for (const auto &ccapi_trade : ccapi_trades) { + auto &trade = trades.emplace_back(); + trade.sec_id = sec_id_; + trade.time_stamp = ts; + ccapi_trade.to_trade_record(trade); + } + } + if (trades.empty()) { + LOG(md, Warn, "Received ccapi trade msg with no trades:\n{}", + event.toStringPretty()); + return; + } + std::vector trade_ptrs; + trade_ptrs.reserve(trades.size()); + for (const auto &trade : trades) { + trade_ptrs.push_back(&trade); + } + listener_->on_trades(trade_ptrs); +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiTradeAdaptor.h b/src/lib/md/CCApiTradeAdaptor.h new file mode 100644 index 000000000..5c6fe63d2 --- /dev/null +++ b/src/lib/md/CCApiTradeAdaptor.h @@ -0,0 +1,39 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/domain/SecurityID.h" +#include "lib/md/CCApiTradeRecord.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/IMarketDataListener.h" +#include + +namespace ccapi { +class Event; +class Message; +class Session; +} // namespace ccapi + +namespace bts::md { + +class CCApiCorrelationID; + +class CCApiTradeAdaptor : public ICCApiMessageHandler { + public: + CCApiTradeAdaptor() = default; + + CCApiTradeAdaptor(domain::SecurityID sec_id, IMarketDataListener *listener); + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override; + + const std::vector &last_trades() const; + + private: + domain::SecurityID sec_id_; + IMarketDataListener *listener_; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiTradeRecord.cpp b/src/lib/md/CCApiTradeRecord.cpp new file mode 100644 index 000000000..5a871976b --- /dev/null +++ b/src/lib/md/CCApiTradeRecord.cpp @@ -0,0 +1,93 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "CCApiTradeRecord.h" +#include "lib/core/BoolName.h" +#include "lib/domain/AggressorSide.h" +#include "lib/domain/Price.h" +#include "lib/domain/Quantity.h" +#include "lib/domain/TradeRecord.h" +#include "lib/io/JsonReader.h" +#include "lib/md/CCApiReader.h" +#include "lib/utils/bts_chrono.h" +#include "lib/utils/datetime.h" +#include +#include +#include +#include +#include +#include + +namespace bts::md { + +using bts::core::BoolName; +using bts::domain::TradeRecord; +using std::vector; + +void CCApiTradeRecord::to_trade_record(TradeRecord &value) const { + // This alias allows the underlying type in 'domain::Quantity' to change + // without breaking this code (will change from uint to double soon) + using QuantityValue = std::remove_reference< + decltype(std::declval().value())>::type; + + value.trade_price = domain::Price::from_double(last_price); + // Temp code to show non-zero values until we have double quantities + value.trade_quantity.value() = + static_cast(std::max(last_size * 1000, 1.0)); + + value.agg_side = is_buyer_maker ? domain::AggressorSide::Buy + : domain::AggressorSide::Sell; +} + +bool CCApiTradeRecord::parse( + const ccapi::Element &source, + CCApiTradeRecord &result, + bool is_aggregated) { + CCApiReader reader{&source.getNameValueMap()}; + return reader.read(CCAPI_LAST_PRICE, result.last_price) && + reader.read(CCAPI_LAST_SIZE, result.last_size) && + reader.read( + (is_aggregated ? CCAPI_AGG_TRADE_ID : CCAPI_TRADE_ID), + result.trade_id) && + reader.read(CCAPI_IS_BUYER_MAKER, result.is_buyer_maker); +} + +std::vector CCApiTradeRecord::parse( + const ccapi::Message &msg) { + std::vector result; + bts_assert(parse(msg, result)); + return result; +} + +bool CCApiTradeRecord::parse( + const ccapi::Message &msg, vector &result) { + bool is_aggregated = false; + switch (msg.getType()) { + case ccapi::Message::Type::MARKET_DATA_EVENTS_AGG_TRADE: + is_aggregated = true; + break; + case ccapi::Message::Type::MARKET_DATA_EVENTS_TRADE: + break; + default: + return false; + } + + const auto &trade_els = msg.getElementList(); + result.reserve(result.size() + trade_els.size()); + for (const auto &trade_el : trade_els) { + auto &trade_rec = result.emplace_back(); + if (!CCApiTradeRecord::parse(trade_el, trade_rec, is_aggregated)) { + return false; + } + } + return true; +} + +std::ostream &operator<<( + std::ostream &out, const CCApiTradeRecord &tr) noexcept { + out << "trade_id: " << tr.trade_id << "; price: " << tr.last_price << "; " + << tr.last_size + << "; is_buyer_maker: " << BoolName::yes_no(tr.is_buyer_maker); + return out; +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiTradeRecord.h b/src/lib/md/CCApiTradeRecord.h new file mode 100644 index 000000000..e5af3e9a4 --- /dev/null +++ b/src/lib/md/CCApiTradeRecord.h @@ -0,0 +1,50 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/core/NaN.h" +#include "lib/core/TimeStamp.h" +#include +#include +#include + +namespace bts::domain { +class TradeRecord; +} + +namespace ccapi { +class Element; +class Message; +} // namespace ccapi + +namespace bts::md { + +/** + * @brief Represents a 'trade' ccapi::Element. Each field has a corresponding + * entry in 'ccapi::Element#getNameValueMap()' + */ +class CCApiTradeRecord { + public: + bool is_buyer_maker = false; + double last_price = core::NaN::value(); + double last_size = core::NaN::value(); + std::uint64_t trade_id = 0; + + static std::vector parse(const ccapi::Message &source); + + static bool parse( + const ccapi::Element &source, + CCApiTradeRecord &result, + bool is_aggregated); + + static bool parse( + const ccapi::Message &msg, std::vector &result); + + void to_trade_record(domain::TradeRecord &value) const; +}; + +void from_json(const nlohmann::json &j, CCApiTradeRecord &r); +void to_json(nlohmann::json &j, const CCApiTradeRecord &r); + +std::ostream &operator<<(std::ostream &out, const CCApiTradeRecord &r) noexcept; + +} // namespace bts::md +#endif diff --git a/src/lib/md/ICCApiMessageHandler.cpp b/src/lib/md/ICCApiMessageHandler.cpp new file mode 100644 index 000000000..d4da329af --- /dev/null +++ b/src/lib/md/ICCApiMessageHandler.cpp @@ -0,0 +1,10 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "ICCApiMessageHandler.h" +#include "lib/md/ICCApiMessageHandler.h" + +namespace bts::md { + +ICCApiMessageHandler::~ICCApiMessageHandler() = default; + +} // namespace bts::md +#endif diff --git a/src/lib/md/ICCApiMessageHandler.h b/src/lib/md/ICCApiMessageHandler.h new file mode 100644 index 000000000..c002dd057 --- /dev/null +++ b/src/lib/md/ICCApiMessageHandler.h @@ -0,0 +1,32 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/md/CCApiCorrelationID.h" + +#include +#include + +namespace ccapi { +class Event; +class Session; +} // namespace ccapi + +namespace bts::md { + +/** + * Provides a callback similar to ccapi::EventHandler::processEvent. This + * class is designed to work with the CCApiEventDemuxer, which provides + * facilities to subscribe only to messages with a particular correlation id. + */ +class ICCApiMessageHandler { + public: + virtual ~ICCApiMessageHandler(); + + virtual void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) = 0; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/MarketDataGateway.cpp b/src/lib/md/MarketDataGateway.cpp index 783036f68..66090cd9b 100644 --- a/src/lib/md/MarketDataGateway.cpp +++ b/src/lib/md/MarketDataGateway.cpp @@ -1,10 +1,11 @@ #ifdef SPARK_ENABLE_CCAPI #include "MarketDataGateway.h" #include "lib/api/domain/Venue.pb.h" -#include "lib/api/md/DepthStreamConfig.pb.h" #include "lib/api/md/MarketDataConfig.pb.h" #include "lib/collections/Contains.h" #include "lib/core/ProtoPrinter.h" +#include "lib/domain/Security.h" +#include "lib/domain/SecurityID.h" #include "lib/logging/logging.h" #include "lib/md/CCApiSession.h" #include "lib/md/IMarketDataListener.h" @@ -13,89 +14,104 @@ #include "lib/md/MarketDataRegistration.h" #include "lib/md/MarketDataRegistrationError.h" #include "lib/md/MarketDataRegistrationFactory.h" +#include "lib/md/MarketDataSubscription.h" +#include "lib/md/ftx/FTXSubscription.h" #include "lib/persistence/IDGenerator.h" #include "lib/utils/bts_assert.h" +#include #include #include +#include #include namespace bts::md { +using bts::api::domain::Venue; +using bts::api::md::MarketDataConfig; +using bts::domain::Security; +using bts::domain::SecurityID; +using bts::persistence::IDGenerator; using std::string; +using std::unique_ptr; +using std::unordered_map; using std::vector; -static void validate( - const api::md::DepthStreamConfig &ds, - std::uint32_t index, - vector &errors) { - if (!api::domain::Venue_IsValid(ds.venue())) { - errors.push_back(fmt::format( - "DepthStreamConfig at index {} 'venue' is invalid", index)); - } - if (ds.num_levels() == 0) { - errors.push_back(fmt::format( - "DepthStreamConfig at index {} 'num_levels' is 0", index)); - } - if (ds.symbols().empty()) { - errors.push_back(fmt::format( - "DepthStreamConfig at index {} 'symbols' is empty", index)); - } -} - -static void validate( - const api::md::MarketDataConfig &cfg, vector errs) { - if (cfg.depth_streams().empty()) { - errs.push_back("MarketDataConfig 'depth_streams' is empty"); - } - std::uint32_t ds_idx = 0; - for (const auto &ds : cfg.depth_streams()) { - validate(ds, ds_idx++, errs); - } +static bool validate(const MarketDataConfig & /*cfg*/, vector &errs) { + auto orig_n_errs = errs.size(); + // TODO + return errs.size() == orig_n_errs; } struct MarketDataGateway::PImpl { + ccapi::EventDispatcher dispatcher; + unordered_map sessions; + MarketDataRegistrationFactory registration_factory; - bts::persistence::IDGenerator &sec_id_store = - bts::persistence::IDGenerator::instance(); - std::unordered_map sessions; + IDGenerator &sec_id_store = IDGenerator::instance(); + unordered_map listeners; + unordered_map> securities; + + CCApiSession &get_session(Venue venue) { + auto it = sessions.find(venue); + if (it == sessions.end()) { + LOG(md, Info, "Creating CCApiSession for venue {}", Venue_Name(venue)); + it = sessions.try_emplace(venue, &dispatcher).first; + } + return it->second; + } }; +#define VENUE_CASE_NAME(x) \ + (MarketDataConfig::descriptor() \ + ->FindFieldByNumber(static_cast(x)) \ + ->name()) + MarketDataGateway::MarketDataGateway() : me_(std::make_unique()) {} MarketDataGateway::~MarketDataGateway() = default; MarketDataRegistration MarketDataGateway::subscribe( - const api::md::MarketDataConfig &cfg, - IMarketDataListener *listener) noexcept { + const MarketDataConfig &cfg, IMarketDataListener *listener) noexcept { auto reg = me_->registration_factory.create(); bts_assert(reg.listener_id.has_value()); - bts_assert(me_->sessions.find(*reg.listener_id) == me_->sessions.end()); - LOG(md, bts::Info, "MarketDataGateway subscribing to data.\n{}\n", + bts_assert(!CONTAINS_KEY(me_->listeners, *reg.listener_id)); + LOG(md, Info, "MarketDataGateway subscribing to data:\n{}", core::ProtoPrinterFactory::create(&cfg)); vector cfg_errs; - validate(cfg, cfg_errs); - if (!cfg_errs.empty()) { + if (!validate(cfg, cfg_errs)) { return MarketDataRegistration{ {MarketDataRegistrationError::Type::CONFIG_ERROR, fmt::to_string(fmt::join(cfg_errs, "\n"))}}; } - MarketDataListenerData listener_data; + MarketDataListenerData &listener_data = me_->listeners[*reg.listener_id]; listener_data.id = *reg.listener_id; listener_data.listener = listener; listener_data.cfg = cfg; - me_->sessions.try_emplace( - listener_data.id, listener_data, &me_->sec_id_store); + switch (cfg.venue_case()) { + case MarketDataConfig::VenueCase::kFtx: { + listener_data.subscription.add(ftx::FTXSubscription{ + &me_->dispatcher, cfg.ftx(), &me_->sec_id_store, + listener_data.listener}); + break; + } + default: + LOG(md, Error, + "Unable to create md subscription because the venue '{}' isn't " + "supported.", + VENUE_CASE_NAME(cfg.venue_case())); + break; + } + // TODO: add more exchanges here return reg; } void MarketDataGateway::unsubscribe(MarketDataListenerID id) { - LOG(md, bts::Info, "Unsubscribing listener {}", id); - bts_assert(CONTAINS_KEY(me_->sessions, id)); - me_->sessions.erase(id); + LOG(md, Info, "Unsubscribing listener {}", id); + me_->listeners.erase(id); } } // namespace bts::md diff --git a/src/lib/md/MarketDataListenerData.h b/src/lib/md/MarketDataListenerData.h index 9f5e64d69..c34421e49 100644 --- a/src/lib/md/MarketDataListenerData.h +++ b/src/lib/md/MarketDataListenerData.h @@ -2,13 +2,25 @@ #include "lib/api/md/MarketDataConfig.pb.h" #include "lib/md/IMarketDataListener.h" #include "lib/md/MarketDataListenerID.h" +#include "lib/md/MarketDataSubscription.h" +#include namespace bts::md { struct MarketDataListenerData { + MarketDataListenerData() = default; + MarketDataListenerData(const MarketDataListenerData &) noexcept = delete; + MarketDataListenerData(MarketDataListenerData &&) noexcept = default; + MarketDataListenerData &operator=(const MarketDataListenerData &) noexcept = + delete; + MarketDataListenerData &operator=(MarketDataListenerData &&) noexcept = + default; + ~MarketDataListenerData() = default; + MarketDataListenerID id; IMarketDataListener *listener; api::md::MarketDataConfig cfg; + MarketDataSubscription subscription; }; } // namespace bts::md diff --git a/src/lib/md/MarketDataSubscription.cpp b/src/lib/md/MarketDataSubscription.cpp new file mode 100644 index 000000000..4bca67083 --- /dev/null +++ b/src/lib/md/MarketDataSubscription.cpp @@ -0,0 +1,27 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "MarketDataSubscription.h" +#include "lib/api/md/MarketDataConfig.pb.h" +#include "lib/md/ftx/FTXSubscription.h" +#include +#include + +namespace bts::md { + +using std::vector; + +struct MarketDataSubscription::PImpl { + vector ftx_subs; + // TODO: add more exchanges +}; + +MarketDataSubscription::MarketDataSubscription() + : me_(std::make_unique()) {} + +MarketDataSubscription::~MarketDataSubscription() = default; + +void MarketDataSubscription::add(ftx::FTXSubscription sub) { + me_->ftx_subs.push_back(std::move(sub)); +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/MarketDataSubscription.h b/src/lib/md/MarketDataSubscription.h new file mode 100644 index 000000000..346b793af --- /dev/null +++ b/src/lib/md/MarketDataSubscription.h @@ -0,0 +1,32 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include + +namespace bts::md::ftx { +class FTXSubscription; +} + +namespace bts::md { + +class MarketDataSubscription { + public: + MarketDataSubscription(); + + MarketDataSubscription(const MarketDataSubscription &) noexcept = delete; + MarketDataSubscription(MarketDataSubscription &&) noexcept = default; + MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept = + delete; + MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept = + default; + + ~MarketDataSubscription(); + + void add(ftx::FTXSubscription); + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/ccApi/CCMain.cpp b/src/lib/md/ccApi/CCMain.cpp index 28c9e375d..163ed63d9 100644 --- a/src/lib/md/ccApi/CCMain.cpp +++ b/src/lib/md/ccApi/CCMain.cpp @@ -1,11 +1,9 @@ #ifdef SPARK_ENABLE_CCAPI #include "CCMain.h" -#include "lib/api/md/DepthStreamConfig.pb.h" #include "lib/api/md/MarketDataConfig.pb.h" #include "lib/config/config.h" #include "lib/config/protos/market_data.pb.h" -#include "lib/core/ProtoReader.h" #include "lib/logging/logging.h" #include "lib/md/MDProducer.h" #include "lib/md/MarketDataGateway.h" @@ -69,17 +67,7 @@ CCMain::~CCMain() {} void CCMain::run() { LOG(md, Info, "Run::run_ccapi t_id:{} t_name:{}", os::get_thread_id(), os::get_thread_name()); - bts::api::md::MarketDataConfig md_config; - md_config.set_num_dispatcher_threads(1); - auto it = md_config.mutable_depth_streams()->Add(); - it->set_venue(::bts::api::domain::Venue::FTX); - it->set_num_levels(5); - it->mutable_symbols()->Add("BTC-PERP"); - it->mutable_symbols()->Add("BTC/USD"); - - char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt"; - putenv(ssl_cert_file); - + const auto &md_config = config::get(); me->gw.subscribe(md_config, &me->cc_listener); size_t n = 0; diff --git a/src/lib/md/ccApi/CCMarketDataListener.cpp b/src/lib/md/ccApi/CCMarketDataListener.cpp index 172b428c3..95e2e377b 100644 --- a/src/lib/md/ccApi/CCMarketDataListener.cpp +++ b/src/lib/md/ccApi/CCMarketDataListener.cpp @@ -159,22 +159,6 @@ void CCMarketDataListener::on_securities( bool pursue = false; domain::Security sec = *s_ptr; const auto &s_comm = sec.security_common; - // temp - { - if (sec.security_common.desc == "BTC-PERP") { - domain::FutureSecurity fsec = {}; - fsec.expiration = get_ts_from_date(2022, 12, 25); - sec.security = fsec; - sec.security_common.min_price_increment_ammount = 1.0; - pursue = true; - } - if (sec.security_common.desc == "BTC/USD") { - domain::SpotSecurity ssec = {}; - sec.security = ssec; - sec.security_common.min_price_increment_ammount = 1.0; - pursue = true; - } - } auto search_result = _find_sec(sec, true); LOG(md, Info, "CCMarketDataListener::on_securities added new security, exchange: " diff --git a/src/lib/md/ftx/FTXFuture.cpp b/src/lib/md/ftx/FTXFuture.cpp new file mode 100644 index 000000000..ceeacdf74 --- /dev/null +++ b/src/lib/md/ftx/FTXFuture.cpp @@ -0,0 +1,181 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "FTXFuture.h" +#include "lib/api/domain/Currency.pb.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/core/TableFormatter.h" +#include "lib/domain/FutureSecurity.h" +#include "lib/domain/Security.h" +#include "lib/io/JsonReader.h" +#include "lib/logging/logging.h" + +#include +#include +#include +#include +#include + +namespace bts::md::ftx { + +using bts::api::domain::Currency; +using bts::api::domain::Venue; +using bts::core::TableFormatter; +using bts::domain::FutureSecurity; +using bts::domain::Security; +using bts::io::JsonReader; +using std::string; +using std::stringstream; +using std::vector; + +static std::string deduce_sym_from_name(const std::string &name) { + std::regex word_regex("(\\w+)"); + auto words_begin = + std::sregex_iterator(name.begin(), name.end(), word_regex); + auto words_end = std::sregex_iterator(); + if (words_begin != words_end) { + return words_begin->str(); + } + return ""; +} + +static std::string deduce_sym(const FTXFuture &ffut) { + if (!ffut.underlying.empty()) { + return ffut.underlying; + } + return deduce_sym_from_name(ffut.name); +} + +void FTXFuture::parse(const ccapi::Message &msg, vector &securities) { + const auto &sec_elements = msg.getElementList(); + for (const auto &sec_el : sec_elements) { + const auto &sec_props = sec_el.getNameValueMap(); + auto it = sec_props.find(CCAPI_HTTP_BODY); + if (it == sec_props.end()) { + continue; + } + const auto &body = it->second; + const auto js = nlohmann::json::parse(body); + vector ftx_futs; + if (!js["success"].get()) { + LOG(md, Error, + "FTXFutureParser received json element w/ 'success' false"); + continue; + } + js["result"].get_to(ftx_futs); + for (const auto &ftx_fut : ftx_futs) { + switch (ftx_fut.type.value) { + case FTXSecurityType::FUTURE: + case FTXSecurityType::PERPETUAL: + break; + case FTXSecurityType::MOVE: + case FTXSecurityType::PREDICTION: + // TODO: Add support for 'MOVE' and 'PREDICTION' futs. They may + // complicate things because they have the same exp dates as + // 'FUTURE' futs. This translates to duplicate sym/exp pairs since + // all fall under the umbrella of a uniform 'Future' type in our + // system. + default: + continue; + } + auto &sec = securities.emplace_back(); + auto &scomm = sec.security_common; + scomm.type = ftx_fut.type.to_security_type(); + scomm.venue = Venue::FTX; + + scomm.exchange_sec_id = ftx_fut.name; + scomm.desc = ftx_fut.name; + scomm.symbol = deduce_sym(ftx_fut); + scomm.asset = scomm.symbol; + scomm.group = scomm.symbol; + + scomm.min_price_increment = ftx_fut.price_increment; + scomm.min_quantity = ftx_fut.size_increment; + scomm.min_quantity_increment = ftx_fut.size_increment; + scomm.currency = Currency::USD; + + auto &fut = sec.security.emplace(); + fut.expiration = ftx_fut.expiry; + LOG(md, Diag, "Converted FTXFuture:\n{}to domain::Security:\n{}", + ftx_fut, sec.to_string(true)); + } + } +} + +void from_json(const nlohmann::json &j, FTXFuture &fut) { + const string js_str = j.dump(4); + LOG(md, Diag, "Parsing FTXFuture from json:\n{}\n", js_str); + + JsonReader jr{&j}; + jr.read("ask", fut.ask); + jr.read("bid", fut.bid); + jr.read("change1h", fut.change_1h); + jr.read("change24h", fut.change_24h); + jr.read("changeBod", fut.change_bod); + jr.read("volumeUsd24h", fut.volume_usd_24h); + jr.read("volume", fut.volume); + jr.read("description", fut.description); + jr.read("enabled", fut.enabled); + jr.read("expired", fut.expired); + jr.read("expiry", fut.expiry); + jr.read("index", fut.index); + jr.read("imfFactor", fut.imf_factor); + jr.read("last", fut.last); + jr.read("lowerBound", fut.lower_bound); + jr.read("mark", fut.mark); + jr.read("name", fut.name); + jr.read("openInterest", fut.open_interest); + jr.read("openInterestUsd", fut.open_interest_usd); + jr.read("perpetual", fut.perpetual); + jr.read("positionLimitWeight", fut.position_limit_weight); + jr.read("postOnly", fut.post_only); + jr.read("priceIncrement", fut.price_increment); + jr.read("sizeIncrement", fut.size_increment); + jr.read("underlying", fut.underlying); + jr.read("upperBound", fut.upper_bound); + jr.read("type", fut.type); +} + +string FTXFuture::to_string_pretty() const { + stringstream out; + TableFormatter::SchemaBuilder schema; + schema.add("ask", [](const auto &f) { return f.ask; }); + schema.add("bid", [](const auto &f) { return f.bid; }); + schema.add("change_1h", [](const auto &f) { return f.change_1h; }); + schema.add("change_24h", [](const auto &f) { return f.change_24h; }); + schema.add("change_bod", [](const auto &f) { return f.change_bod; }); + schema.add("volume_usd_24h", [](const auto &f) { return f.volume_usd_24h; }); + schema.add("volume", [](const auto &f) { return f.volume; }); + schema.add("description", [](const auto &f) { return f.description; }); + schema.add("enabled", [](const auto &f) { return f.enabled; }); + schema.add("expired", [](const auto &f) { return f.expired; }); + schema.add("expiry", [](const auto &f) { return f.expiry; }); + schema.add("index", [](const auto &f) { return f.index; }); + schema.add("imf_factor", [](const auto &f) { return f.imf_factor; }); + schema.add("last", [](const auto &f) { return f.last; }); + schema.add("lower_bound", [](const auto &f) { return f.lower_bound; }); + schema.add("mark", [](const auto &f) { return f.mark; }); + schema.add("name", [](const auto &f) { return f.name; }); + schema.add("open_interest", [](const auto &f) { return f.open_interest; }); + schema.add( + "open_interest_usd", [](const auto &f) { return f.open_interest_usd; }); + schema.add("perpetual", [](const auto &f) { return f.perpetual; }); + schema.add("position_limit_weight", [](const auto &f) { + return f.position_limit_weight; + }); + schema.add("post_only", [](const auto &f) { return f.post_only; }); + schema.add( + "price_increment", [](const auto &f) { return f.price_increment; }); + schema.add("size_increment", [](const auto &f) { return f.size_increment; }); + schema.add("underlying", [](const auto &f) { return f.underlying; }); + schema.add("upper_bound", [](const auto &f) { return f.upper_bound; }); + schema.add("type", [](const auto &f) { return f.type; }); + TableFormatter::format_record(out, *this, schema.result); + return out.str(); +} + +std::ostream &operator<<(std::ostream &out, const FTXFuture &fut) noexcept { + out << fut.name; + return out; +} + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXFuture.h b/src/lib/md/ftx/FTXFuture.h new file mode 100644 index 000000000..10ab39d32 --- /dev/null +++ b/src/lib/md/ftx/FTXFuture.h @@ -0,0 +1,73 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/core/TimeStamp.h" +#include "lib/md/ftx/FTXSecurityType.h" + +#include +#include +#include +#include +#include + +namespace bts::domain { +class Security; +} + +namespace ccapi { +class Message; +} + +namespace bts::md::ftx { + +/*! + * @brief Represents a 'future' at FTX + * + * @details The fields mirror those in the response json for 'futures' + * requests. https://docs.ftx.com/reference/list-all-futures + */ +class FTXFuture { + public: + double ask; + double bid; + double change_1h; + double change_24h; + double change_bod; + double volume_usd_24h; + double volume; + std::string description; + bool enabled; + bool expired; + core::TimeStamp expiry; + double index; + double imf_factor; + double last; + double lower_bound; + double mark; + std::string name; + double open_interest; + double open_interest_usd; + bool perpetual; + double position_limit_weight; + bool post_only; + double price_increment; + double size_increment; + std::string underlying; + double upper_bound; + FTXSecurityType type; + + static void parse( + const ccapi::Message &msg, std::vector &securities); + + /** + * @brief Provides a detailed string representation of the object in + * easy-to-read table format, with a row for each field. + */ + std::string to_string_pretty() const; +}; + +void from_json(const nlohmann::json &j, FTXFuture &fut); + +std::ostream &operator<<(std::ostream &out, const FTXFuture &fut) noexcept; + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXHistoricalPriceAdaptor.cpp b/src/lib/md/ftx/FTXHistoricalPriceAdaptor.cpp new file mode 100644 index 000000000..f6fa569db --- /dev/null +++ b/src/lib/md/ftx/FTXHistoricalPriceAdaptor.cpp @@ -0,0 +1,54 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "FTXHistoricalPriceAdaptor.h" +#include "lib/domain/Security.h" +#include "lib/domain/SecurityMarketDataStatistics.h" +#include "lib/md/IMarketDataListener.h" +#include "lib/md/ftx/FTXHistoricalPriceRecord.h" + +#include +#include +#include + +namespace bts::md::ftx { + +using bts::domain::SecurityID; +using std::vector; + +struct FTXHistoricalPriceAdaptor::PImpl { + SecurityID sec_id; + IMarketDataListener *listener; +}; + +FTXHistoricalPriceAdaptor::FTXHistoricalPriceAdaptor( + SecurityID sec_id, IMarketDataListener *listener) + : me_(std::make_unique()) { + me_->sec_id = sec_id; + me_->listener = listener; +} + +FTXHistoricalPriceAdaptor::~FTXHistoricalPriceAdaptor() = default; + +void FTXHistoricalPriceAdaptor::on_messages( + const ccapi::Event & /*event*/, + ccapi::Session * /*session*/, + const vector &msgs, + const CCApiCorrelationID & /*corr_id*/) { + vector price_recs; + for (const auto *msg : msgs) { + if (!FTXHistoricalPriceRecord::parse(*msg, price_recs)) { + continue; + } + bts_assert(!price_recs.empty()); + auto latest_it = std::max_element( + price_recs.begin(), price_recs.end(), + [](const auto &lhs, const auto &rhs) { + return lhs.start_time < rhs.start_time; + }); + auto sec_stats = latest_it->to_security_statistics(me_->sec_id); + me_->listener->on_security_statistics(sec_stats); + break; + } +} + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXHistoricalPriceAdaptor.h b/src/lib/md/ftx/FTXHistoricalPriceAdaptor.h new file mode 100644 index 000000000..4c3f81fba --- /dev/null +++ b/src/lib/md/ftx/FTXHistoricalPriceAdaptor.h @@ -0,0 +1,31 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/domain/SecurityID.h" +#include "lib/md/ICCApiMessageHandler.h" + +namespace bts::md { +class IMarketDataListener; +} + +namespace bts::md::ftx { + +class FTXHistoricalPriceAdaptor : public ICCApiMessageHandler { + public: + FTXHistoricalPriceAdaptor( + domain::SecurityID sec_id, IMarketDataListener *listener); + + ~FTXHistoricalPriceAdaptor(); + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXHistoricalPriceRecord.cpp b/src/lib/md/ftx/FTXHistoricalPriceRecord.cpp new file mode 100644 index 000000000..c678c4045 --- /dev/null +++ b/src/lib/md/ftx/FTXHistoricalPriceRecord.cpp @@ -0,0 +1,90 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "FTXHistoricalPriceRecord.h" +#include "lib/domain/Price.h" +#include "lib/domain/SecurityMarketDataStatistics.h" +#include "lib/io/JsonReader.h" +#include "lib/md/CCApiReader.h" +#include +#include +#include +#include +#include +#include + +namespace bts::md::ftx { + +using bts::domain::Price; +using bts::domain::SecurityID; +using bts::domain::SecurityMarketDataStatistics; +using bts::io::JsonReader; +using std::map; +using std::string; +using std::vector; + +static bool parse( + const map &name_values, + std::vector &price_recs) { + CCApiReader ccapi_reader{&name_values}; + nlohmann::json js; + if (!ccapi_reader.read(CCAPI_HTTP_BODY, js)) { + return false; + } + JsonReader jreader{&js}; + if (!jreader.read("success")) { + LOG(md, Error, + "FTXHistoricalPriceRecord json field 'success' is false:\n{}", js); + return false; + } + if (!jreader.read("result", price_recs)) { + LOG(md, Error, + "Failed to read FTXHistoricalPriceRecord json from field " + "'result':\n{}", + js); + } + return !price_recs.empty(); +} + +bool FTXHistoricalPriceRecord::parse( + const ccapi::Message &msg, + std::vector &price_recs) { + auto orig_sz = price_recs.size(); + for (const auto &sec_el : msg.getElementList()) { + const auto &hp_props = sec_el.getNameValueMap(); + auto it = hp_props.find(CCAPI_HTTP_BODY); + if (it == hp_props.end()) { + continue; + } + bts::md::ftx::parse(hp_props, price_recs); + } + return price_recs.size() > orig_sz; +} + +void FTXHistoricalPriceRecord::to_security_statistics( + SecurityMarketDataStatistics &sec_stats) const { + sec_stats.high = Price::from_double(high); + sec_stats.low = Price::from_double(low); + sec_stats.open = Price::from_double(open); + sec_stats.settle = Price::from_double(close); + sec_stats.volume.value() = volume; +} + +SecurityMarketDataStatistics FTXHistoricalPriceRecord::to_security_statistics( + SecurityID &sec_id) const { + SecurityMarketDataStatistics sec_stats; + to_security_statistics(sec_stats); + sec_stats.security_id = sec_id; + return sec_stats; +} + +void from_json(const nlohmann::json &j, FTXHistoricalPriceRecord &p) { + JsonReader jr{&j}; + jr.read("startTime", p.start_time); + jr.read("open", p.open); + jr.read("close", p.close); + jr.read("high", p.high); + jr.read("low", p.low); + jr.read("volume", p.volume); +} + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXHistoricalPriceRecord.h b/src/lib/md/ftx/FTXHistoricalPriceRecord.h new file mode 100644 index 000000000..353a6c95a --- /dev/null +++ b/src/lib/md/ftx/FTXHistoricalPriceRecord.h @@ -0,0 +1,41 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/core/TimeStamp.h" +#include "lib/domain/SecurityID.h" +#include + +namespace bts::domain { +class SecurityMarketDataStatistics; +} + +namespace ccapi { +class Event; +class Message; +} // namespace ccapi + +namespace bts::md::ftx { + +class FTXHistoricalPriceRecord { + public: + core::TimeStamp start_time; + double open; + double close; + double high; + double low; + double volume; + + static bool parse( + const ccapi::Message &msg, + std::vector &price_recs); + + void to_security_statistics( + domain::SecurityMarketDataStatistics &sec_stats) const; + + domain::SecurityMarketDataStatistics to_security_statistics( + domain::SecurityID &sec_id) const; +}; + +void from_json(const nlohmann::json &j, FTXHistoricalPriceRecord &p); + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXMarket.cpp b/src/lib/md/ftx/FTXMarket.cpp new file mode 100644 index 000000000..07c222f91 --- /dev/null +++ b/src/lib/md/ftx/FTXMarket.cpp @@ -0,0 +1,217 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "FTXMarket.h" +#include "lib/api/domain/Currency.pb.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/core/NaN.h" +#include "lib/core/TableFormatter.h" +#include "lib/domain/FutureSecurity.h" +#include "lib/domain/Security.h" +#include "lib/domain/SpotSecurity.h" +#include "lib/io/JsonReader.h" +#include "lib/logging/logging.h" +#include "lib/md/ftx/FTXSecurityType.h" + +#include +#include +#include +#include +#include + +namespace bts::md::ftx { + +using bts::api::domain::Currency; +using bts::api::domain::Currency_Name; +using bts::api::domain::Currency_Parse; +using bts::api::domain::SecurityType; +using bts::api::domain::Venue; +using bts::core::NaN; +using bts::core::TableFormatter; +using bts::domain::Security; +using bts::io::JsonReader; +using std::ostream; +using std::string; +using std::vector; + +static std::string deduce_symbol(const std::string &name) { + std::regex word_regex("(\\w+)"); + auto words_begin = + std::sregex_iterator(name.begin(), name.end(), word_regex); + auto words_end = std::sregex_iterator(); + if (words_begin != words_end) { + return words_begin->str(); + } + return ""; +} + +static std::string deduce_symbol(const FTXMarket &fsec) { + if (!fsec.underlying.empty()) { + return fsec.underlying; + } + switch (fsec.type.value) { + case FTXSecurityType::SPOT: + return fsec.base_currency; + default: + return deduce_symbol(fsec.name); + } +} + +void FTXMarket::parse(const ccapi::Message &msg, vector &securities) { + const auto &sec_elements = msg.getElementList(); + for (const auto &sec_el : sec_elements) { + const auto &sec_props = sec_el.getNameValueMap(); + auto it = sec_props.find(CCAPI_HTTP_BODY); + if (it == sec_props.end()) { + continue; + } + const auto &body = it->second; + const auto js = nlohmann::json::parse(body); + vector ftx_mkts; + if (!js["success"].get()) { + LOG(md, Error, + "FTXSecurityParser received json element w/ 'success' false"); + continue; + } + js["result"].get_to(ftx_mkts); + for (const auto &ftx_mkt : ftx_mkts) { + auto &sec = securities.emplace_back(); + auto &scomm = sec.security_common; + scomm.desc = ftx_mkt.name; + scomm.venue = Venue::FTX; + + scomm.symbol = deduce_symbol(ftx_mkt); + scomm.asset = scomm.symbol; + scomm.group = scomm.symbol; + + scomm.min_price_increment = ftx_mkt.price_increment; + scomm.min_quantity = ftx_mkt.min_provide_size; + scomm.min_quantity_increment = ftx_mkt.size_increment; + scomm.type = ftx_mkt.type.to_security_type(); + + switch (scomm.type) { + case SecurityType::UNSET: + bts_assert_msgf( + false, "Failed to parse ftx security type from '{}'", + ftx_mkt.type); + break; + case api::domain::SPOT: + sec.security.emplace(); + break; + case api::domain::FUTURE: + sec.security.emplace(); + break; + case api::domain::SPREAD: + bts_assert_msg(false, "Spreads not supported"); + break; + case api::domain::OPTION: + bts_assert_msg(false, "Options not supported"); + break; + case api::domain::SYNTHETIC: + bts_assert_msg(false, "Synthetics not supported"); + break; + case api::domain::SecurityType_INT_MIN_SENTINEL_DO_NOT_USE_: + case api::domain::SecurityType_INT_MAX_SENTINEL_DO_NOT_USE_: + bts_assert_msgf(false, "Bad security type value '{}'", scomm.type); + break; + } + + // currency + constexpr const auto default_currency = Currency::USD; + if (ftx_mkt.quote_currency.empty()) { + scomm.currency = default_currency; + } else if (!Currency_Parse(ftx_mkt.quote_currency, &scomm.currency)) { + scomm.currency = default_currency; + LOG(md, Error, + "Failed to parse currency type from string '{}'. Falling back" + "to default value '{}'", + ftx_mkt.quote_currency, Currency_Name(default_currency)); + } + LOG(md, Diag, "domain::Security (from FTXMarket):\n{}", + sec.to_string(true)); + } + } +} + +void from_json(const nlohmann::json &j, FTXMarket &mkt) { + const string js_str = j.dump(4); + LOG(md, Diag, "Parsing FTXMarket from json:\n{}\n", js_str); + + JsonReader jr{&j}; + // type-independent props + jr.read("name", mkt.name); + jr.read("type", mkt.type); + jr.read("quoteVolume24h", mkt.quote_volume_24h); + jr.read("change1h", mkt.change_1h); + jr.read("change24h", mkt.change_24h); + jr.read("changeBod", mkt.change_bod); + jr.read("highLeverageFeeExempt", mkt.high_leverage_fee_exempt); + jr.read("minProvideSize", mkt.min_provide_size); + + jr.read("enabled", mkt.enabled); + jr.read_or("ask", mkt.ask, NaN::value()); + jr.read_or("bid", mkt.bid, NaN::value()); + jr.read_or("last", mkt.last, NaN::value()); + jr.read("postOnly", mkt.post_only); + jr.read_or("price", mkt.price, NaN::value()); + jr.read("priceIncrement", mkt.price_increment); + jr.read("sizeIncrement", mkt.size_increment); + jr.read("restricted", mkt.restricted); + jr.read("volumeUsd24h", mkt.volume_usd_24h); + jr.read("largeOrderThreshold", mkt.large_order_threshold); + jr.read("isEtfMarket", mkt.is_etf_market); + + // type-dependent props + switch (mkt.type.value) { + case FTXSecurityType::FUTURE: + // these only apply to futures + jr.read("underlying", mkt.underlying); + break; + case FTXSecurityType::SPOT: + // these only apply to spots + jr.read("baseCurrency", mkt.base_currency); + jr.read("quoteCurrency", mkt.quote_currency); + break; + default: + LOG(md, Error, "Unexpected FTXSecurityType '{}'", + j["type"].get()); + break; + } +} + +ostream &operator<<(ostream &out, const FTXMarket &mkt) noexcept { + core::TableFormatter::SchemaBuilder schema; + schema.add("name", [](const auto &s) { return s.name; }); + schema.add("base_currency", [](const auto &s) { return s.base_currency; }); + schema.add("quote_currency", [](const auto &s) { return s.quote_currency; }); + schema.add( + "quote_volume_24h", [](const auto &s) { return s.quote_volume_24h; }); + schema.add("change_1h", [](const auto &s) { return s.change_1h; }); + schema.add("change_24h", [](const auto &s) { return s.change_24h; }); + schema.add("change_bod", [](const auto &s) { return s.change_bod; }); + schema.add("high_leverage_fee_exempt", [](const auto &s) { + return s.high_leverage_fee_exempt; + }); + schema.add( + "min_provide_size", [](const auto &s) { return s.min_provide_size; }); + schema.add("type", [](const auto &s) { return s.type; }); + schema.add("underlying", [](const auto &s) { return s.underlying; }); + schema.add("enabled", [](const auto &s) { return s.enabled; }); + schema.add("ask", [](const auto &s) { return s.ask; }); + schema.add("bid", [](const auto &s) { return s.bid; }); + schema.add("last", [](const auto &s) { return s.last; }); + schema.add("post_only", [](const auto &s) { return s.post_only; }); + schema.add("price", [](const auto &s) { return s.price; }); + schema.add( + "price_increment", [](const auto &s) { return s.price_increment; }); + schema.add("size_increment", [](const auto &s) { return s.size_increment; }); + schema.add("restricted", [](const auto &s) { return s.restricted; }); + schema.add("volume_usd_24h", [](const auto &s) { return s.volume_usd_24h; }); + schema.add("large_order_threshold", [](const auto &s) { + return s.large_order_threshold; + }); + schema.add("is_etf_market", [](const auto &s) { return s.is_etf_market; }); + TableFormatter::format_record(out, mkt, schema.result); + return out; +} + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXMarket.h b/src/lib/md/ftx/FTXMarket.h new file mode 100644 index 000000000..bc0a0e2a3 --- /dev/null +++ b/src/lib/md/ftx/FTXMarket.h @@ -0,0 +1,61 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/md/ftx/FTXSecurityType.h" +#include +#include +#include +#include +#include + +namespace bts::domain { +class Security; +} + +namespace ccapi { +class Message; +} + +namespace bts::md::ftx { + +/** + * Represents a 'market' at FTX + * + * @details The fields mirror those in the response json for 'markets' + * requests. https://docs.ftx.com/reference/get-markets + */ +class FTXMarket { + public: + std::string name; + std::string base_currency; + std::string quote_currency; + double quote_volume_24h; + double change_1h; + double change_24h; + double change_bod; + bool high_leverage_fee_exempt; + double min_provide_size; + FTXSecurityType type; + std::string underlying; + bool enabled; + double ask; + double bid; + double last; + bool post_only; + double price; + double price_increment; + double size_increment; + bool restricted; + double volume_usd_24h; + double large_order_threshold; + bool is_etf_market; + + static void parse( + const ccapi::Message &msg, std::vector &securities); +}; + +void from_json(const nlohmann::json &j, FTXMarket &sec); + +std::ostream &operator<<(std::ostream &out, const FTXMarket &sec) noexcept; + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXMessageType.h b/src/lib/md/ftx/FTXMessageType.h new file mode 100644 index 000000000..0a13aa2c0 --- /dev/null +++ b/src/lib/md/ftx/FTXMessageType.h @@ -0,0 +1,35 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/core/EnumMacros.h" +#include "lib/core/FMT_IMPL.h" +#include +#include +#include + +namespace bts::md::ftx { + +// clang-format off +#define FOR_ALL_VALUES(per_value) \ + per_value(FTXMessageType, NONE) \ + per_value(FTXMessageType, FUTURES) \ + per_value(FTXMessageType, SECURITIES) \ + per_value(FTXMessageType, DEPTH) \ + per_value(FTXMessageType, TICKER) \ + per_value(FTXMessageType, STATISTICS) +// clang-format on + +/** + * @brief Represents the different message types at FTX + * + * @details Each enum value is associated with a particular type of request (or + * subscription) that is sent to the exchange (FTX), and a particular handler + * type designed to adapt the messages for an IMarketDataListener. + */ +ENUM_CLASS_DEF(FTXMessageType, FOR_ALL_VALUES) + +#undef FOR_ALL_VALUES +} // namespace bts::md::ftx + +FMT_IMPL(bts::md::ftx::FTXMessageType, t, "{}", t.name()); + +#endif diff --git a/src/lib/md/ftx/FTXSecurityRequestor.cpp b/src/lib/md/ftx/FTXSecurityRequestor.cpp new file mode 100644 index 000000000..261efefff --- /dev/null +++ b/src/lib/md/ftx/FTXSecurityRequestor.cpp @@ -0,0 +1,275 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "FTXSecurityRequestor.h" +#include "lib/api/domain/SecurityType.pb.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/api/md/FTXMarketDataConfig.pb.h" +#include "lib/collections/Contains.h" +#include "lib/collections/Remove.h" +#include "lib/domain/Security.h" +#include "lib/domain/SecurityCriteria.h" +#include "lib/logging/logging.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/CCApiMessageHandler.h" +#include "lib/md/CCApiSession.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/ftx/FTXFuture.h" +#include "lib/md/ftx/FTXMarket.h" +#include "lib/md/ftx/FTXMessageType.h" +#include "lib/persistence/IDGenerator.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bts::md::ftx { + +using api::domain::SecurityType; +using api::domain::Venue; +using api::domain::Venue_Name; +using bts::domain::Security; +using bts::domain::SecurityID; +using std::string; +using std::unordered_set; +using std::vector; + +#define VENUE_NAME (Venue_Name(Venue::FTX)) + +#define FTX_MSG_TYPE_NAME(x) \ + std::string { FTXMessageType{x}.name() } + +static ccapi::Request markets_req() { + auto secs_corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, FTX_MSG_TYPE_NAME(FTXMessageType::SECURITIES)); + ccapi::Request secs_req{ + ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, "ftx"}; + secs_req.appendParam( + {{CCAPI_HTTP_PROTOCOL, "https"}, + {CCAPI_HTTP_HOST, "ftx.com"}, + {CCAPI_HTTP_PORT, "443"}, + {CCAPI_HTTP_PATH, "/api/markets"}, + {CCAPI_HTTP_METHOD, "GET"}}); + secs_req.setCorrelationId(secs_corr_id.to_string()); + return secs_req; +} + +static ccapi::Request futures_req() { + auto futs_corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, FTX_MSG_TYPE_NAME(FTXMessageType::FUTURES)); + ccapi::Request futs_req{ + ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, "ftx"}; + futs_req.appendParam( + {{CCAPI_HTTP_PROTOCOL, "https"}, + {CCAPI_HTTP_HOST, "ftx.com"}, + {CCAPI_HTTP_PORT, "443"}, + {CCAPI_HTTP_PATH, "/api/futures"}, + {CCAPI_HTTP_METHOD, "GET"}}); + futs_req.setCorrelationId(futs_corr_id.to_string()); + return futs_req; +} + +static vector get_sec_descs(const vector &sec_ptrs) { + vector descs{sec_ptrs.size()}; + std::transform( + sec_ptrs.begin(), sec_ptrs.end(), descs.begin(), + [](const auto &s) { return s.desc(); }); + return descs; +} + +static void validate_response(const ccapi::Event &event, bool is_expected) { + bts_assert_msgf( + event.getType() == ccapi::Event::Type::RESPONSE, + "Received event w/ unexpected type:\n{}", event.toStringPretty()); + bts_assert_msgf( + is_expected, "Received unexpected response from FTX:\n{}", + event.toStringPretty()); +} + +struct FTXSecurityRequestor::PImpl { + // General Notes: + // FTX securities must be obtained from 2 different urls: + // 1) https://ftx.com/api/markets + // 2) https://ftx.com/api/futures + // Although all securities are listed in the response from (1), not all + // information is provided for futures (notably, the expiration date is + // missing!). This missing information is obtained from (2). Therefore, + // futures are published when (2) is received, while all other securities + // are published when (1) is received. + + domain::SecurityCriteria criteria; + unordered_set securities; + persistence::IDGenerator *sec_id_store; + // True when a 'markets' request is in flight + bool mkts_req_pending = false; + // True when a 'futures' request is in flight + bool futs_req_pending = false; + // True when someone called FTXSecurityRequestor::request while another + // request was already in flight. In this case, the 'external_req_pending' + // flag is used to signal that another request should be sent immediately + // after the current one finishes. + bool external_req_pending = false; + + // These proxy handlers don't actually process new securities. Member methods + // in this class process the securities; we simply bind them to proxy handler + // callbacks. This is easier than defining separate ICCApiMessageHandler + // impls for 'markets', and 'futures', esp since they would need access to + // state contained in this class. + CCApiMessageHandler futs_handler_proxy; + CCApiMessageHandler mkts_handler_proxy; + Callback callback; + + // The session automatically starts/stops in its ctr/dtr + CCApiSession session; + + explicit PImpl( + ccapi::EventDispatcher *dispatcher, + domain::SecurityCriteria criteria, + persistence::IDGenerator *sec_id_store) + : criteria(std::move(criteria)), sec_id_store(sec_id_store), + session(dispatcher) { + mkts_handler_proxy.callback().bind<&PImpl::on_markets>(*this); + futs_handler_proxy.callback().bind<&PImpl::on_futures>(*this); + } + + bool is_busy() const { return mkts_req_pending || futs_req_pending; } + + void request() { + external_req_pending = false; + mkts_req_pending = true; + futs_req_pending = true; + session.request(markets_req(), &mkts_handler_proxy); + session.request(futures_req(), &futs_handler_proxy); + } + + /** + * @brief Parses securities from a ccapi::Message + * + * @details Only new securities that satisfy the 'SecurityCriteria' are added + * to \p added_secs + * + * @tparam TParser + * @param msg The ccapi::Message to parse + * @param types The types of securities to parse + * @param parser Ther parser to use + */ + template + void parse_secs( + const ccapi::Message &msg, + const vector &types, + vector &added_secs) { + T::parse(msg, added_secs); + // Exclude secs that don't satisfy the SecurityCriteria + auto is_sec_excluded = [this, &types](const Security &sec) { + return !CONTAINS(types, sec.type()) || + !criteria.includes_or_undefined(sec) || + securities.count(sec.desc()) > 0; + }; + REMOVE_IF(added_secs, is_sec_excluded); + // Assign security ids + for (auto &sec : added_secs) { + // Since the ids haven't been set yet, we should expect them to be 0 + bts_assert(sec.id().value() == 0); + SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + sec.security_common.id = sec_id; + } + } + + /** + * @brief Parses securities from ccapi messages, updates the securities set, + * and subscribes depth for newly arrived securities + * + * @details Securities that don't pass the criteria filter are ignored + * + * @tparam TParser The type of parser + * @param msgs Vector of ccapi messages containing securities + * @param types The types of securities to update (types must be explicitly + * specified to be included) + */ + template + void update_securities( + const vector &msgs, + const vector &types) { + vector added_secs; + for (const auto *msg : msgs) { + bts_assert_msgf( + msg->getType() == ccapi::Message::Type::GENERIC_PUBLIC_REQUEST, + "Received msg w/ unexpected type:\n{}", msg->toStringPretty()); + parse_secs(*msg, types, added_secs); + } + auto added_sec_descs = get_sec_descs(added_secs); + LOG(md, Info, "Received FTX securities:\n{}", + fmt::to_string(fmt::join(added_sec_descs, "\n"))); + callback(added_secs); + } + + /** + * Handles 'markets' update from CCApiSession + */ + bool on_markets( + const ccapi::Event &event, + ccapi::Session * /*session*/, + const vector &msgs, + const CCApiCorrelationID & /*corr_id*/) { + validate_response(event, mkts_req_pending); + LOG(md, Diag, "Received FTX markets:\n{}", event.toStringPretty()); + update_securities(msgs, {SecurityType::SPOT}); + mkts_req_pending = false; + if (external_req_pending && !is_busy()) { + request(); + } + return true; + } + + /** + * Handles 'futures' update from CCApiSession + */ + bool on_futures( + const ccapi::Event &event, + ccapi::Session * /*session*/, + const vector &msgs, + const CCApiCorrelationID & /*corr_id*/) { + validate_response(event, futs_req_pending); + LOG(md, Diag, "Received FTX futures:\n{}", event.toStringPretty()); + update_securities(msgs, {SecurityType::FUTURE}); + futs_req_pending = false; + if (external_req_pending && !is_busy()) { + request(); + } + return true; + } +}; + +FTXSecurityRequestor::FTXSecurityRequestor( + ccapi::EventDispatcher *dispatcher, + domain::SecurityCriteria criteria, + persistence::IDGenerator *sec_id_store) + : me_(std::make_unique( + dispatcher, std::move(criteria), sec_id_store)) {} + +FTXSecurityRequestor::FTXSecurityRequestor(FTXSecurityRequestor &&) noexcept = + default; +FTXSecurityRequestor &FTXSecurityRequestor::operator=( + FTXSecurityRequestor &&) noexcept = default; + +FTXSecurityRequestor::~FTXSecurityRequestor() = default; + +void FTXSecurityRequestor::request() { + if (is_busy()) { + me_->external_req_pending = true; + return; + } + me_->request(); +} + +bool FTXSecurityRequestor::is_busy() const { return me_->is_busy(); } + +FTXSecurityRequestor::Callback &FTXSecurityRequestor::callback() { + return me_->callback; +} + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXSecurityRequestor.h b/src/lib/md/ftx/FTXSecurityRequestor.h new file mode 100644 index 000000000..a9312f5c4 --- /dev/null +++ b/src/lib/md/ftx/FTXSecurityRequestor.h @@ -0,0 +1,79 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/domain/SecurityID.h" +#include "lib/utils/Delegate.h" +#include +#include +#include +#include + +namespace ccapi { +class EventDispatcher; +} + +namespace bts::api::md { +class FTXMarketDataConfig; +} + +namespace bts::md { +class IMarketDataListener; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::domain { +class Security; +class SecurityCriteria; +} // namespace bts::domain + +namespace bts::md::ftx { + +/** + * @brief Requests securities from FTX + * + * @details Uses ccapi infrastructure to send the request + */ +class FTXSecurityRequestor { + public: + FTXSecurityRequestor( + ccapi::EventDispatcher *dispatcher, + domain::SecurityCriteria criteria, + persistence::IDGenerator *sec_id_store); + + FTXSecurityRequestor(const FTXSecurityRequestor &) noexcept = delete; + FTXSecurityRequestor(FTXSecurityRequestor &&) noexcept; + FTXSecurityRequestor &operator=(const FTXSecurityRequestor &) noexcept = + delete; + FTXSecurityRequestor &operator=(FTXSecurityRequestor &&) noexcept; + + ~FTXSecurityRequestor(); + + using Callback = core::Delegate)>; + + Callback &callback(); + + /** + * @brief Request securities from the exchange + * + * @details Does NOT generate a live-updating stream of securities. Callers + * must call 'request' periodically to keep their securities in sync with the + * active set at the exchange. + */ + void request(); + + /** + * @brief Indicates whether or not a request is in-progress + * + * @details Concurrent requests are not supported + */ + bool is_busy() const; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXSecurityType.h b/src/lib/md/ftx/FTXSecurityType.h new file mode 100644 index 000000000..a93da0de1 --- /dev/null +++ b/src/lib/md/ftx/FTXSecurityType.h @@ -0,0 +1,65 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/api/domain/SecurityType.pb.h" +#include "lib/core/EnumMacros.h" +#include +#include + +namespace bts::md::ftx { + +// clang-format off +#define FOR_ALL_VALUES(per_value) \ + per_value(FTXSecurityType, NONE) \ + per_value(FTXSecurityType, SPOT) \ + per_value(FTXSecurityType, FUTURE) \ + per_value(FTXSecurityType, PERPETUAL) \ + per_value(FTXSecurityType, PREDICTION) \ + per_value(FTXSecurityType, MOVE) +// clang-format on + +/** + * @brief Represents the different security types at FTX + * + * @details The names come directly from FTX's 'markets' and 'futures' response + * payload (json). For more details see the docs: + * https://docs.ftx.com, + * https://docs.ftx.com/#markets, + * https://docs.ftx.com/#futures + */ +class FTXSecurityType { + public: + enum Value { FOR_ALL_VALUES(ENUM_DECLARE_VALUE) }; + + ENUM_BASIC_MEMBERS(FTXSecurityType) + + constexpr api::domain::SecurityType to_security_type() const; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +ENUM_OUTPUT_OPS(FTXSecurityType) +ENUM_NAME_DEF(FTXSecurityType, FOR_ALL_VALUES) +ENUM_PARSE_DEF(FTXSecurityType, FOR_ALL_VALUES) +ENUM_FROM_JSON(FTXSecurityType) +ENUM_EQUALITY_OPS(FTXSecurityType) + +constexpr api::domain::SecurityType FTXSecurityType::to_security_type() const { + switch (value) { + case SPOT: + return api::domain::SecurityType::SPOT; + case FUTURE: + case PERPETUAL: + case PREDICTION: + case MOVE: + return api::domain::SecurityType::FUTURE; + case NONE: + default: + return api::domain::SecurityType::UNSET; + } +} + +#undef FOR_ALL_VALUES +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXSubscription.cpp b/src/lib/md/ftx/FTXSubscription.cpp new file mode 100644 index 000000000..31f55fef1 --- /dev/null +++ b/src/lib/md/ftx/FTXSubscription.cpp @@ -0,0 +1,243 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "FTXSubscription.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/api/md/FTXMarketDataConfig.pb.h" +#include "lib/collections/Contains.h" +#include "lib/core/ProtoPrinter.h" +#include "lib/domain/Security.h" +#include "lib/domain/SecurityCriteria.h" +#include "lib/domain/SecurityMarketDataStatistics.h" +#include "lib/logging/logging.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/CCApiMarketDepthAdaptor.h" +#include "lib/md/CCApiMessageHandler.h" +#include "lib/md/CCApiMessageLogger.h" +#include "lib/md/CCApiSession.h" +#include "lib/md/CCApiTradeAdaptor.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/IMarketDataListener.h" +#include "lib/md/MarketDataListenerData.h" +#include "lib/md/ftx/FTXHistoricalPriceAdaptor.h" +#include "lib/md/ftx/FTXHistoricalPriceRecord.h" +#include "lib/md/ftx/FTXMessageType.h" +#include "lib/md/ftx/FTXSecurityRequestor.h" +#include "lib/persistence/IDGenerator.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bts::md::ftx { + +using bts::api::domain::Venue; +using bts::api::domain::Venue_Name; +using bts::api::md::FTXMarketDataConfig; +using bts::core::ProtoPrinterFactory; +using bts::domain::Security; +using bts::domain::SecurityCriteria; +using bts::domain::SecurityID; +using bts::persistence::IDGenerator; +using std::string; +using std::unique_ptr; +using std::unordered_map; +using std::vector; + +#define VENUE_NAME (Venue_Name(Venue::FTX)) + +#define FTX_MSG_TYPE_NAME(x) \ + std::string { FTXMessageType{x}.name() } + +static ccapi::Request create_stats_req(const string &sec_desc) { + const int SECS_PER_DAY = 86400; + auto corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, FTX_MSG_TYPE_NAME(FTXMessageType::STATISTICS), sec_desc); + ccapi::Request req{ + ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, + CCAPI_EXCHANGE_NAME_FTX}; + req.appendParam( + {{CCAPI_HTTP_PROTOCOL, "https"}, + {CCAPI_HTTP_HOST, "ftx.com"}, + {CCAPI_HTTP_PORT, "443"}, + {CCAPI_HTTP_PATH, fmt::format("/api/markets/{}/candles", sec_desc)}, + {CCAPI_HTTP_QUERY_STRING, fmt::format("resolution={}", SECS_PER_DAY)}, + {CCAPI_HTTP_METHOD, "GET"}}); + req.setCorrelationId(corr_id.to_string()); + return req; +} + +static ccapi::Subscription create_depth_sub( + const string &sec_desc, const string &opts) { + auto corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, FTXMessageType::DEPTH, sec_desc); + return ccapi::Subscription{ + CCAPI_EXCHANGE_NAME_FTX, sec_desc, CCAPI_MARKET_DEPTH, opts, + corr_id.to_string()}; +} + +static ccapi::Subscription create_ticker_sub( + const string &sec_desc, const string &opts) { + const auto corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, FTXMessageType::TICKER, sec_desc); + return ccapi::Subscription{ + CCAPI_EXCHANGE_NAME_FTX, sec_desc, CCAPI_TRADE, opts, + corr_id.to_string()}; +} + +struct FTXSubscription::PImpl { + // General Notes: + // FTX securities must be obtained from 2 different urls: + // 1) https://ftx.com/api/markets + // 2) https://ftx.com/api/futures + // Although all securities are listed in the response from (1), not all + // information is provided for futures (notably, the expiration date is + // missing!). This missing information is obtained from (2). Therefore, + // futures are published when (2) is received, while all other securities + // are published when (1) is received. + + unordered_map> securities; + IDGenerator *sec_id_store; + // Don't actually need this, but keeping it around for diagnostic purposes + // (i.e., inspection in debugger, temp logs, etc.) + FTXMarketDataConfig cfg; + SecurityCriteria criteria; + FTXSecurityRequestor sec_requestor; + // The session automatically starts/stops in its ctr/dtr + CCApiSession session; + IMarketDataListener *listener; + + // These aliases are useful for quickly switching the handler types (e.g., + // a special handler that logs messages for testing/diagnostic purposes) + using DepthHandler = CCApiMarketDepthAdaptor; + using TickerHandler = CCApiTradeAdaptor; + using StatsHandler = FTXHistoricalPriceAdaptor; + + unordered_map depth_handlers; + unordered_map ticker_handlers; + unordered_map stats_handlers; + + explicit PImpl( + ccapi::EventDispatcher *dispatcher, + FTXMarketDataConfig cfg, + persistence::IDGenerator *sec_id_store, + IMarketDataListener *listener) + : sec_id_store(sec_id_store), cfg(std::move(cfg)), + criteria(domain::SecurityCriteria::from_proto( + this->cfg.security_criteria())), + sec_requestor(dispatcher, criteria, sec_id_store), session(dispatcher), + listener(listener) { + LOG(md, Info, "Initializing FTXSubscription:\n{}", + ProtoPrinterFactory::create(&this->cfg)); + sec_requestor.callback().bind<&PImpl::on_securities>(*this); + sec_requestor.request(); + } + + DepthHandler &create_depth_handler(string sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + auto [it, is_new] = + depth_handlers.try_emplace(sec_desc, sec_id, listener); + bts_assert(is_new); + return it->second; + } + + TickerHandler &create_ticker_handler(string sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + auto [it, is_new] = + ticker_handlers.try_emplace(sec_desc, sec_id, listener); + bts_assert_msgf( + is_new, "Ticker handler already created for '{}'", sec_desc); + return it->second; + } + + StatsHandler &create_stats_handler(string sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + auto [it, is_new] = + stats_handlers.try_emplace(sec_desc, sec_id, listener); + bts_assert_msgf( + is_new, "Statistics handler already created for '{}'", sec_desc); + return it->second; + } + + /** + * @brief Subscribes to depth, ticker, and price data for a list of + * securities + */ + template + void subscribe_market_data(SecDescs securities) { + const string depth_opts = + fmt::format(CCAPI_MARKET_DEPTH_MAX "={}", cfg.num_book_levels()); + const string ticker_opts = ""; + + // Batch subscribe/request ticker, depth, and stats + CCApiSession::BatchSubscription depth_subs; + CCApiSession::BatchSubscription ticker_subs; + CCApiSession::BatchRequest stats_reqs; + + for (const auto &sec_desc : securities) { + depth_subs.push_back( + {create_depth_sub(sec_desc, depth_opts), + &create_depth_handler(sec_desc)}); + ticker_subs.push_back( + {create_ticker_sub(sec_desc, ticker_opts), + &create_ticker_handler(sec_desc)}); + stats_reqs.push_back( + {create_stats_req(sec_desc), &create_stats_handler(sec_desc)}); + } + + session.subscribe( + depth_subs, {ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH}); + session.subscribe( + ticker_subs, {ccapi::Message::Type::MARKET_DATA_EVENTS_TRADE}); + session.request( + stats_reqs, {ccapi::Message::Type::GENERIC_PUBLIC_REQUEST}); + } + + /** + * Updates the securities map and subscribes market data for newly arrived + * securities + */ + void on_securities(vector secs) { + // 'sec_descs' is used to make depth subscriptions + vector sec_descs; + sec_descs.reserve(secs.size()); + // 'sec_ptrs' is passed to IMarketDataListeners + vector sec_ptrs; + sec_ptrs.reserve(secs.size()); + + for (auto &s : secs) { + auto id = s.id(); + auto desc = s.desc(); + auto [it, is_new] = securities.try_emplace( + id, std::make_unique(std::move(s))); + bts_assert_msgf(is_new, "Received duplicate FTX security '{}'", desc); + sec_descs.push_back(std::move(desc)); + // add ptr to entry in 'securities' (not the loop var, we moved that) + sec_ptrs.push_back(it->second.get()); + } + + listener->on_securities(sec_ptrs); + subscribe_market_data(sec_descs); + } +}; + +FTXSubscription::FTXSubscription( + ccapi::EventDispatcher *dispatcher, + FTXMarketDataConfig cfg, + IDGenerator *sec_id_store, + IMarketDataListener *listener) + : me_(std::make_unique( + dispatcher, std::move(cfg), sec_id_store, listener)) {} + +FTXSubscription::FTXSubscription(FTXSubscription &&) noexcept = default; +FTXSubscription &FTXSubscription::operator=(FTXSubscription &&) noexcept = + default; + +FTXSubscription::~FTXSubscription() = default; + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXSubscription.h b/src/lib/md/ftx/FTXSubscription.h new file mode 100644 index 000000000..b49aca0a0 --- /dev/null +++ b/src/lib/md/ftx/FTXSubscription.h @@ -0,0 +1,50 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include +#include + +namespace ccapi { +class EventDispatcher; +} + +namespace bts::api::md { +class FTXMarketDataConfig; +} + +namespace bts::md { +class IMarketDataListener; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::md::ftx { + +/** + * @brief Subscribes to FTX market data + * + * @details Uses ccapi infrastructure to request/subscribe data + */ +class FTXSubscription { + public: + FTXSubscription( + ccapi::EventDispatcher *dispatcher, + api::md::FTXMarketDataConfig cfg, + persistence::IDGenerator *sec_id_store, + IMarketDataListener *listener); + + FTXSubscription(const FTXSubscription &) noexcept = delete; + FTXSubscription(FTXSubscription &&) noexcept; + FTXSubscription &operator=(const FTXSubscription &) noexcept = delete; + FTXSubscription &operator=(FTXSubscription &&) noexcept; + + ~FTXSubscription(); + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/utils/Delegate.h b/src/lib/utils/Delegate.h index cae44c889..455ea11cd 100644 --- a/src/lib/utils/Delegate.h +++ b/src/lib/utils/Delegate.h @@ -36,9 +36,9 @@ class Delegate { static R invoke_static_handler(void *, Args... arg) { return handler(std::forward(arg)...); } - + static R null_target(Arg...) { - if constexpr (std::is_same::value) + if constexpr (std::is_same::value) return; else return {}; @@ -81,3 +81,10 @@ class Delegate { }; } // namespace bts::spartan + +namespace bts::core { + +template +using Delegate = bts::spartan::Delegate; + +} // namespace bts::core diff --git a/src/lib/utils/datetime.cpp b/src/lib/utils/datetime.cpp index f192a64e7..779b7c35a 100644 --- a/src/lib/utils/datetime.cpp +++ b/src/lib/utils/datetime.cpp @@ -190,7 +190,13 @@ std::string iso_date_time_string(PTime const &t) { PTime parse_iso_date_time_tz(const std::string &date) { // example 1) 2022-10-27T09:30-5:00 // example 2) 2022-10-27T09:30+5:00 + +#ifdef _WIN32 + struct tm tt = {0, 0, 0, 0, 0, 0, 0, 0, 0}; +#else struct tm tt = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +#endif + int tz_hours; int tz_minutes; int num_scanned = sscanf( @@ -213,7 +219,8 @@ PTime parse_iso_date_time_string(std::string const &t) { constexpr const std::size_t time_idx = sizeof("YYYY-MM-DD") - 1; auto actual_time_idx = t.find('T', time_idx); if (actual_time_idx != std::string::npos) { - constexpr const std::size_t time_zone_idx = time_idx + sizeof("HH:MM") - 1; + constexpr const std::size_t time_zone_idx = + time_idx + sizeof("HH:MM") - 1; // time zone must be after time bts_assert(actual_time_idx < time_zone_idx); if (t.find('+', time_zone_idx) != std::string::npos || diff --git a/src/server/sparkd/cmds/md/cmds/Run.cpp b/src/server/sparkd/cmds/md/cmds/Run.cpp index 1564531fb..29711f4e4 100644 --- a/src/server/sparkd/cmds/md/cmds/Run.cpp +++ b/src/server/sparkd/cmds/md/cmds/Run.cpp @@ -115,7 +115,7 @@ void Run::run( LOG(md, Info, "Run::run cleanup:{} run_on_md_thread:{} t_id:{} t_name:{}", cleanup, run_on_md_thread, os::get_thread_id(), get_thread_name()); if (type == Run::MDType::Configured && - !config::get().use_md()) { + !config::get().use_md()) { LOG(md, Info, "Ignoring request to run md: 'use_md' is configured false"); return; } @@ -126,7 +126,7 @@ void Run::run( if (cleanup) md::cmds::Cleanup::execute(); if (type == Run::MDType::Configured) { - const auto &conf = config::get(); + const auto &conf = config::get(); if (conf.use_replay()) type = Run::MDType::Replay; else if (conf.use_onixs()) @@ -155,7 +155,7 @@ void Run::run( if (is_run_emumba) { threading::make_thread("Emumba", []() { run_emumba(); }).detach(); } - if (bts::config::get().use_sim()) { + if (bts::config::get().use_sim()) { threading::make_thread("ExchangeSimGLBX", [&mbo_writer]() { sim::start_exchange_sim(api::Venue::VENUE_SIM, mbo_writer); }).detach();