From 64601914b2ce09ef8082d738dcd9d9de77639cb2 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 5 Jan 2023 11:20:34 -0500 Subject: [PATCH] feat: Add Binance US market data --- CMakeLists.txt | 4 + .../spark_api/security/SecurityExtensions.cs | 4 + .../api/md/BinanceUsMarketDataConfig.proto | 21 ++ src/lib/api/md/MarketDataConfig.proto | 2 + .../api/service_helpers/AdminServiceUtils.cpp | 6 + src/lib/config/profiles/default/Symbols.conf | 9 + src/lib/config/protos/symbols.proto | 1 + src/lib/domain_model/Security.proto | 7 + src/lib/md/MarketDataGateway.cpp | 8 + src/lib/md/MarketDataSubscription.cpp | 6 + src/lib/md/MarketDataSubscription.h | 5 + src/lib/md/PlatformLoader.cpp | 25 ++ src/lib/md/Shm/FixedSecurity.cpp | 10 + src/lib/md/binance/BinanceSpot.cpp | 4 +- src/lib/md/binance/BinanceSpot.h | 5 +- src/lib/md/binance_us/BinanceUsMessageType.h | 37 ++ .../binance_us/BinanceUsPriceStatistics.cpp | 125 +++++++ .../md/binance_us/BinanceUsPriceStatistics.h | 65 ++++ .../BinanceUsPriceStatisticsAdaptor.cpp | 75 ++++ .../BinanceUsPriceStatisticsAdaptor.h | 44 +++ .../binance_us/BinanceUsSecurityRequestor.cpp | 247 ++++++++++++ .../binance_us/BinanceUsSecurityRequestor.h | 80 ++++ .../md/binance_us/BinanceUsSubscription.cpp | 350 ++++++++++++++++++ src/lib/md/binance_us/BinanceUsSubscription.h | 54 +++ .../BinanceUsdsFuturesSubscription.cpp | 8 - src/lib/md/ccApi/CCMarketDataListener.cpp | 2 + src/lib/md/relay/QueryServer.cpp | 3 + .../sparkd/cmds/md/common/VenueOption.cpp | 3 + 28 files changed, 1199 insertions(+), 11 deletions(-) create mode 100644 src/lib/api/md/BinanceUsMarketDataConfig.proto create mode 100644 src/lib/md/binance_us/BinanceUsMessageType.h create mode 100644 src/lib/md/binance_us/BinanceUsPriceStatistics.cpp create mode 100644 src/lib/md/binance_us/BinanceUsPriceStatistics.h create mode 100644 src/lib/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp create mode 100644 src/lib/md/binance_us/BinanceUsPriceStatisticsAdaptor.h create mode 100644 src/lib/md/binance_us/BinanceUsSecurityRequestor.cpp create mode 100644 src/lib/md/binance_us/BinanceUsSecurityRequestor.h create mode 100644 src/lib/md/binance_us/BinanceUsSubscription.cpp create mode 100644 src/lib/md/binance_us/BinanceUsSubscription.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e105f5b36..7c4f28bd8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,6 +57,10 @@ else() add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo") add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr") + # ** Binance US ** + add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo") + add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr") + # ** Binance USDS Futures ** add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com") add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo") diff --git a/src/client/spark_api/security/SecurityExtensions.cs b/src/client/spark_api/security/SecurityExtensions.cs index 1cb71593e..ebe6be874 100644 --- a/src/client/spark_api/security/SecurityExtensions.cs +++ b/src/client/spark_api/security/SecurityExtensions.cs @@ -100,6 +100,8 @@ namespace bts.api { return security.Ftx.SecurityDesc; } else if (security.HasBinance) { return security.Binance.SecurityDesc; + } else if (security.HasBinanceUs) { + return security.BinanceUs.SecurityDesc; } else if (security.HasBinanceUsdsFutures) { return security.BinanceUsdsFutures.SecurityDesc; } @@ -144,6 +146,7 @@ namespace bts.api { domain.Venue.Deribit => Venue.Deribit, domain.Venue.Ftx => Venue.Ftx, domain.Venue.Binance => Venue.Binance, + domain.Venue.BinanceUs => Venue.BinanceUs, domain.Venue.BinanceUsdsFutures => Venue.BinanceUsdsFutures, domain.Venue.Glbx => Venue.Glbx, domain.Venue.None => Venue.Unset, @@ -157,6 +160,7 @@ namespace bts.api { Venue.Deribit => domain.Venue.Deribit, Venue.Ftx => domain.Venue.Ftx, Venue.Binance => domain.Venue.Binance, + Venue.BinanceUs => domain.Venue.BinanceUs, Venue.BinanceUsdsFutures => domain.Venue.BinanceUsdsFutures, Venue.Glbx => domain.Venue.Glbx, Venue.Unset => domain.Venue.None, diff --git a/src/lib/api/md/BinanceUsMarketDataConfig.proto b/src/lib/api/md/BinanceUsMarketDataConfig.proto new file mode 100644 index 000000000..4412b9786 --- /dev/null +++ b/src/lib/api/md/BinanceUsMarketDataConfig.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +import "lib/api/domain/SecurityCriteria.proto"; + +package bts.api.md; + +option csharp_namespace = "bts.api.md"; +option cc_enable_arenas = true; + +message BinanceUsMarketDataConfig { + uint32 num_depth_levels = 1; + uint32 depth_throttle_ms = 2; + bts.api.domain.SecurityCriteria security_criteria = 3; + // Market data isn't subscribed for securities listed here (even if they are + // included by 'security_criteria'). There are some securities that don't + // fit into our system just yet, such as 'BTCBIDR', which trades in prices + // larger than our cpp 'domain::Price' class supports. + repeated string excluded_securities = 4; + string url_rest_base = 5; + string url_ws_base = 6; +} diff --git a/src/lib/api/md/MarketDataConfig.proto b/src/lib/api/md/MarketDataConfig.proto index c1d9f0714..5f014504c 100644 --- a/src/lib/api/md/MarketDataConfig.proto +++ b/src/lib/api/md/MarketDataConfig.proto @@ -2,6 +2,7 @@ syntax = "proto3"; import "lib/api/md/FTXMarketDataConfig.proto"; import "lib/api/md/BinanceMarketDataConfig.proto"; +import "lib/api/md/BinanceUsMarketDataConfig.proto"; import "lib/api/md/BinanceUsdsFuturesMarketDataConfig.proto"; option csharp_namespace = "bts.api.md"; @@ -13,4 +14,5 @@ message MarketDataConfig { repeated FTXMarketDataConfig ftx = 1; repeated BinanceMarketDataConfig binance = 2; repeated BinanceUsdsFuturesMarketDataConfig binance_usds_futures = 3; + repeated BinanceUsMarketDataConfig binance_us = 4; } diff --git a/src/lib/api/service_helpers/AdminServiceUtils.cpp b/src/lib/api/service_helpers/AdminServiceUtils.cpp index 8ad05c182..946adaad4 100644 --- a/src/lib/api/service_helpers/AdminServiceUtils.cpp +++ b/src/lib/api/service_helpers/AdminServiceUtils.cpp @@ -54,6 +54,7 @@ bool AdminServiceUtils::has_exchange( case api::Venue::VENUE_DERIBIT: // TODO case api::Venue::VENUE_FTX: case api::Venue::VENUE_BINANCE: + case api::Venue::VENUE_BINANCE_US: case api::Venue::VENUE_BINANCE_USDS_FUTURES: case api::Venue::VENUE_BTS: case api::Venue::VENUE_CBOE: @@ -166,6 +167,7 @@ AdminServiceUtils::resolve_connection( case api::Venue::VENUE_DERIBIT: case api::Venue::VENUE_FTX: case api::Venue::VENUE_BINANCE: + case api::Venue::VENUE_BINANCE_US: case api::Venue::VENUE_BINANCE_USDS_FUTURES: rv = resolve_sim_connection(data, account, sec); break; @@ -418,6 +420,7 @@ std::string AdminServiceUtils::find_account_exchange_account( case api::Venue::VENUE_DERIBIT: // TODO case api::Venue::VENUE_FTX: // TODO case api::Venue::VENUE_BINANCE: // TODO + case api::Venue::VENUE_BINANCE_US: case api::Venue::VENUE_BINANCE_USDS_FUTURES: case api::Venue::VENUE_BTS: case api::Venue::VENUE_CBOE: @@ -697,6 +700,7 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record( } break; case api::VENUE_FTX: // TODO case api::VENUE_BINANCE: // TODO + case api::VENUE_BINANCE_US: case api::VENUE_BINANCE_USDS_FUTURES: case api::Venue::VENUE_UNSET: case api::Venue::VENUE_BTS: @@ -1089,6 +1093,7 @@ std::string AdminServiceUtils::find_user_exchange_id( case api::Venue::VENUE_DERIBIT: // TODO case api::Venue::VENUE_FTX: // TODO case api::Venue::VENUE_BINANCE: // TODO + case api::Venue::VENUE_BINANCE_US: case api::Venue::VENUE_BINANCE_USDS_FUTURES: case api::Venue::VENUE_UNSET: case api::Venue::VENUE_BTS: @@ -1120,6 +1125,7 @@ std::string AdminServiceUtils::find_user_company_id( case api::Venue::VENUE_DERIBIT: case api::Venue::VENUE_FTX: // TODO case api::Venue::VENUE_BINANCE: // TODO + case api::Venue::VENUE_BINANCE_US: case api::Venue::VENUE_BINANCE_USDS_FUTURES: case api::Venue::VENUE_SIM: case api::Venue::VENUE_SIMICE: diff --git a/src/lib/config/profiles/default/Symbols.conf b/src/lib/config/profiles/default/Symbols.conf index baf5e99b5..c8f725b93 100644 --- a/src/lib/config/profiles/default/Symbols.conf +++ b/src/lib/config/profiles/default/Symbols.conf @@ -33,6 +33,10 @@ SymbolLists { key: "BINANCE" symbol: @SymbolList.symbol } + list { + key: "BINANCE_US" + symbol: @SymbolList.symbol + } list { key: "BINANCE_USDS_FUTURES" symbol: @SymbolList.symbol @@ -106,6 +110,11 @@ Symbols { option: @SymbolLists.list[BINANCE].symbol spread: @SymbolLists.list[BINANCE].symbol } + binance_us { + future: @SymbolLists.list[BINANCE_US].symbol + option: @SymbolLists.list[BINANCE_US].symbol + spread: @SymbolLists.list[BINANCE_US].symbol + } binance_usds_futures { future: @SymbolLists.list[BINANCE_USDS_FUTURES].symbol option: @SymbolLists.list[BINANCE_USDS_FUTURES].symbol diff --git a/src/lib/config/protos/symbols.proto b/src/lib/config/protos/symbols.proto index 4337f39e1..3e74060d2 100644 --- a/src/lib/config/protos/symbols.proto +++ b/src/lib/config/protos/symbols.proto @@ -26,4 +26,5 @@ message Symbols { optional SecurityExchangeList ftx = 60; optional SecurityExchangeList binance = 70; optional SecurityExchangeList binance_usds_futures = 80; + optional SecurityExchangeList binance_us = 90; } diff --git a/src/lib/domain_model/Security.proto b/src/lib/domain_model/Security.proto index 82be7eec4..3560d044b 100644 --- a/src/lib/domain_model/Security.proto +++ b/src/lib/domain_model/Security.proto @@ -17,6 +17,7 @@ enum Venue { VENUE_FTX = 410; VENUE_BINANCE = 420; VENUE_BINANCE_USDS_FUTURES = 430; + VENUE_BINANCE_US = 440; } enum SecurityType { @@ -120,6 +121,11 @@ message BinanceUsdsFuturesExchangeData { string security_desc = 2; } +message BinanceUsExchangeData { + int64 exchange_id = 1; + string security_desc = 2; +} + message CBOECFEExchangeData { int64 exchange_id = 10; string security_desc = 20; @@ -223,6 +229,7 @@ message Security { FTXExchangeData ftx = 250; BinanceExchangeData binance = 260; BinanceUsdsFuturesExchangeData binance_usds_futures = 270; + BinanceUsExchangeData binance_us = 280; } } message SecurityList { diff --git a/src/lib/md/MarketDataGateway.cpp b/src/lib/md/MarketDataGateway.cpp index 775e04cf1..f06ab9d06 100644 --- a/src/lib/md/MarketDataGateway.cpp +++ b/src/lib/md/MarketDataGateway.cpp @@ -14,6 +14,7 @@ #include "lib/md/MarketDataRegistrationFactory.h" #include "lib/md/MarketDataSubscription.h" #include "lib/md/binance/BinanceSubscription.h" +#include "lib/md/binance_us/BinanceUsSubscription.h" #include "lib/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h" #include "lib/md/ftx/FTXSubscription.h" #include "lib/persistence/IDGenerator.h" @@ -90,6 +91,13 @@ MarketDataRegistration MarketDataGateway::subscribe( listener_data.listener}); } + // Binance US + for (const auto &binance_us : cfg.binance_us()) { + listener_data.subscription.add(binance_us::BinanceUsSubscription{ + &me_->dispatcher, binance_us, &me_->sec_id_store, + listener_data.listener}); + } + // Binance USDS Futures for (const auto &binance_ufs : cfg.binance_usds_futures()) { listener_data.subscription.add( diff --git a/src/lib/md/MarketDataSubscription.cpp b/src/lib/md/MarketDataSubscription.cpp index 4c49855a4..ceee67d56 100644 --- a/src/lib/md/MarketDataSubscription.cpp +++ b/src/lib/md/MarketDataSubscription.cpp @@ -2,6 +2,7 @@ #include "MarketDataSubscription.h" #include "lib/api/md/MarketDataConfig.pb.h" #include "lib/md/binance/BinanceSubscription.h" +#include "lib/md/binance_us/BinanceUsSubscription.h" #include "lib/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h" #include "lib/md/ftx/FTXSubscription.h" #include @@ -14,6 +15,7 @@ using std::vector; struct MarketDataSubscription::PImpl { vector ftx_subs; vector binance_subs; + vector binance_us_subs; vector binance_usds_futures_subs; // TODO: add more exchanges @@ -28,6 +30,10 @@ void MarketDataSubscription::add(binance::BinanceSubscription sub) { me_->binance_subs.push_back(std::move(sub)); } +void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) { + me_->binance_us_subs.push_back(std::move(sub)); +} + void MarketDataSubscription::add( binance_usds_futures::BinanceUsdsFuturesSubscription sub) { me_->binance_usds_futures_subs.push_back(std::move(sub)); diff --git a/src/lib/md/MarketDataSubscription.h b/src/lib/md/MarketDataSubscription.h index f39238699..54370a08b 100644 --- a/src/lib/md/MarketDataSubscription.h +++ b/src/lib/md/MarketDataSubscription.h @@ -6,6 +6,10 @@ namespace bts::md::binance { class BinanceSubscription; } +namespace bts::md::binance_us { +class BinanceUsSubscription; +} + namespace bts::md::binance_usds_futures { class BinanceUsdsFuturesSubscription; } @@ -30,6 +34,7 @@ class MarketDataSubscription { ~MarketDataSubscription(); void add(binance::BinanceSubscription); + void add(binance_us::BinanceUsSubscription); void add(binance_usds_futures::BinanceUsdsFuturesSubscription); void add(ftx::FTXSubscription); diff --git a/src/lib/md/PlatformLoader.cpp b/src/lib/md/PlatformLoader.cpp index 2593019b2..67bb011f9 100644 --- a/src/lib/md/PlatformLoader.cpp +++ b/src/lib/md/PlatformLoader.cpp @@ -285,6 +285,31 @@ size_t get_system_md_symbols( } } + { + if (exchange == api::VENUE_BINANCE_US || exchange == api::VENUE_UNSET) { + auto const &e = config::get().binance_us(); + if (type == api::SECURITYTYPE_FUTURE || + type == api::SECURITYTYPE_UNSET) { + add_symbols_to_result( + e.future(), api::VENUE_BINANCE_US, api::SECURITYTYPE_FUTURE, &result); + } + if (type == api::SECURITYTYPE_OPTION || + type == api::SECURITYTYPE_UNSET) { + add_symbols_to_result( + e.option(), api::VENUE_BINANCE_US, api::SECURITYTYPE_OPTION, &result); + } + if (type == api::SECURITYTYPE_STOCK || + type == api::SECURITYTYPE_UNSET) { + add_symbols_to_result( + e.option(), api::VENUE_BINANCE_US, api::SECURITYTYPE_STOCK, &result); + } + if (type == api::SECURITYTYPE_SPREAD || + type == api::SECURITYTYPE_UNSET) + add_symbols_to_result( + e.spread(), api::VENUE_BINANCE_US, api::SECURITYTYPE_SPREAD, &result); + } + } + { if (exchange == api::VENUE_BINANCE_USDS_FUTURES || exchange == api::VENUE_UNSET) { auto const &e = config::get().binance_usds_futures(); diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index b7518b631..a81d6e0ad 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -202,6 +202,10 @@ void convert_security(const FixedSecurity &source, Security *presult) { dest.mutable_binance()->set_security_desc(source.security_desc()); dest.mutable_binance()->set_exchange_id(source.numeric_id); break; + case Venue::VENUE_BINANCE_US: + dest.mutable_binance_us()->set_security_desc(source.security_desc()); + dest.mutable_binance_us()->set_exchange_id(source.numeric_id); + break; case Venue::VENUE_BINANCE_USDS_FUTURES: dest.mutable_binance_usds_futures()->set_security_desc( source.security_desc()); @@ -315,6 +319,8 @@ uint64_t exchange_id(const Security &security) { return security.ftx().exchange_id(); case Security::ExchangeDataCase::kBinance: return security.binance().exchange_id(); + case Security::ExchangeDataCase::kBinanceUs: + return security.binance_us().exchange_id(); case Security::ExchangeDataCase::kBinanceUsdsFutures: return security.binance_usds_futures().exchange_id(); // Debatable whether this should be an error or not. @@ -375,6 +381,10 @@ void convert_security( dest.numeric_id = src.binance().exchange_id(); dest.security_desc(src.binance().security_desc()); break; + case Security::ExchangeDataCase::kBinanceUs: + dest.numeric_id = src.binance_us().exchange_id(); + dest.security_desc(src.binance_us().security_desc()); + break; case Security::ExchangeDataCase::kBinanceUsdsFutures: dest.numeric_id = src.binance_usds_futures().exchange_id(); dest.security_desc(src.binance_usds_futures().security_desc()); diff --git a/src/lib/md/binance/BinanceSpot.cpp b/src/lib/md/binance/BinanceSpot.cpp index 045ceda20..03b203bdc 100644 --- a/src/lib/md/binance/BinanceSpot.cpp +++ b/src/lib/md/binance/BinanceSpot.cpp @@ -45,7 +45,7 @@ static std::string deduce_sym(const BinanceSpot &bspot) { } void BinanceSpot::parse( - const ccapi::Message &msg, vector &securities) { + const ccapi::Message &msg, vector &securities, Venue venue) { const auto &sec_elements = msg.getElementList(); for (const auto &sec_el : sec_elements) { const auto &sec_props = sec_el.getNameValueMap(); @@ -69,7 +69,7 @@ void BinanceSpot::parse( auto &sec = securities.emplace_back(); auto &scomm = sec.security_common; scomm.type = SecurityType::SPOT; - scomm.venue = Venue::BINANCE_USDS_FUTURES; + scomm.venue = venue; scomm.exchange_sec_id = bspot.symbol; scomm.desc = bspot.symbol; diff --git a/src/lib/md/binance/BinanceSpot.h b/src/lib/md/binance/BinanceSpot.h index 2e195152d..2a210cd07 100644 --- a/src/lib/md/binance/BinanceSpot.h +++ b/src/lib/md/binance/BinanceSpot.h @@ -1,5 +1,6 @@ #pragma once #ifdef SPARK_ENABLE_CCAPI +#include "lib/api/domain/Venue.h" #include "lib/md/binance/BinanceContractType.h" #include "lib/md/binance/BinanceFilterTuple.h" #include "lib/md/binance/BinanceInstrumentStatus.h" @@ -47,7 +48,9 @@ class BinanceSpot { BinanceFilterTuple filter_tuple; static void parse( - const ccapi::Message &msg, std::vector &securities); + const ccapi::Message &msg, + std::vector &securities, + api::domain::Venue venue = api::domain::Venue::BINANCE); }; void from_json(const nlohmann::json &j, BinanceSpot &spot); diff --git a/src/lib/md/binance_us/BinanceUsMessageType.h b/src/lib/md/binance_us/BinanceUsMessageType.h new file mode 100644 index 000000000..540717558 --- /dev/null +++ b/src/lib/md/binance_us/BinanceUsMessageType.h @@ -0,0 +1,37 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/core/EnumMacros.h" +#include "lib/core/FMT_IMPL.h" +#include +#include +#include + +namespace bts::md::binance_us { + +// clang-format off +#define FOR_ALL_VALUES(per_value) \ + per_value(BinanceUsMessageType, NONE) \ + per_value(BinanceUsMessageType, EXCHANGE_INFO) \ + per_value(BinanceUsMessageType, DEPTH) \ + per_value(BinanceUsMessageType, MINI_TICKER) \ + per_value(BinanceUsMessageType, TICKER) \ + per_value(BinanceUsMessageType, STATISTICS) +// clang-format on + +/** + * @brief Represents the different types of messages received from Binance US + * + * @details Each enum value is associated with a particular type of request (or + * subscription) that is sent to the exchange, and a particular handler type + * designed to adapt the messages for an IMarketDataListener. + * Note: Some message types are native to the exchange, while others are CCApi + * constructs. + */ +ENUM_CLASS_DEF(BinanceUsMessageType, FOR_ALL_VALUES) + +#undef FOR_ALL_VALUES +} // namespace bts::md::binance_us + +FMT_IMPL(bts::md::binance_us::BinanceUsMessageType, t, "{}", t.name()); + +#endif diff --git a/src/lib/md/binance_us/BinanceUsPriceStatistics.cpp b/src/lib/md/binance_us/BinanceUsPriceStatistics.cpp new file mode 100644 index 000000000..52cd6d48b --- /dev/null +++ b/src/lib/md/binance_us/BinanceUsPriceStatistics.cpp @@ -0,0 +1,125 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "BinanceUsPriceStatistics.h" +#include "lib/core/TimeStamp.h" +#include "lib/domain/Price.h" +#include "lib/domain/Quantity.h" +#include "lib/domain/SecurityMarketDataStatistics.h" +#include "lib/io/JsonDumper.h" +#include "lib/io/JsonReader.h" +#include "lib/md/CCApiReader.h" + +#include +#include +#include + +namespace bts::md::binance_us { + +using bts::core::TimeStamp; +using bts::domain::Price; +using bts::domain::Quantity; +using bts::domain::SecurityID; +using bts::domain::SecurityMarketDataStatistics; +using bts::io::JsonReader; +using std::int64_t; +using std::map; +using std::ostream; +using std::string; +using std::vector; + +static bool parse( + const map &name_values, + vector &result) { + CCApiReader ccapi_reader{&name_values}; + nlohmann::json js; + if (!ccapi_reader.read(CCAPI_HTTP_BODY, js)) { + return false; + } + JsonReader jr{&js}; + if (!jr.read(result)) { + LOG(md, Error, "Failed to read BinanceUsPriceChangeStatistics json:\n{}", + js); + return false; + } + return true; +} + +bool BinanceUsPriceStatistics::parse( + const ccapi::Message &msg, vector &result) { + auto orig_sz = result.size(); + for (const auto &sec_el : msg.getElementList()) { + const auto &hp_props = sec_el.getNameValueMap(); + auto it = hp_props.find(CCAPI_HTTP_BODY); + if (it == hp_props.end()) { + continue; + } + bts::md::binance_us::parse(hp_props, result); + } + return result.size() > orig_sz; +} + +void BinanceUsPriceStatistics::to_security_statistics( + SecurityMarketDataStatistics &result) const { + result.open = Price::from_double(open_price); + result.settle = Price::from_double(prev_close_price); + result.high = Price::from_double(high_price); + result.low = Price::from_double(low_price); + result.volume.value() = static_cast(quote_volume); +} + +void from_json(const nlohmann::json &j, BinanceUsPriceStatistics &result) { + JsonReader jr{&j}; + jr.read("symbol", result.symbol); + jr.read("priceChange", result.price_change); + jr.read("priceChangePercent", result.price_change_percent); + jr.read("weightedAvgPrice", result.weighted_avg_price); + jr.read("prevClosePrice", result.prev_close_price); + jr.read("lastPrice", result.last_price); + jr.read("lastQty", result.last_qty); + jr.read("bidPrice", result.bid_price); + jr.read("bidQty", result.bid_qty); + jr.read("askPrice", result.ask_price); + jr.read("askQty", result.ask_qty); + jr.read("openPrice", result.open_price); + jr.read("highPrice", result.high_price); + jr.read("lowPrice", result.low_price); + jr.read("volume", result.volume); + jr.read("quoteVolume", result.quote_volume); + result.open_time = TimeStamp::from_millis(jr.read("openTime")); + result.close_time = TimeStamp::from_millis(jr.read("closeTime")); + jr.read("firstId", result.first_id); + jr.read("lastId", result.last_id); + jr.read("count", result.count); +} + +void to_json(nlohmann::json &j, const BinanceUsPriceStatistics &p) { + j = nlohmann::json{ + {"symbol", p.symbol}, + {"priceChange", p.price_change}, + {"priceChangePercent", p.price_change_percent}, + {"weightedAvgPrice", p.weighted_avg_price}, + {"prevClosePrice", p.prev_close_price}, + {"lastPrice", p.last_price}, + {"lastQty", p.last_qty}, + {"bidPrice", p.bid_price}, + {"bidQty", p.bid_qty}, + {"askPrice", p.ask_price}, + {"askQty", p.ask_qty}, + {"openPrice", p.open_price}, + {"highPrice", p.high_price}, + {"lowPrice", p.low_price}, + {"volume", p.volume}, + {"quoteVolume", p.quote_volume}, + {"openTime", static_cast(p.open_time.millis())}, + {"closeTime", static_cast(p.close_time.millis())}, + {"firstId", p.first_id}, + {"lastId", p.last_id}, + {"count", p.count}}; +} + +ostream &operator<<(ostream &out, const BinanceUsPriceStatistics &p) noexcept { + out << io::JsonDumper::dump(p); + return out; +} + +} // namespace bts::md::binance_us +#endif diff --git a/src/lib/md/binance_us/BinanceUsPriceStatistics.h b/src/lib/md/binance_us/BinanceUsPriceStatistics.h new file mode 100644 index 000000000..a55184071 --- /dev/null +++ b/src/lib/md/binance_us/BinanceUsPriceStatistics.h @@ -0,0 +1,65 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/core/TimeStamp.h" + +#include +#include +#include +#include +#include + +namespace ccapi { +class Message; +} + +namespace bts::domain { +class SecurityMarketDataStatistics; +} + +namespace bts::md::binance_us { + +/** + * @brief Represents a '24hr Price Change Statistics' update at Binance US + * + * @a https://docs.binance.us/#get-24h-price-change-statistics + */ +class BinanceUsPriceStatistics { + public: + std::string symbol; + double price_change; + double price_change_percent; + double weighted_avg_price; + double prev_close_price; + double last_price; + double last_qty; + double bid_price; + double bid_qty; + double ask_price; + double ask_qty; + double open_price; + double high_price; + double low_price; + double volume; + double quote_volume; + core::TimeStamp open_time; + core::TimeStamp close_time; + std::int64_t first_id; + std::int64_t last_id; + std::uint32_t count; + + static bool parse( + const ccapi::Message &msg, + std::vector &result); + + void to_security_statistics( + domain::SecurityMarketDataStatistics &result) const; +}; + +void from_json(const nlohmann::json &j, BinanceUsPriceStatistics &result); +void to_json(nlohmann::json &j, const BinanceUsPriceStatistics &p); + +std::ostream &operator<<( + std::ostream &out, const BinanceUsPriceStatistics &p) noexcept; + +} // namespace bts::md::binance_us +#endif diff --git a/src/lib/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp b/src/lib/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp new file mode 100644 index 000000000..6a799541d --- /dev/null +++ b/src/lib/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp @@ -0,0 +1,75 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "BinanceUsPriceStatisticsAdaptor.h" +#include "lib/domain/Security.h" +#include "lib/domain/SecurityID.h" +#include "lib/domain/SecurityMarketDataStatistics.h" +#include "lib/md/IMarketDataListener.h" +#include "lib/md/binance_us/BinanceUsPriceStatistics.h" + +#include +#include +#include + +namespace bts::md::binance_us { + +using bts::domain::SecurityID; +using bts::domain::SecurityMarketDataStatistics; +using std::pair; +using std::string; +using std::vector; + +#define VENUE_NAME (Venue_Name(Venue::BINANCE)) + +struct BinanceUsPriceStatisticsAdaptor::PImpl { + vector> securities; + IMarketDataListener *listener; + + SecurityID get_sec_id(const string &sec_desc) const { + for (const auto &[desc, id] : securities) { + if (desc == sec_desc) { + return id; + } + } + bts_assert_msgf( + false, + "Failed to find domain::SecurityID for Binance US security '{}'", + sec_desc); + return SecurityID{0}; + } +}; + +BinanceUsPriceStatisticsAdaptor::BinanceUsPriceStatisticsAdaptor( + vector> securities, IMarketDataListener *listener) + : me_(std::make_unique()) { + bts_assert(listener != nullptr); + me_->securities = std::move(securities); + me_->listener = listener; +} + +BinanceUsPriceStatisticsAdaptor::~BinanceUsPriceStatisticsAdaptor() = default; + +void BinanceUsPriceStatisticsAdaptor::on_messages( + const ccapi::Event & /*event*/, + ccapi::Session * /*session*/, + const vector &msgs, + const CCApiCorrelationID & /*corr_id*/) { + vector stats_recs; + for (const auto *msg : msgs) { + vector binance_stats_recs; + if (!BinanceUsPriceStatistics::parse(*msg, binance_stats_recs)) { + continue; + } + for (const auto &bsr : binance_stats_recs) { + auto &sec_stats = stats_recs.emplace_back(); + bsr.to_security_statistics(sec_stats); + sec_stats.security_id = me_->get_sec_id(bsr.symbol); + } + break; + } + for (const auto &stats_rec : stats_recs) { + me_->listener->on_security_statistics(stats_rec); + } +} + +} // namespace bts::md::binance_us +#endif diff --git a/src/lib/md/binance_us/BinanceUsPriceStatisticsAdaptor.h b/src/lib/md/binance_us/BinanceUsPriceStatisticsAdaptor.h new file mode 100644 index 000000000..a080f5229 --- /dev/null +++ b/src/lib/md/binance_us/BinanceUsPriceStatisticsAdaptor.h @@ -0,0 +1,44 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/domain/SecurityID.h" +#include "lib/md/ICCApiMessageHandler.h" +#include +#include +#include + +namespace bts::md { +class IMarketDataListener; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::md::binance_us { + +/** + * @brief Adapts Binance US '24hr Ticker Price Change Statistics' updates for + * an IMarketDataListener, by translating and forwarding event payloads to the + * listener via 'IMarketDataListener::on_security_statistics'. + */ +class BinanceUsPriceStatisticsAdaptor : public ICCApiMessageHandler { + public: + BinanceUsPriceStatisticsAdaptor( + std::vector> securities, + IMarketDataListener *listener); + + ~BinanceUsPriceStatisticsAdaptor(); + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md::binance_us +#endif diff --git a/src/lib/md/binance_us/BinanceUsSecurityRequestor.cpp b/src/lib/md/binance_us/BinanceUsSecurityRequestor.cpp new file mode 100644 index 000000000..be35f7270 --- /dev/null +++ b/src/lib/md/binance_us/BinanceUsSecurityRequestor.cpp @@ -0,0 +1,247 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "BinanceUsSecurityRequestor.h" +#include "lib/api/domain/SecurityType.pb.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/api/md/BinanceMarketDataConfig.pb.h" +#include "lib/collections/Contains.h" +#include "lib/collections/Remove.h" +#include "lib/domain/Security.h" +#include "lib/domain/SecurityCriteria.h" +#include "lib/logging/logging.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/CCApiMessageHandler.h" +#include "lib/md/CCApiSession.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/binance/BinanceMessageType.h" +#include "lib/md/binance/BinanceSpot.h" +#include "lib/persistence/IDGenerator.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bts::md::binance_us { + +using api::domain::SecurityType; +using api::domain::Venue; +using api::domain::Venue_Name; +using bts::domain::Security; +using bts::domain::SecurityCriteria; +using bts::domain::SecurityID; +using bts::md::binance::BinanceMessageType; +using bts::md::binance::BinanceSpot; +using bts::persistence::IDGenerator; +using std::string; +using std::unordered_set; +using std::vector; + +#define VENUE Venue::BINANCE_US +#define VENUE_NAME (Venue_Name(Venue::BINANCE_US)) +#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_US +#define BINANCE_MSG_TYPE_NAME(x) \ + std::string { BinanceMessageType{x}.name() } + +static ccapi::Request exchange_info_req() { + auto corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, BINANCE_MSG_TYPE_NAME(BinanceMessageType::EXCHANGE_INFO)); + ccapi::Request req{ + ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, CCAPI_VENUE_NAME}; + req.appendParam( + {{CCAPI_HTTP_PROTOCOL, "https"}, + {CCAPI_HTTP_PORT, "443"}, + {CCAPI_HTTP_PATH, BINANCE_US_INSTRUMENTS_PATH}, + {CCAPI_HTTP_METHOD, "GET"}}); + req.setCorrelationId(corr_id.to_string()); + return req; +} + +static vector get_sec_descs(const vector &sec_ptrs) { + vector descs{sec_ptrs.size()}; + std::transform( + sec_ptrs.begin(), sec_ptrs.end(), descs.begin(), + [](const auto &s) { return s.desc(); }); + return descs; +} + +static void validate_response(const ccapi::Event &event, bool is_expected) { + bts_assert_msgf( + event.getType() == ccapi::Event::Type::RESPONSE, + "Received event w/ unexpected type:\n{}", event.toStringPretty()); + bts_assert_msgf( + is_expected, "Received unexpected response from Binance US:\n{}", + event.toStringPretty()); +} + +struct BinanceUsSecurityRequestor::PImpl { + // General Notes: + // Securities are in the 'symbols' field (list) in the json from + // https://api.binance.us/api/v3/exchangeInfo + + // The session automatically starts/stops in its ctr/dtr + CCApiSession *const session; + // Maps security descs (strings) to numeric ids + IDGenerator *const sec_id_store; + + // 'criteria' and 'excluded_securities' collectively form the security + // filter predicate. Securities are forwarded to the callback if and only if + // they are included by 'criteria', and absent from 'excluded_securities'. + const SecurityCriteria criteria; + const unordered_set excluded_securities; + const unordered_set securities; + + // These proxy handlers don't actually process new securities. Member methods + // in this class process the securities; we simply bind them to proxy handler + // callbacks. This is easier than defining separate ICCApiMessageHandler + // impls for 'markets', and 'futures', esp since they would need access to + // state contained in this class. + CCApiMessageHandler exch_info_handler_proxy; + Callback callback; + + // True when an 'exchange info' request is in flight + bool exch_info_req_pending = false; + // True when someone called BinanceUsSecurityRequestor::request while another + // request was already in flight. In this case, the 'external_req_pending' + // flag is used to signal that another request should be sent immediately + // after the current one finishes. + bool external_req_pending = false; + + explicit PImpl( + CCApiSession *const session, + domain::SecurityCriteria criteria, + persistence::IDGenerator *const sec_id_store, + unordered_set excluded_securities) + : session(session), sec_id_store(sec_id_store), + criteria(std::move(criteria)), + excluded_securities(std::move(excluded_securities)) { + exch_info_handler_proxy.callback().bind<&PImpl::on_exchange_info>(*this); + } + + bool is_busy() const { return exch_info_req_pending; } + + void request() { + external_req_pending = false; + exch_info_req_pending = true; + session->request(exchange_info_req(), &exch_info_handler_proxy); + } + + /** + * @brief Parses securities from a ccapi::Message + * + * @details Only new securities that satisfy the 'SecurityCriteria' are added + * to \p added_secs + * + * @tparam TParser + * @param msg The ccapi::Message to parse + * @param types The types of securities to parse + * @param parser Ther parser to use + */ + template + void parse_secs( + const ccapi::Message &msg, + const vector &types, + vector &added_secs) { + T::parse(msg, added_secs, VENUE); + // Exclude secs that don't satisfy the SecurityCriteria + auto is_sec_excluded = [this, &types](const Security &sec) { + return !CONTAINS(types, sec.type()) || + !criteria.includes_or_undefined(sec) || + CONTAINS(excluded_securities, sec.desc()) || + securities.count(sec.desc()) > 0; + }; + REMOVE_IF(added_secs, is_sec_excluded); + // Assign security ids + for (auto &sec : added_secs) { + // Since the ids haven't been set yet, we should expect them to be 0 + bts_assert(sec.id().value() == 0); + SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + sec.security_common.id = sec_id; + } + } + + /** + * @brief Parses securities from ccapi messages, updates the securities set, + * and subscribes depth for newly arrived securities + * + * @details Securities that don't pass the criteria filter are ignored + * + * @tparam TParser The type of parser + * @param msgs Vector of ccapi messages containing securities + * @param types The types of securities to update (types must be explicitly + * specified to be included) + */ + template + void update_securities( + const vector &msgs, + const vector &types) { + vector added_secs; + for (const auto *msg : msgs) { + bts_assert_msgf( + msg->getType() == ccapi::Message::Type::GENERIC_PUBLIC_REQUEST, + "Received msg w/ unexpected type:\n{}", msg->toStringPretty()); + parse_secs(*msg, types, added_secs); + } + auto added_sec_descs = get_sec_descs(added_secs); + LOG(md, Info, "Received Binance US securities:\n{}", + fmt::to_string(fmt::join(added_sec_descs, "\n"))); + callback(added_secs); + } + + /** + * Handles 'exchange info' update from CCApiSession + */ + bool on_exchange_info( + const ccapi::Event &event, + ccapi::Session * /*session*/, + const vector &msgs, + const CCApiCorrelationID & /*corr_id*/) { + validate_response(event, exch_info_req_pending); + LOG(md, Diag, "Received Binance US exchange info (spots):\n{}", + event.toStringPretty()); + update_securities(msgs, {SecurityType::SPOT}); + exch_info_req_pending = false; + if (external_req_pending && !is_busy()) { + request(); + } + return true; + } +}; + +BinanceUsSecurityRequestor::BinanceUsSecurityRequestor( + CCApiSession *const session, + SecurityCriteria criteria, + IDGenerator *const sec_id_store, + unordered_set excluded_securities) + : me_(std::make_unique( + session, + std::move(criteria), + sec_id_store, + std::move(excluded_securities))) {} + +BinanceUsSecurityRequestor::BinanceUsSecurityRequestor( + BinanceUsSecurityRequestor &&) noexcept = default; +BinanceUsSecurityRequestor &BinanceUsSecurityRequestor::operator=( + BinanceUsSecurityRequestor &&) noexcept = default; + +BinanceUsSecurityRequestor::~BinanceUsSecurityRequestor() = default; + +void BinanceUsSecurityRequestor::request() { + if (is_busy()) { + me_->external_req_pending = true; + return; + } + me_->request(); +} + +bool BinanceUsSecurityRequestor::is_busy() const { return me_->is_busy(); } + +BinanceUsSecurityRequestor::Callback &BinanceUsSecurityRequestor::callback() { + return me_->callback; +} + +} // namespace bts::md::binance_us +#endif diff --git a/src/lib/md/binance_us/BinanceUsSecurityRequestor.h b/src/lib/md/binance_us/BinanceUsSecurityRequestor.h new file mode 100644 index 000000000..97b4beae6 --- /dev/null +++ b/src/lib/md/binance_us/BinanceUsSecurityRequestor.h @@ -0,0 +1,80 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/domain/SecurityID.h" +#include "lib/utils/Delegate.h" +#include +#include +#include +#include + +namespace ccapi { +class EventDispatcher; +} + +namespace bts::md { +class IMarketDataListener; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::domain { +class Security; +class SecurityCriteria; +} // namespace bts::domain + +namespace bts::md { +class CCApiSession; +} + +namespace bts::md::binance_us { + +/** + * @brief Requests securities from Binance US + */ +class BinanceUsSecurityRequestor { + public: + BinanceUsSecurityRequestor( + CCApiSession *const session, + domain::SecurityCriteria criteria, + persistence::IDGenerator *const sec_id_store, + std::unordered_set excluded_securities = {}); + + BinanceUsSecurityRequestor(const BinanceUsSecurityRequestor &) noexcept = + delete; + BinanceUsSecurityRequestor(BinanceUsSecurityRequestor &&) noexcept; + BinanceUsSecurityRequestor &operator=( + const BinanceUsSecurityRequestor &) noexcept = delete; + BinanceUsSecurityRequestor &operator=( + BinanceUsSecurityRequestor &&) noexcept; + + ~BinanceUsSecurityRequestor(); + + using Callback = core::Delegate)>; + + Callback &callback(); + + /** + * @brief Request securities from the exchange + * + * @details Does NOT generate a live-updating stream of securities. Callers + * must call 'request' periodically to keep their securities in sync with the + * active set at the exchange. + */ + void request(); + + /** + * @brief Indicates whether or not a request is in-progress + * + * @details Concurrent requests are not supported + */ + bool is_busy() const; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md::binance_us +#endif diff --git a/src/lib/md/binance_us/BinanceUsSubscription.cpp b/src/lib/md/binance_us/BinanceUsSubscription.cpp new file mode 100644 index 000000000..5716043de --- /dev/null +++ b/src/lib/md/binance_us/BinanceUsSubscription.cpp @@ -0,0 +1,350 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "BinanceUsSubscription.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/api/md/BinanceUsMarketDataConfig.pb.h" +#include "lib/collections/Contains.h" +#include "lib/collections/UnorderedSetFactory.h" +#include "lib/core/FMT.h" +#include "lib/core/StringJoiner.h" +#include "lib/domain/Security.h" +#include "lib/domain/SecurityCriteria.h" +#include "lib/domain/SecurityMarketDataStatistics.h" +#include "lib/io/ProtoPrinter.h" +#include "lib/io/QueryParams.h" +#include "lib/logging/logging.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/CCApiMarketDepthAdaptor.h" +#include "lib/md/CCApiMessageHandler.h" +#include "lib/md/CCApiMessageLogger.h" +#include "lib/md/CCApiSession.h" +#include "lib/md/CCApiTradeAdaptor.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/IMarketDataListener.h" +#include "lib/md/binance/BinanceSpot.h" +#include "lib/md/binance_us/BinanceUsMessageType.h" +#include "lib/md/binance_us/BinanceUsPriceStatistics.h" +#include "lib/md/binance_us/BinanceUsPriceStatisticsAdaptor.h" +#include "lib/md/binance_us/BinanceUsSecurityRequestor.h" +#include "lib/persistence/IDGenerator.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bts::md::binance_us { + +using bts::api::domain::Venue; +using bts::api::domain::Venue_Name; +using bts::api::md::BinanceUsMarketDataConfig; +using bts::domain::Security; +using bts::domain::SecurityCriteria; +using bts::domain::SecurityID; +using bts::io::ProtoPrinter; +using bts::io::QueryParams; +using bts::persistence::IDGenerator; +using std::pair; +using std::string; +using std::uint32_t; +using std::unique_ptr; +using std::unordered_map; +using std::vector; + +#define VENUE_NAME (Venue_Name(Venue::BINANCE_US)) +#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_US + +/** + * @brief Apply overrides from our config to a ccapi::SessionConfigs object + * that is otherwise populated with defaults + */ +static ccapi::SessionConfigs mk_session_cfg( + const BinanceUsMarketDataConfig &cfg) { + ccapi::SessionConfigs rv; + if (!cfg.url_rest_base().empty()) { + rv.urlRestBase[CCAPI_VENUE_NAME] = cfg.url_rest_base(); + } + if (!cfg.url_ws_base().empty()) { + rv.urlWebsocketBase[CCAPI_VENUE_NAME] = cfg.url_ws_base(); + } + return rv; +} + +static string mk_depth_opts(const BinanceUsMarketDataConfig &cfg) { + // For details on supported opts, read the docs + // https://binance-docs.github.io/apidocs/spot/en/#partial-book-depth-streams + QueryParams opts; + switch (cfg.num_depth_levels()) { + case 5: + case 10: + case 20: + // The exchange only supports these num levels + opts[CCAPI_MARKET_DEPTH_MAX] = std::to_string(cfg.num_depth_levels()); + break; + case 0: + // Leave num levels unspecified (let exchange choose default) + break; + default: + LOG(md, Warn, "Invalid num depth levels '{}'. Must be one of [5, 10, 20]", + cfg.num_depth_levels()); + break; + } + + switch (cfg.depth_throttle_ms()) { + case 100: + case 1000: + // The exchange only supports these throttle speeds + opts[CCAPI_MARKET_DEPTH] = std::to_string(cfg.depth_throttle_ms()); + break; + case 0: + // Leave throttle unspecified (let exchange choose default) + break; + default: + LOG(md, Warn, + "Invalid depth throttle '{}ms'. Must be one of [100, 250, 500])", + cfg.depth_throttle_ms()); + break; + } + + return opts.to_string(); +} + +static ccapi::Subscription mk_depth_sub( + const string &sec_desc, const string &opts) { + auto corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, BinanceUsMessageType::DEPTH, sec_desc); + return ccapi::Subscription{ + CCAPI_VENUE_NAME, sec_desc, CCAPI_MARKET_DEPTH, opts, + corr_id.to_string()}; +} + +static CCApiCorrelationID mk_stats_corr_id(const string &desc) { + return CCApiCorrelationID::from_parts( + VENUE_NAME, BinanceUsMessageType(BinanceUsMessageType::STATISTICS), + desc); +} + +static ccapi::Request mk_stats_req( + const vector &sec_descs, uint32_t batch_id) { + const auto corr_id = mk_stats_corr_id(FMT("BATCH_{}", batch_id)); + ccapi::Request req{ + ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, CCAPI_VENUE_NAME}; + req.setCorrelationId(corr_id.to_string()); + auto quoted_sec_descs = + sec_descs | + ranges::views::transform([](const auto &s) { return "\"" + s + "\""; }); + const string symbols_query_arg = FMT_JOIN(quoted_sec_descs, ","); + req.appendParam( + {{CCAPI_HTTP_PATH, BINANCE_PRICE_STATS_PATH}, + {CCAPI_HTTP_QUERY_STRING, FMT("symbols=[{}]", symbols_query_arg)}, + {CCAPI_HTTP_METHOD, "GET"}}); + return req; +} + +static string mk_ticker_opts(const BinanceUsMarketDataConfig & /*cfg*/) { + // Placeholder (for consistency's sake) + return ""; +} + +static ccapi::Subscription mk_ticker_sub( + const string &sec_desc, const string &opts) { + auto ticker_corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, BinanceUsMessageType::TICKER, sec_desc); + return ccapi::Subscription{ + CCAPI_VENUE_NAME, sec_desc, CCAPI_TRADE, opts, + ticker_corr_id.to_string()}; +} + +struct BinanceUsSubscription::PImpl { + // General Notes: + // All securities are listed in the 'symbols' field of the json response + // to the 'exchangeInfo' request. + + unordered_map> securities; + IDGenerator *sec_id_store; + + // Don't actually need this, but keeping it around for diagnostic purposes + // (i.e., inspection in debugger, temp logs, etc.) + const BinanceUsMarketDataConfig cfg; + const SecurityCriteria criteria; + + // The session automatically starts/stops in its ctr/dtr + CCApiSession session; + IMarketDataListener *listener; + BinanceUsSecurityRequestor sec_requestor; + + // These aliases are useful for quickly switching the handler types (e.g., + // a special handler that logs messages for testing/diagnostic purposes) + using DepthHandler = CCApiMarketDepthAdaptor; + using TickerHandler = CCApiTradeAdaptor; + using StatsHandler = BinanceUsPriceStatisticsAdaptor; + + unordered_map depth_handlers; + unordered_map ticker_handlers; + // 'sec_stats_handler_map' maps security (description) to the StatsHandler + // subscribed to that security's data. Since each StatsHandler processes + // data for multiple securities, the actual instances are contained in a + // different map, 'stats_handlers' (unique ids are generated w/ a counter, + // 'stats_handlers_id_counter'). + unordered_map sec_stats_handler_map; + unordered_map> stats_handlers; + uint32_t stats_handlers_id_counter = 0; + + explicit PImpl( + ccapi::EventDispatcher *dispatcher, + BinanceUsMarketDataConfig cfg, + IDGenerator *sec_id_store, + IMarketDataListener *listener) + : sec_id_store(sec_id_store), cfg(std::move(cfg)), + criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), + session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener), + sec_requestor( + &session, + this->criteria, + sec_id_store, + collections::UnorderedSetFactory::create( + this->cfg.excluded_securities().begin(), + this->cfg.excluded_securities().end())) { + LOG(md, Info, "Initializing BinanceUsSubscription:\n{}", + ProtoPrinter::to_string(this->cfg)); + sec_requestor.callback().bind<&PImpl::on_securities>(*this); + sec_requestor.request(); + } + + DepthHandler &mk_depth_handler(const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + auto [it, is_new] = + depth_handlers.try_emplace(sec_desc, sec_id, listener); + bts_assert(is_new); + return it->second; + } + + TickerHandler &mk_ticker_handler(const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + auto [it, is_new] = + ticker_handlers.try_emplace(sec_desc, sec_id, listener); + bts_assert_msgf( + is_new, "Ticker handler already created for '{}'", sec_desc); + return it->second; + } + + StatsHandler &mk_stats_handler( + const vector &sec_descs, uint32_t hnd_id) { + vector> sec_desc_id_pairs; + sec_desc_id_pairs.reserve(sec_descs.size()); + for (const auto &sec_desc : sec_descs) { + sec_desc_id_pairs.emplace_back( + sec_desc, + SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}); + } + auto [it, is_new] = stats_handlers.try_emplace( + hnd_id, std::make_unique(sec_desc_id_pairs, listener)); + bts_assert_msgf( + is_new, "StatsHandler already created with id '{}'", hnd_id); + return *it->second; + } + + void add_stats_handler( + const vector &sec_descs, CCApiSession::BatchRequest &reqs) { + if (sec_descs.empty()) { + return; + } + const auto hnd_id = stats_handlers_id_counter++; + reqs.push_back( + {mk_stats_req(sec_descs, hnd_id), + &mk_stats_handler(sec_descs, hnd_id)}); + } + + /** + * @brief Subscribes to depth, ticker, and price data for a list of + * securities + */ + template + void subscribe_market_data(SecDescs securities) { + const auto depth_opts = mk_depth_opts(cfg); + const auto ticker_opts = mk_ticker_opts(cfg); + + // Batch subscribe/request ticker, depth, and stats + CCApiSession::BatchSubscription depth_subs; + CCApiSession::BatchSubscription ticker_subs; + CCApiSession::BatchRequest stats_reqs; + + const int N_SECS_PER_STATS_REQ = 16; + vector curr_stats_sec_descs; + for (const auto &sec_desc : securities) { + depth_subs.push_back( + {mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)}); + ticker_subs.push_back( + {mk_ticker_sub(sec_desc, ticker_opts), + &mk_ticker_handler(sec_desc)}); + curr_stats_sec_descs.push_back(sec_desc); + if (curr_stats_sec_descs.size() == N_SECS_PER_STATS_REQ) { + add_stats_handler(curr_stats_sec_descs, stats_reqs); + curr_stats_sec_descs.clear(); + } + } + if (!curr_stats_sec_descs.empty()) { + add_stats_handler(curr_stats_sec_descs, stats_reqs); + } + + session.subscribe( + depth_subs, {ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH}); + session.subscribe( + ticker_subs, {ccapi::Message::Type::MARKET_DATA_EVENTS_TRADE}); + session.request( + stats_reqs, {ccapi::Message::Type::GENERIC_PUBLIC_REQUEST}); + } + + /** + * Updates the securities map and subscribes market data for newly arrived + * securities + */ + void on_securities(vector secs) { + // 'sec_descs' is used to make depth subscriptions + vector sec_descs; + sec_descs.reserve(secs.size()); + // 'sec_ptrs' is passed to IMarketDataListeners + vector sec_ptrs; + sec_ptrs.reserve(secs.size()); + + for (auto &s : secs) { + auto id = s.id(); + auto desc = s.desc(); + auto [it, is_new] = securities.try_emplace( + id, std::make_unique(std::move(s))); + bts_assert_msgf( + is_new, "Received duplicate Binance US security '{}'", desc); + sec_descs.push_back(std::move(desc)); + // add ptr to entry in 'securities' (not the loop var, we moved that) + sec_ptrs.push_back(it->second.get()); + } + + listener->on_securities(sec_ptrs); + subscribe_market_data(sec_descs); + } +}; + +BinanceUsSubscription::BinanceUsSubscription( + ccapi::EventDispatcher *dispatcher, + BinanceUsMarketDataConfig cfg, + IDGenerator *sec_id_store, + IMarketDataListener *listener) + : me_(std::make_unique( + dispatcher, std::move(cfg), sec_id_store, listener)) {} + +BinanceUsSubscription::BinanceUsSubscription( + BinanceUsSubscription &&) noexcept = default; +BinanceUsSubscription &BinanceUsSubscription::operator=( + BinanceUsSubscription &&) noexcept = default; + +BinanceUsSubscription::~BinanceUsSubscription() = default; + +} // namespace bts::md::binance_us +#endif diff --git a/src/lib/md/binance_us/BinanceUsSubscription.h b/src/lib/md/binance_us/BinanceUsSubscription.h new file mode 100644 index 000000000..d205b0c39 --- /dev/null +++ b/src/lib/md/binance_us/BinanceUsSubscription.h @@ -0,0 +1,54 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include +#include + +namespace ccapi { +class EventDispatcher; +} + +namespace bts::api::md { +class BinanceUsMarketDataConfig; +} + +namespace bts::md { +class IMarketDataListener; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::md::binance_us { + +/** + * @brief Subscribes to market data at Binance US + * + * @a https://docs.binance.us/#get-exchange-information + */ +class BinanceUsSubscription { + public: + BinanceUsSubscription( + ccapi::EventDispatcher *dispatcher, + api::md::BinanceUsMarketDataConfig cfg, + persistence::IDGenerator *sec_id_store, + IMarketDataListener *listener); + + BinanceUsSubscription(const BinanceUsSubscription &) noexcept = delete; + BinanceUsSubscription(BinanceUsSubscription &&) noexcept; + BinanceUsSubscription &operator=(const BinanceUsSubscription &) noexcept = + delete; + BinanceUsSubscription &operator=(BinanceUsSubscription &&) noexcept; + + ~BinanceUsSubscription(); + + void add_listener(IMarketDataListener *listener); + void remove_listener(IMarketDataListener *listener); + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md::binance_us +#endif diff --git a/src/lib/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp b/src/lib/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp index 1db516f8c..f471a987b 100644 --- a/src/lib/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp +++ b/src/lib/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp @@ -5,30 +5,25 @@ #include "lib/collections/Contains.h" #include "lib/collections/UnorderedSetFactory.h" #include "lib/core/FMT.h" -#include "lib/core/StringJoiner.h" #include "lib/domain/Security.h" #include "lib/domain/SecurityCriteria.h" #include "lib/domain/SecurityID.h" -#include "lib/domain/SecurityMarketDataStatistics.h" #include "lib/io/ProtoPrinter.h" #include "lib/io/QueryParams.h" #include "lib/logging/logging.h" #include "lib/md/CCApiCorrelationID.h" #include "lib/md/CCApiMarketDepthAdaptor.h" #include "lib/md/CCApiMessageHandler.h" -#include "lib/md/CCApiMessageLogger.h" #include "lib/md/CCApiSession.h" #include "lib/md/CCApiTradeAdaptor.h" #include "lib/md/ICCApiMessageHandler.h" #include "lib/md/IMarketDataListener.h" -#include "lib/md/MarketDataListenerData.h" #include "lib/md/binance_usds_futures/BinanceUsdsFuturesMessageType.h" #include "lib/md/binance_usds_futures/BinanceUsdsFuturesPriceStatisticsAdaptor.h" #include "lib/md/binance_usds_futures/BinanceUsdsFuturesSecurityRequestor.h" #include "lib/persistence/IDGenerator.h" #include -#include #include #include #include @@ -36,8 +31,6 @@ #include #include #include -#include -#include #include #include @@ -60,7 +53,6 @@ using std::vector; #define VENUE_NAME (Venue_Name(Venue::BINANCE_USDS_FUTURES)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_USDS_FUTURES -#define VENUE_HOST CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE #define VENUE_PRICE_STATS_PATH BINANCE_USDS_FUTURES_PRICE_STATS_PATH /** diff --git a/src/lib/md/ccApi/CCMarketDataListener.cpp b/src/lib/md/ccApi/CCMarketDataListener.cpp index 285278737..e4cac628e 100644 --- a/src/lib/md/ccApi/CCMarketDataListener.cpp +++ b/src/lib/md/ccApi/CCMarketDataListener.cpp @@ -56,6 +56,8 @@ static bts::api::Venue to_historic_venue(bts::api::domain::Venue venue) { return api::Venue::VENUE_FTX; case api::domain::Venue::BINANCE: return api::Venue::VENUE_BINANCE; + case api::domain::Venue::BINANCE_US: + return api::Venue::VENUE_BINANCE_US; case api::domain::Venue::BINANCE_USDS_FUTURES: return api::Venue::VENUE_BINANCE_USDS_FUTURES; case api::domain::Venue::SIM: diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 80716a983..f9116ba21 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -174,6 +174,9 @@ struct MDConsumerGroup { case api::VENUE_BINANCE: this->mic = "BINANCE"; break; + case api::VENUE_BINANCE_US: + this->mic = "BINANCE_US"; + break; case api::VENUE_BINANCE_USDS_FUTURES: this->mic = "BINANCE_USDS_FUTURES"; break; diff --git a/src/server/sparkd/cmds/md/common/VenueOption.cpp b/src/server/sparkd/cmds/md/common/VenueOption.cpp index 4c531beb2..f7ac4e145 100644 --- a/src/server/sparkd/cmds/md/common/VenueOption.cpp +++ b/src/server/sparkd/cmds/md/common/VenueOption.cpp @@ -16,6 +16,7 @@ EnumOptionNames VenueOption::names() { NAME_PAIR(api::VENUE_DERIBIT), NAME_PAIR(api::VENUE_FTX), NAME_PAIR(api::VENUE_BINANCE), + NAME_PAIR(api::VENUE_BINANCE_US), NAME_PAIR(api::VENUE_BINANCE_USDS_FUTURES)}; } @@ -37,6 +38,8 @@ std::string VenueOption::name(api::Venue venue) { return "ftx"; case api::VENUE_BINANCE: return "binance"; + case api::VENUE_BINANCE_US: + return "binance_us"; case api::VENUE_BINANCE_USDS_FUTURES: return "binance_usds_futures"; default: