From 2cf6411ca73a7eb44fbfceb7b19ac263e1797122 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 17 Jan 2025 07:20:59 +0000 Subject: [PATCH] fix: Send exchangeInfo req via our HttpClient instead of ccapi Bugs (fixed): * We get a boost beast exception about the read limit being exceeded when requesting securities from Binance prod url (testnet works). --- src/server/sparkd/md/binance/BinanceSpot.cpp | 68 ++++++++++--------- src/server/sparkd/md/binance/BinanceSpot.h | 5 ++ .../sparkd/md/binance/BinanceSubscription.cpp | 56 ++++++++++++--- 3 files changed, 87 insertions(+), 42 deletions(-) diff --git a/src/server/sparkd/md/binance/BinanceSpot.cpp b/src/server/sparkd/md/binance/BinanceSpot.cpp index bd7f0cb22..f12553e2b 100644 --- a/src/server/sparkd/md/binance/BinanceSpot.cpp +++ b/src/server/sparkd/md/binance/BinanceSpot.cpp @@ -40,6 +40,42 @@ static std::string deduce_sym(const BinanceSpot &bspot) { return deduce_sym_from_name(bspot.symbol); } +bool BinanceSpot::parse( + const std::string &msg, vector &securities, Venue venue) { + const auto js = nlohmann::json::parse(msg); + JsonReader jr{&js}; + vector binance_spots; + if (!jr.read("symbols", binance_spots)) { + const auto js_str = js.dump(2); + LOG(md, Error, + "Invalid json. Expected field 'symbols' to contain list of spot " + "secs:\n{}", + js_str); + return false; + } + for (const auto &bspot : binance_spots) { + auto &sec = securities.emplace_back(); + auto &scomm = sec.security_common; + scomm.venue = venue; + + scomm.exchange_sec_id = bspot.symbol; + scomm.desc = bspot.symbol; + scomm.symbol = deduce_sym(bspot); + scomm.asset = scomm.symbol; + scomm.group = scomm.symbol; + + scomm.min_price_increment = bspot.filters.price.tick_size; + scomm.min_quantity = bspot.filters.lot_size.min_qty; + scomm.min_quantity_increment = bspot.filters.lot_size.step_size; + scomm.currency = bspot.quoteAsset.name(); + + sec.security.emplace(); + LOG(md, Diag, "Converted BinanceSpot:\n{}\nto domain::Security:\n{}", + bspot, sec.to_string(true)); + } + return true; +} + void BinanceSpot::parse( const ccapi::Message &msg, vector &securities, Venue venue) { const auto &sec_elements = msg.getElementList(); @@ -50,37 +86,7 @@ void BinanceSpot::parse( continue; } const auto &body = it->second; - const auto js = nlohmann::json::parse(body); - JsonReader jr{&js}; - vector binance_spots; - if (!jr.read("symbols", binance_spots)) { - const auto js_str = js.dump(2); - LOG(md, Error, - "Invalid json. Expected field 'symbols' to contain list of spot " - "secs:\n{}", - js_str); - continue; - } - for (const auto &bspot : binance_spots) { - auto &sec = securities.emplace_back(); - auto &scomm = sec.security_common; - scomm.venue = venue; - - scomm.exchange_sec_id = bspot.symbol; - scomm.desc = bspot.symbol; - scomm.symbol = deduce_sym(bspot); - scomm.asset = scomm.symbol; - scomm.group = scomm.symbol; - - scomm.min_price_increment = bspot.filters.price.tick_size; - scomm.min_quantity = bspot.filters.lot_size.min_qty; - scomm.min_quantity_increment = bspot.filters.lot_size.step_size; - scomm.currency = bspot.quoteAsset.name(); - - sec.security.emplace(); - LOG(md, Diag, "Converted BinanceSpot:\n{}\nto domain::Security:\n{}", - bspot, sec.to_string(true)); - } + parse(body, securities, venue); } } diff --git a/src/server/sparkd/md/binance/BinanceSpot.h b/src/server/sparkd/md/binance/BinanceSpot.h index e8deffbc1..97f781e19 100644 --- a/src/server/sparkd/md/binance/BinanceSpot.h +++ b/src/server/sparkd/md/binance/BinanceSpot.h @@ -51,6 +51,11 @@ class BinanceSpot { const ccapi::Message &msg, std::vector &securities, api::domain::Venue venue = api::domain::Venue::VENUE_BINANCE); + + static bool parse( + const std::string &msg, + std::vector &securities, + api::domain::Venue venue = api::domain::Venue::VENUE_BINANCE); }; void from_json(const nlohmann::json &j, BinanceSpot &spot); diff --git a/src/server/sparkd/md/binance/BinanceSubscription.cpp b/src/server/sparkd/md/binance/BinanceSubscription.cpp index bead58867..8ba963fcb 100644 --- a/src/server/sparkd/md/binance/BinanceSubscription.cpp +++ b/src/server/sparkd/md/binance/BinanceSubscription.cpp @@ -1,12 +1,16 @@ #include "BinanceSubscription.h" #include "lib/api/domain/Venue.pb.h" #include "lib/api/md/BinanceMarketDataConfig.pb.h" +#include "lib/collections/Contains.h" +#include "lib/collections/Remove.h" #include "lib/collections/UnorderedSetFactory.h" #include "lib/core/FMT.h" #include "lib/domain/Security.h" #include "lib/domain/SecurityCriteria.h" #include "lib/io/ProtoPrinter.h" #include "lib/io/QueryParams.h" +#include "lib/io/http/http_client.h" +#include "lib/io/http/http_response.h" #include "lib/logging/logging.h" #include "lib/md/IMarketDataListener.h" #include "lib/persistence/IDGenerator.h" @@ -14,6 +18,7 @@ #include "server/sparkd/md/binance/BinancePriceStatistics.h" #include "server/sparkd/md/binance/BinancePriceStatisticsAdaptor.h" #include "server/sparkd/md/binance/BinanceSecurityRequestor.h" +#include "server/sparkd/md/binance/BinanceSpot.h" #include "server/sparkd/md/ccapi/CCApiCorrelationID.h" #include "server/sparkd/md/ccapi/CCApiMarketDepthAdaptor.h" #include "server/sparkd/md/ccapi/CCApiMessageLogger.h" @@ -49,6 +54,7 @@ using std::string; using std::uint32_t; using std::unique_ptr; using std::unordered_map; +using std::unordered_set; using std::vector; #define VENUE Venue::VENUE_BINANCE @@ -156,12 +162,21 @@ static ccapi::Subscription mk_ticker_sub( ticker_corr_id.to_string()}; } +static vector get_sec_descs(const vector &secs) { + vector descs{secs.size()}; + std::transform(secs.begin(), secs.end(), descs.begin(), [](const auto &s) { + return s.desc(); + }); + return descs; +} + struct BinanceSubscription::PImpl { // General Notes: // All securities are listed in the 'symbols' field of the json response // to the 'exchangeInfo' request. unordered_map> securities; + unordered_set known_sec_names; IDGenerator *sec_id_store; // Don't actually need this, but keeping it around for diagnostic purposes @@ -172,7 +187,6 @@ struct BinanceSubscription::PImpl { // The session automatically starts/stops in its ctr/dtr CCApiSession session; IMarketDataListener *listener; - BinanceSecurityRequestor sec_requestor; // These aliases are useful for quickly switching the handler types (e.g., // a special handler that logs messages for testing/diagnostic purposes) @@ -198,18 +212,38 @@ struct BinanceSubscription::PImpl { IMarketDataListener *listener) : sec_id_store(sec_id_store), cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), - session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener), - sec_requestor( - &session, - this->criteria, - sec_id_store, - collections::UnorderedSetFactory::create( - this->cfg.excluded_securities().begin(), - this->cfg.excluded_securities().end())) { + session(dispatcher, mk_session_cfg(this->cfg), {}), + listener(listener) { LOG(md, Info, "Initializing BinanceSubscription:\n{}", ProtoPrinter::to_string(this->cfg)); - sec_requestor.callback().bind<&PImpl::on_securities>(*this); - sec_requestor.request(); + // Request the list of all spots at Binance. Match the results against our SecurityCriteria, + // and subscribe to md for each matching spot. + std::string exch_info_url = this->cfg.url_rest_base(); + exch_info_url.append("/api/v3/exchangeInfo"); + bts::http::HttpResponse exch_info_res; + bts::http::HttpClient::get(exch_info_url, {}, &exch_info_res); + std::vector secs; + BinanceSpot::parse(exch_info_res.body, secs); + // Exclude secs that don't satisfy the SecurityCriteria + auto is_sec_excluded = [this](const Security &sec) { + return sec.type() != api::domain::SecurityType::SECURITY_TYPE_SPOT || + !criteria.includes_or_undefined(sec) || + CONTAINS(this->cfg.excluded_securities(), sec.desc()) || + known_sec_names.find(sec.desc()) != known_sec_names.end(); + }; + REMOVE_IF(secs, is_sec_excluded); + // Assign security ids + for (auto &sec : secs) { + // Since the ids haven't been set yet, we should expect them to be 0 + bts_assert(sec.id().value() == 0); + SecurityID sec_id{ + sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())}; + sec.security_common.id = sec_id; + } + auto sec_descs = get_sec_descs(secs); + LOG(md, Info, "Received Binance securities:\n{}", + fmt::to_string(fmt::join(sec_descs, "\n"))); + on_securities(secs); } DepthHandler &mk_depth_handler(