fix: Send exchangeInfo req via our HttpClient instead of ccapi

Bugs (fixed):
  * We get a boost beast exception about the read limit being exceeded
    when requesting securities from Binance prod url (testnet works).
This commit is contained in:
John Colvin committed 2025-01-19 14:21:11 +00:00
1 parent e99df129d5
commit 2cf6411ca7
3 files changed
+87 -42

No files matched your search

+37 -31
View File
@@ -40,6 +40,42 @@ static std::string deduce_sym(const BinanceSpot &bspot) {
return deduce_sym_from_name(bspot.symbol);
}
bool BinanceSpot::parse(
const std::string &msg, vector<Security> &securities, Venue venue) {
const auto js = nlohmann::json::parse(msg);
JsonReader jr{&js};
vector<BinanceSpot> binance_spots;
if (!jr.read("symbols", binance_spots)) {
const auto js_str = js.dump(2);
LOG(md, Error,
"Invalid json. Expected field 'symbols' to contain list of spot "
"secs:\n{}",
js_str);
return false;
}
for (const auto &bspot : binance_spots) {
auto &sec = securities.emplace_back();
auto &scomm = sec.security_common;
scomm.venue = venue;
scomm.exchange_sec_id = bspot.symbol;
scomm.desc = bspot.symbol;
scomm.symbol = deduce_sym(bspot);
scomm.asset = scomm.symbol;
scomm.group = scomm.symbol;
scomm.min_price_increment = bspot.filters.price.tick_size;
scomm.min_quantity = bspot.filters.lot_size.min_qty;
scomm.min_quantity_increment = bspot.filters.lot_size.step_size;
scomm.currency = bspot.quoteAsset.name();
sec.security.emplace<SpotSecurity>();
LOG(md, Diag, "Converted BinanceSpot:\n{}\nto domain::Security:\n{}",
bspot, sec.to_string(true));
}
return true;
}
void BinanceSpot::parse(
const ccapi::Message &msg, vector<Security> &securities, Venue venue) {
const auto &sec_elements = msg.getElementList();
@@ -50,37 +86,7 @@ void BinanceSpot::parse(
continue;
}
const auto &body = it->second;
const auto js = nlohmann::json::parse(body);
JsonReader jr{&js};
vector<BinanceSpot> binance_spots;
if (!jr.read("symbols", binance_spots)) {
const auto js_str = js.dump(2);
LOG(md, Error,
"Invalid json. Expected field 'symbols' to contain list of spot "
"secs:\n{}",
js_str);
continue;
}
for (const auto &bspot : binance_spots) {
auto &sec = securities.emplace_back();
auto &scomm = sec.security_common;
scomm.venue = venue;
scomm.exchange_sec_id = bspot.symbol;
scomm.desc = bspot.symbol;
scomm.symbol = deduce_sym(bspot);
scomm.asset = scomm.symbol;
scomm.group = scomm.symbol;
scomm.min_price_increment = bspot.filters.price.tick_size;
scomm.min_quantity = bspot.filters.lot_size.min_qty;
scomm.min_quantity_increment = bspot.filters.lot_size.step_size;
scomm.currency = bspot.quoteAsset.name();
sec.security.emplace<SpotSecurity>();
LOG(md, Diag, "Converted BinanceSpot:\n{}\nto domain::Security:\n{}",
bspot, sec.to_string(true));
}
parse(body, securities, venue);
}
}
@@ -51,6 +51,11 @@ class BinanceSpot {
const ccapi::Message &msg,
std::vector<domain::Security> &securities,
api::domain::Venue venue = api::domain::Venue::VENUE_BINANCE);
static bool parse(
const std::string &msg,
std::vector<domain::Security> &securities,
api::domain::Venue venue = api::domain::Venue::VENUE_BINANCE);
};
void from_json(const nlohmann::json &j, BinanceSpot &spot);
@@ -1,12 +1,16 @@
#include "BinanceSubscription.h"
#include "lib/api/domain/Venue.pb.h"
#include "lib/api/md/BinanceMarketDataConfig.pb.h"
#include "lib/collections/Contains.h"
#include "lib/collections/Remove.h"
#include "lib/collections/UnorderedSetFactory.h"
#include "lib/core/FMT.h"
#include "lib/domain/Security.h"
#include "lib/domain/SecurityCriteria.h"
#include "lib/io/ProtoPrinter.h"
#include "lib/io/QueryParams.h"
#include "lib/io/http/http_client.h"
#include "lib/io/http/http_response.h"
#include "lib/logging/logging.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/persistence/IDGenerator.h"
@@ -14,6 +18,7 @@
#include "server/sparkd/md/binance/BinancePriceStatistics.h"
#include "server/sparkd/md/binance/BinancePriceStatisticsAdaptor.h"
#include "server/sparkd/md/binance/BinanceSecurityRequestor.h"
#include "server/sparkd/md/binance/BinanceSpot.h"
#include "server/sparkd/md/ccapi/CCApiCorrelationID.h"
#include "server/sparkd/md/ccapi/CCApiMarketDepthAdaptor.h"
#include "server/sparkd/md/ccapi/CCApiMessageLogger.h"
@@ -49,6 +54,7 @@ using std::string;
using std::uint32_t;
using std::unique_ptr;
using std::unordered_map;
using std::unordered_set;
using std::vector;
#define VENUE Venue::VENUE_BINANCE
@@ -156,12 +162,21 @@ static ccapi::Subscription mk_ticker_sub(
ticker_corr_id.to_string()};
}
static vector<string> get_sec_descs(const vector<Security> &secs) {
vector<string> descs{secs.size()};
std::transform(secs.begin(), secs.end(), descs.begin(), [](const auto &s) {
return s.desc();
});
return descs;
}
struct BinanceSubscription::PImpl {
// General Notes:
// All securities are listed in the 'symbols' field of the json response
// to the 'exchangeInfo' request.
unordered_map<SecurityID, unique_ptr<Security>> securities;
unordered_set<string> known_sec_names;
IDGenerator *sec_id_store;
// Don't actually need this, but keeping it around for diagnostic purposes
@@ -172,7 +187,6 @@ struct BinanceSubscription::PImpl {
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
BinanceSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
// a special handler that logs messages for testing/diagnostic purposes)
@@ -198,18 +212,38 @@ struct BinanceSubscription::PImpl {
IMarketDataListener *listener)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
sec_requestor(
&session,
this->criteria,
sec_id_store,
collections::UnorderedSetFactory<string>::create(
this->cfg.excluded_securities().begin(),
this->cfg.excluded_securities().end())) {
session(dispatcher, mk_session_cfg(this->cfg), {}),
listener(listener) {
LOG(md, Info, "Initializing BinanceSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
sec_requestor.request();
// Request the list of all spots at Binance. Match the results against our SecurityCriteria,
// and subscribe to md for each matching spot.
std::string exch_info_url = this->cfg.url_rest_base();
exch_info_url.append("/api/v3/exchangeInfo");
bts::http::HttpResponse exch_info_res;
bts::http::HttpClient::get(exch_info_url, {}, &exch_info_res);
std::vector<domain::Security> secs;
BinanceSpot::parse(exch_info_res.body, secs);
// Exclude secs that don't satisfy the SecurityCriteria
auto is_sec_excluded = [this](const Security &sec) {
return sec.type() != api::domain::SecurityType::SECURITY_TYPE_SPOT ||
!criteria.includes_or_undefined(sec) ||
CONTAINS(this->cfg.excluded_securities(), sec.desc()) ||
known_sec_names.find(sec.desc()) != known_sec_names.end();
};
REMOVE_IF(secs, is_sec_excluded);
// Assign security ids
for (auto &sec : secs) {
// Since the ids haven't been set yet, we should expect them to be 0
bts_assert(sec.id().value() == 0);
SecurityID sec_id{
sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())};
sec.security_common.id = sec_id;
}
auto sec_descs = get_sec_descs(secs);
LOG(md, Info, "Received Binance securities:\n{}",
fmt::to_string(fmt::join(sec_descs, "\n")));
on_securities(secs);
}
DepthHandler &mk_depth_handler(