Files
spark/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp
T
John Colvin 84c0684b63 refactor: Normalize sec key indexing for new (domain) md sources
This is part of a larger effort to unify the way we index securities in order
to generate unique ids. Older md sources (e.g., MDP3Listener) use
SecurityIDRecorder to generate ids, which uses a different database and
different key-string formatting than the IDGenerator used by the newer md
sources (e.g., GeneralMarketDataListener and various ccapi-based sources).
2024-11-20 22:17:08 +00:00

351 lines
13 KiB
C++

#include "BinanceUsSubscription.h"
#include "lib/api/domain/Venue.pb.h"
#include "lib/api/md/BinanceUsMarketDataConfig.pb.h"
#include "lib/collections/UnorderedSetFactory.h"
#include "lib/core/FMT.h"
#include "lib/domain/Security.h"
#include "lib/domain/SecurityCriteria.h"
#include "lib/io/ProtoPrinter.h"
#include "lib/io/QueryParams.h"
#include "lib/logging/logging.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/persistence/IDGenerator.h"
#include "server/sparkd/md/binance/BinanceSpot.h"
#include "server/sparkd/md/binance_us/BinanceUsMessageType.h"
#include "server/sparkd/md/binance_us/BinanceUsPriceStatistics.h"
#include "server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.h"
#include "server/sparkd/md/binance_us/BinanceUsSecurityRequestor.h"
#include "server/sparkd/md/ccapi/CCApiCorrelationID.h"
#include "server/sparkd/md/ccapi/CCApiMarketDepthAdaptor.h"
#include "server/sparkd/md/ccapi/CCApiMessageLogger.h"
#include "server/sparkd/md/ccapi/CCApiSession.h"
#include "server/sparkd/md/ccapi/CCApiTradeAdaptor.h"
#include <ccapi_cpp/ccapi_event_dispatcher.h>
#include <ccapi_cpp/ccapi_event_handler.h>
#include <ccapi_cpp/ccapi_macro.h>
#include <ccapi_cpp/ccapi_message.h>
#include <ccapi_cpp/ccapi_request.h>
#include <ccapi_cpp/ccapi_session.h>
#include <ccapi_cpp/ccapi_session_configs.h>
#include <ccapi_cpp/ccapi_subscription.h>
#include <range/v3/view.hpp>
#include <range/v3/view/transform.hpp>
#include <string>
#include <unordered_map>
namespace bts::md::binance_us {
using bts::api::domain::SecurityType;
using bts::api::domain::Venue;
using bts::api::domain::Venue_Name;
using bts::api::md::BinanceUsMarketDataConfig;
using bts::domain::Security;
using bts::domain::SecurityCriteria;
using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::pair;
using std::string;
using std::tuple;
using std::uint32_t;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
#define VENUE Venue::VENUE_BINANCE_US
#define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_US))
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_US
/**
* @brief Apply overrides from our config to a ccapi::SessionConfigs object
* that is otherwise populated with defaults
*/
static ccapi::SessionConfigs mk_session_cfg(
const BinanceUsMarketDataConfig &cfg) {
ccapi::SessionConfigs rv;
if (!cfg.url_rest_base().empty()) {
rv.urlRestBase[CCAPI_VENUE_NAME] = cfg.url_rest_base();
}
if (!cfg.url_ws_base().empty()) {
rv.urlWebsocketBase[CCAPI_VENUE_NAME] = cfg.url_ws_base();
}
return rv;
}
static string mk_depth_opts(const BinanceUsMarketDataConfig &cfg) {
// For details on supported opts, read the docs
// https://binance-docs.github.io/apidocs/spot/en/#partial-book-depth-streams
QueryParams opts;
switch (cfg.num_depth_levels()) {
case 5:
case 10:
case 20:
// The exchange only supports these num levels
opts[CCAPI_MARKET_DEPTH_MAX] = std::to_string(cfg.num_depth_levels());
break;
case 0:
// Leave num levels unspecified (let exchange choose default)
break;
default:
LOG(md, Warn, "Invalid num depth levels '{}'. Must be one of [5, 10, 20]",
cfg.num_depth_levels());
break;
}
switch (cfg.depth_throttle_ms()) {
case 100:
case 1000:
// The exchange only supports these throttle speeds
opts[CCAPI_MARKET_DEPTH] = std::to_string(cfg.depth_throttle_ms());
break;
case 0:
// Leave throttle unspecified (let exchange choose default)
break;
default:
LOG(md, Warn,
"Invalid depth throttle '{}ms'. Must be one of [100, 250, 500])",
cfg.depth_throttle_ms());
break;
}
return opts.to_string();
}
static ccapi::Subscription mk_depth_sub(
const string &sec_desc, const string &opts) {
auto corr_id = CCApiCorrelationID::from_parts(
VENUE_NAME, BinanceUsMessageType::DEPTH, sec_desc);
return ccapi::Subscription{
CCAPI_VENUE_NAME, sec_desc, CCAPI_MARKET_DEPTH, opts,
corr_id.to_string()};
}
static CCApiCorrelationID mk_stats_corr_id(const string &desc) {
return CCApiCorrelationID::from_parts(
VENUE_NAME, BinanceUsMessageType(BinanceUsMessageType::STATISTICS),
desc);
}
static ccapi::Request mk_stats_req(
const vector<pair<SecurityType, string>> &sec_descs, uint32_t batch_id) {
const auto corr_id = mk_stats_corr_id(FMT("BATCH_{}", batch_id));
ccapi::Request req{
ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, CCAPI_VENUE_NAME};
req.setCorrelationId(corr_id.to_string());
auto quoted_sec_descs =
sec_descs | ranges::views::transform(
[](const auto &s) { return "\"" + s.second + "\""; });
const string symbols_query_arg = FMT_JOIN(quoted_sec_descs, ",");
req.appendParam(
{{CCAPI_HTTP_PATH, BINANCE_US_PRICE_STATS_PATH},
{CCAPI_HTTP_QUERY_STRING, FMT("symbols=[{}]", symbols_query_arg)},
{CCAPI_HTTP_METHOD, "GET"}});
return req;
}
static string mk_ticker_opts(const BinanceUsMarketDataConfig & /*cfg*/) {
// Placeholder (for consistency's sake)
return "";
}
static ccapi::Subscription mk_ticker_sub(
const string &sec_desc, const string &opts) {
auto ticker_corr_id = CCApiCorrelationID::from_parts(
VENUE_NAME, BinanceUsMessageType::TICKER, sec_desc);
return ccapi::Subscription{
CCAPI_VENUE_NAME, sec_desc, CCAPI_TRADE, opts,
ticker_corr_id.to_string()};
}
struct BinanceUsSubscription::PImpl {
// General Notes:
// All securities are listed in the 'symbols' field of the json response
// to the 'exchangeInfo' request.
unordered_map<SecurityID, unique_ptr<Security>> securities;
IDGenerator *sec_id_store;
// Don't actually need this, but keeping it around for diagnostic purposes
// (i.e., inspection in debugger, temp logs, etc.)
const BinanceUsMarketDataConfig cfg;
const SecurityCriteria criteria;
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
BinanceUsSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
// a special handler that logs messages for testing/diagnostic purposes)
using DepthHandler = CCApiMarketDepthAdaptor;
using TickerHandler = CCApiTradeAdaptor;
using StatsHandler = BinanceUsPriceStatisticsAdaptor;
unordered_map<string, DepthHandler> depth_handlers;
unordered_map<string, TickerHandler> ticker_handlers;
// 'sec_stats_handler_map' maps security (description) to the StatsHandler
// subscribed to that security's data. Since each StatsHandler processes
// data for multiple securities, the actual instances are contained in a
// different map, 'stats_handlers' (unique ids are generated w/ a counter,
// 'stats_handlers_id_counter').
unordered_map<string, StatsHandler *> sec_stats_handler_map;
unordered_map<uint32_t, std::unique_ptr<StatsHandler>> stats_handlers;
uint32_t stats_handlers_id_counter = 0;
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
BinanceUsMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
sec_requestor(
&session,
this->criteria,
sec_id_store,
collections::UnorderedSetFactory<string>::create(
this->cfg.excluded_securities().begin(),
this->cfg.excluded_securities().end())) {
LOG(md, Info, "Initializing BinanceUsSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
sec_requestor.request();
}
DepthHandler &mk_depth_handler(
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert(is_new);
return it->second;
}
TickerHandler &mk_ticker_handler(
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
}
StatsHandler &mk_stats_handler(
const vector<std::pair<SecurityType, string>> &sec_descs,
uint32_t hnd_id) {
vector<tuple<SecurityType, string, SecurityID>> sec_desc_id_tuples;
sec_desc_id_tuples.reserve(sec_descs.size());
for (auto [type, name] : sec_descs) {
SecurityID id{sec_id_store->get_or_add(VENUE, type, name)};
sec_desc_id_tuples.emplace_back(type, std::move(name), id);
}
auto [it, is_new] = stats_handlers.try_emplace(
hnd_id,
std::make_unique<StatsHandler>(sec_desc_id_tuples, listener));
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", hnd_id);
return *it->second;
}
void add_stats_handler(
const vector<pair<SecurityType, string>> &sec_descs,
CCApiSession::BatchRequest &reqs) {
if (sec_descs.empty()) {
return;
}
const auto hnd_id = stats_handlers_id_counter++;
reqs.push_back(
{mk_stats_req(sec_descs, hnd_id),
&mk_stats_handler(sec_descs, hnd_id)});
}
/**
* @brief Subscribes to depth, ticker, and price data for a list of
* securities
*/
template <class SecDescs>
void subscribe_market_data(SecDescs securities) {
const auto depth_opts = mk_depth_opts(cfg);
const auto ticker_opts = mk_ticker_opts(cfg);
// Batch subscribe/request ticker, depth, and stats
CCApiSession::BatchSubscription depth_subs;
CCApiSession::BatchSubscription ticker_subs;
CCApiSession::BatchRequest stats_reqs;
const int N_SECS_PER_STATS_REQ = 16;
vector<pair<SecurityType, string>> curr_stats_sec_descs;
for (const auto &[type, name] : securities) {
depth_subs.push_back(
{mk_depth_sub(name, depth_opts), &mk_depth_handler(type, name)});
ticker_subs.push_back(
{mk_ticker_sub(name, ticker_opts),
&mk_ticker_handler(type, name)});
curr_stats_sec_descs.push_back(std::make_pair(type, name));
if (curr_stats_sec_descs.size() == N_SECS_PER_STATS_REQ) {
add_stats_handler(curr_stats_sec_descs, stats_reqs);
curr_stats_sec_descs.clear();
}
}
if (!curr_stats_sec_descs.empty()) {
add_stats_handler(curr_stats_sec_descs, stats_reqs);
}
session.subscribe(
depth_subs, {ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH});
session.subscribe(
ticker_subs, {ccapi::Message::Type::MARKET_DATA_EVENTS_TRADE});
session.request(
stats_reqs, {ccapi::Message::Type::GENERIC_PUBLIC_REQUEST});
}
/**
* Updates the securities map and subscribes market data for newly arrived
* securities
*/
void on_securities(vector<Security> secs) {
// 'sec_descs' is used to make depth subscriptions
vector<pair<SecurityType, string>> sec_descs;
sec_descs.reserve(secs.size());
// 'sec_ptrs' is passed to IMarketDataListeners
vector<const Security *> sec_ptrs;
sec_ptrs.reserve(secs.size());
for (auto &s : secs) {
auto id = s.id();
auto desc = s.desc();
auto [it, is_new] = securities.try_emplace(
id, std::make_unique<Security>(std::move(s)));
bts_assert_msgf(
is_new, "Received duplicate Binance US security '{}'", desc);
sec_descs.push_back(std::make_pair(s.type(), std::move(desc)));
// add ptr to entry in 'securities' (not the loop var, we moved that)
sec_ptrs.push_back(it->second.get());
}
listener->on_securities(sec_ptrs);
subscribe_market_data(sec_descs);
}
};
BinanceUsSubscription::BinanceUsSubscription(
ccapi::EventDispatcher *dispatcher,
BinanceUsMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
BinanceUsSubscription::BinanceUsSubscription(
BinanceUsSubscription &&) noexcept = default;
BinanceUsSubscription &BinanceUsSubscription::operator=(
BinanceUsSubscription &&) noexcept = default;
BinanceUsSubscription::~BinanceUsSubscription() = default;
} // namespace bts::md::binance_us