refactor: Simplify md gateway/binance subscription

* Temporarily get rid of the register_listener fn, and just use
  subscribe/unsubscribe to immediately connect/disconnect listeners to
  exchange feeds. The 'register, then subscribe per sec-id' paradigm is
  tricky to implement without access to security ids.
This commit is contained in:
John Colvin committed 2022-06-07 23:07:33 -04:00
1 parent 23927edf46
commit 91a79a5d96
12 files changed
+222 -468

No files matched your search

+60 -168
View File
@@ -10,10 +10,10 @@
#include "lib/io/URL.h"
#include "lib/io/websocket/WebsocketClient.h"
#include "lib/logging/logging.h"
#include "lib/md/CompositeMarketDataListener.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/MarketDataListenerData.h"
#include "lib/md/MarketDataListenerID.h"
#include "lib/md/binance/BinanceDepthStreamSite.h"
#include "lib/md/binance/BinanceDepthStreamSiteKey.h"
#include "lib/md/binance/BinanceDiffDepthSubscription.h"
#include "lib/md/binance/BinanceDiffDepthUpdate.h"
@@ -60,106 +60,49 @@ struct MarketDataGateway::PImpl {
MarketDataRegistrationFactory registration_factory;
std::unordered_map<
binance::BinanceDepthStreamSiteKey,
binance::BinanceDepthStreamSite>
binance_sites;
collections::UnorderedSetMap<domain::SecurityID, MarketDataListenerID>
sec_listeners;
binance::BinanceDiffDepthSubscription>
binance_subs;
std::unordered_map<domain::SecurityID, SecData> sec_data_map;
std::unordered_map<MarketDataListenerID, MarketDataListenerData>
registered_listeners;
void on_binance_depth_changed(
const std::string &symbol,
bts::md::binance::BinanceDiffDepthUpdate upd) {
LOG(md, bts::Diag, "depth changed. sym: {}");
auto sid = domain::SecurityResolver::symbol_to_id(
domain::Venue::Binance, symbol);
auto &sec_data = sec_data_map[sid];
upd.apply_to_book(sec_data.book);
for (const auto lid : sec_data.listeners) {
auto &listener = registered_listeners[lid];
listener.listener->on_book(sec_data.book);
}
}
binance::BinanceDepthStreamSite &get_binance_site(
const binance::BinanceDepthStreamSiteKey &k) {
return binance_sites.try_emplace(k, k).first->second;
}
std::pair<binance::BinanceDepthStreamSite *, bool> try_emplace_binance_site(
const binance::BinanceDepthStreamSiteKey &k) {
auto p = binance_sites.try_emplace(k, k);
return {&p.first->second, p.second};
}
void subscribe(
MarketDataListenerID id,
const api::md::binance::BinanceDepthStreamParams &depth_stream) {
binance::BinanceDepthStreamSiteKey site_key(depth_stream);
auto site_it = binance_sites.find(site_key);
bts_assert(site_it != binance_sites.end());
auto &site = site_it->second;
auto &ld = registered_listeners[id];
for (const auto &sym : depth_stream.symbols()) {
auto sid = domain::SecurityResolver::symbol_to_id(
domain::Venue::Binance, sym);
bts_assert_msgf(
ld.securities.insert(sid).second,
"Cannot subscribe. Listener already subscribed to sec. sid: "
"{}; sym: {}",
sym, sid);
auto res = sec_listeners.add(sid, id);
if (res.added_key) {
site.add_symbol(sym);
void subscribe_binance(
const api::md::binance::BinanceMarketDataSubscriptionParams &sub,
IMarketDataListener *listener) {
std::vector<binance::BinanceDiffDepthSubscription *> new_subs;
for (const auto &depth_stream : sub.depth_streams()) {
md::binance::BinanceDepthStreamSiteKey site_key(depth_stream);
auto [sub_it, is_new_sub] = binance_subs.try_emplace(
site_key, md::binance::BinanceDepthStreamParams{depth_stream});
if (is_new_sub) {
new_subs.push_back(&sub_it->second);
continue;
}
auto &sub = sub_it->second;
// add symbols to piggy back on existing subscription rather than
// create a new one
for (const auto &symbol : depth_stream.symbols()) {
sub.add_listener(symbol, listener);
}
}
for (auto *sub : new_subs) {
sub->connect();
}
}
void unsubscribe(
MarketDataListenerID id,
const api::md::binance::BinanceDepthStreamParams &depth_stream) {
binance::BinanceDepthStreamSiteKey site_key(depth_stream);
auto site_it = binance_sites.find(site_key);
bts_assert(site_it != binance_sites.end());
auto &site = site_it->second;
bts_assert(id.value() != 0);
bts_assert(registered_listeners.find(id) != registered_listeners.end());
auto &ld = registered_listeners[id];
LOG(md, bts::Diag, "Unsubscribing {} symbols for listener {} on site {}",
depth_stream.symbols().size(), id, site_key);
for (const auto &sym : depth_stream.symbols()) {
auto sid = domain::SecurityResolver::symbol_to_id(
domain::Venue::Binance, sym);
bts_assert_msgf(
ld.securities.erase(sid) == 1,
"Cannot unsubscribe. Listener not subscribed to sec. sid: {}; "
"sym: {}",
sid, sym);
auto res = sec_listeners.remove(sid, id);
if (res.removed_key) {
site.remove_symbol(sym);
void unsubscribe_binance(
const api::md::binance::BinanceMarketDataSubscriptionParams &sub,
IMarketDataListener *listener) {
for (const auto &depth_stream : sub.depth_streams()) {
md::binance::BinanceDepthStreamSiteKey site_key(depth_stream);
auto sub_it = binance_subs.find(site_key);
bts_assert(sub_it != binance_subs.end());
auto &sub = sub_it->second;
for (const auto &symbol : depth_stream.symbols()) {
bts_assert(sub.remove_listener(symbol, listener));
}
}
}
void subscribe(
MarketDataListenerID id,
const api::md::binance::BinanceMarketDataSubscriptionParams
&sub_params) {
for (const auto &depth_stream : sub_params.depth_streams()) {
subscribe(id, depth_stream);
}
}
void unsubscribe(
MarketDataListenerID id,
const api::md::binance::BinanceMarketDataSubscriptionParams
&sub_params) {
for (const auto &depth_stream : sub_params.depth_streams()) {
unsubscribe(id, depth_stream);
}
}
};
MarketDataGateway::MarketDataGateway()
@@ -167,51 +110,29 @@ MarketDataGateway::MarketDataGateway()
MarketDataGateway::~MarketDataGateway() = default;
MarketDataRegistration MarketDataGateway::register_listener(
MarketDataRegistration MarketDataGateway::subscribe(
const MarketDataSubscription &sub, IMarketDataListener *listener) noexcept {
auto reg = me_->registration_factory.create();
bts_assert(reg.listener_id.has_value());
bts_assert(
me_->registered_listeners.find(*reg.listener_id) ==
me_->registered_listeners.end());
LOG(md, bts::Info, "MarketDataGateway subscribing to data.\n{}\n",
core::ProtoPrinter<api::md::MarketDataSubscriptionParams>(
&sub.params()));
auto init_listener = [&]() {
auto *conn = &me_->registered_listeners[reg.listener_id.value()];
conn->id = reg.listener_id.value();
conn->listener = listener;
};
auto &listener_data = me_->registered_listeners[*reg.listener_id];
listener_data.id = *reg.listener_id;
listener_data.listener = listener;
listener_data.subscription = sub;
switch (sub.venue()) {
case api::domain::Venue::BINANCE: {
case api::domain::Venue::BINANCE:
bts_assert(
sub.params().VenueParams_case() ==
api::md::MarketDataSubscriptionParams::kBinance);
init_listener();
std::vector<binance::BinanceDepthStreamSite *> new_sites;
LOG(md, bts::Info, "MarketDataGateway registering listener.\n{}\n",
core::ProtoPrinter<api::md::MarketDataSubscriptionParams>(
&sub.params()));
const auto &depth_streams = sub.params().binance().depth_streams();
for (const auto &depth_stream : depth_streams) {
binance::BinanceDepthStreamSiteKey site_key(depth_stream);
auto [site, is_new] = me_->try_emplace_binance_site(site_key);
bts_assert(site);
site->subscribe(reg.listener_id.value(), depth_stream.symbols());
if (is_new) {
new_sites.push_back(site);
}
}
// Although the websocket connection could be made lazily (i.e., in the
// `subscribe` method), establishing the connection ahead of time keeps
// that step out of the hot(ter) path
for (auto *site : new_sites) {
site->on_depth_changed().bind<&PImpl::on_binance_depth_changed>(*me_);
site->connect();
}
me_->subscribe_binance(sub.params().binance(), listener);
break;
}
default:
return MarketDataRegistration{
MarketDataListenerID{0},
@@ -221,54 +142,25 @@ MarketDataRegistration MarketDataGateway::register_listener(
return reg;
}
void MarketDataGateway::subscribe(
MarketDataListenerID id,
const std::vector<domain::SecurityID> &securities) {
auto venue_secs = domain::SecurityResolver::map_venue_to_id(securities);
for (const auto &[venue, secs] : venue_secs) {
switch (venue) {
case domain::Venue::Binance: {
auto syms = ranges::views::transform(secs, [](auto sid) {
return domain::SecurityResolver::id_to_symbol(sid);
});
auto sub_params = create_binance_subscription_params(syms);
me_->subscribe(id, sub_params);
break;
}
default:
break;
}
}
}
void MarketDataGateway::unsubscribe(
MarketDataListenerID id,
const std::vector<domain::SecurityID> &securities) {
LOG(md, bts::Info, "Unsubscribing {} secs for listener {}",
securities.size(), id);
auto venue_secs = domain::SecurityResolver::map_venue_to_id(securities);
for (const auto &[venue, secs] : venue_secs) {
switch (venue) {
case domain::Venue::Binance: {
auto syms = ranges::views::transform(secs, [](auto sid) {
return domain::SecurityResolver::id_to_symbol(sid);
});
auto sub = create_binance_subscription_params(syms);
me_->unsubscribe(id, sub);
}
default:
break;
}
}
}
void MarketDataGateway::unregister_listener(MarketDataListenerID id) {
LOG(md, bts::Info, "Unregistering listener {}", id);
void MarketDataGateway::unsubscribe(MarketDataListenerID id) {
LOG(md, bts::Info, "Unsubscribing listener {}", id);
bts_assert(CONTAINS_KEY(me_->registered_listeners, id));
auto &ld = me_->registered_listeners[id];
std::vector<domain::SecurityID> sec_vec(
ld.securities.begin(), ld.securities.end());
unsubscribe(id, sec_vec);
auto *listener = ld.listener;
bts_assert(listener);
auto &sub = ld.subscription;
switch (sub.venue()) {
case api::domain::Venue::BINANCE:
bts_assert(
sub.params().VenueParams_case() ==
api::md::MarketDataSubscriptionParams::kBinance);
me_->unsubscribe_binance(sub.params().binance(), listener);
break;
default:
break;
}
me_->registered_listeners.erase(id);
}
+2 -15
View File
@@ -22,25 +22,12 @@ class MarketDataGateway {
/**
* @brief Connects a market data listener to a market data source
*
* @details The first call to the Gateway is to register the listener. The
* listener is assigned an ID if the registration is successful. Subsequent
* subscription of securities and unregister_listener is is via this id.
*
* @return MarketDataRegistration
*/
MarketDataRegistration register_listener(
MarketDataRegistration subscribe(
const MarketDataSubscription &, IMarketDataListener *) noexcept;
void subscribe(
MarketDataListenerID id,
const std::vector<domain::SecurityID> &securities);
void unsubscribe(
MarketDataListenerID id,
const std::vector<domain::SecurityID> &securities);
void unregister_listener(MarketDataListenerID id);
void unsubscribe(MarketDataListenerID id);
private:
struct PImpl;
+2 -1
View File
@@ -2,6 +2,7 @@
#include "lib/domain/SecurityID.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/MarketDataListenerID.h"
#include "lib/md/MarketDataSubscription.h"
#include <unordered_set>
namespace bts::md {
@@ -9,7 +10,7 @@ namespace bts::md {
struct MarketDataListenerData {
MarketDataListenerID id;
IMarketDataListener *listener;
std::unordered_set<domain::SecurityID> securities;
MarketDataSubscription subscription;
};
} // namespace bts::md
@@ -1,41 +0,0 @@
#pragma once
#include "system/Delegate.h"
#include <memory>
namespace bts::md::binance {
class BinanceDepthSnapshot;
class BinanceDiffDepthUpdate;
class BinanceDiffDepthListener;
/**
* @brief Bootstraps Binance's incremental 'Diff. Depth' stream with an
* up-to-date depth snapshot, seeding observers with the state to which the
* next incremental update applies
*/
class BinanceDepthBootstrapper {
public:
BinanceDepthBootstrapper();
BinanceDepthBootstrapper(const BinanceDepthBootstrapper &) = delete;
BinanceDepthBootstrapper(BinanceDepthBootstrapper &&) noexcept = default;
BinanceDepthBootstrapper &operator=(const BinanceDepthBootstrapper &) =
delete;
BinanceDepthBootstrapper &operator=(BinanceDepthBootstrapper &&) noexcept =
default;
~BinanceDepthBootstrapper();
using OnDepthChanged =
spartan::Delegate<void(const BinanceDiffDepthUpdate &)>;
void on_snapshot(const BinanceDepthSnapshot &snapshot);
void on_diff_update(const BinanceDiffDepthUpdate &diff_update);
OnDepthChanged &on_depth_changed();
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md::binance
@@ -1,9 +1,10 @@
#include "BinanceDepthBootstrapper.h"
#include "BinanceDepthListenerAdaptor.h"
#include "BinanceDepthSnapshot.h"
#include "BinanceDiffDepthUpdate.h"
#include "lib/core/EnumBitwiseOperators.h"
#include "lib/io/websocket/WebsocketClient.h"
#include "lib/logging/logging.h"
#include "lib/md/CompositeMarketDataListener.h"
#include "lib/md/IMarketDataListener.h"
#include <algorithm>
@@ -18,18 +19,21 @@ enum class State {
BITWISE_OPS(State, std::int32_t)
struct BinanceDepthBootstrapper::PImpl {
struct BinanceDepthListenerAdaptor::PImpl {
BinanceDepthSnapshot last_snapshot;
BinanceDiffDepthUpdate buffered_update;
State state = State::WaitingForSnapshot | State::WaitingForUpdate;
OnDepthChanged on_depth_changed;
CompositeMarketDataListener listener;
std::string symbol;
domain::Book book;
void on_diff_update(const BinanceDiffDepthUpdate &diff_update) {
LOG(md, Info, "Received Binance {} depth update", symbol);
bts_assert(diff_update.last_update_id >= diff_update.first_update_id);
if (state == State::PublishingUpdates) {
bts_assert(
diff_update.last_update_id > buffered_update.last_update_id);
on_depth_changed(diff_update);
publish_update(diff_update);
return;
}
@@ -45,7 +49,7 @@ struct BinanceDepthBootstrapper::PImpl {
// with the new buffer, try to bootstrap the outgoing updates.
if (buffered_update.includes_event(last_snapshot.last_update_id)) {
start(last_snapshot.apply_diff(buffered_update));
start_publishing_updates(last_snapshot.apply_diff(buffered_update));
}
}
@@ -53,42 +57,59 @@ struct BinanceDepthBootstrapper::PImpl {
if (state == State::PublishingUpdates) {
return;
}
if (buffered_update.includes_event(snapshot.last_update_id)) {
start(snapshot.apply_diff(buffered_update));
start_publishing_updates(snapshot.apply_diff(buffered_update));
return;
}
// publish a 'synthetic incremental update' as the delta between this
// snapshot and the last one.
auto synth_update = snapshot - last_snapshot;
// Update last_snapshot before invoking the depth-changed callback to
// avoid exposing stale state in re-entrant calls.
last_snapshot = snapshot;
on_depth_changed(synth_update);
publish_update(snapshot - last_snapshot);
}
void start(const BinanceDepthSnapshot &snapshot) {
void start_publishing_updates(const BinanceDepthSnapshot &snapshot) {
bts_assert(state != State::PublishingUpdates);
LOG(md, Info, "Begin publishing Binance depth updates");
state = State::PublishingUpdates;
last_snapshot = snapshot;
on_depth_changed(snapshot.as_initial_update());
publish_update(snapshot.as_initial_update());
}
/**
* Sends update out to the listener (should only be called after the
* incremental sequence has been bootstrapped)
*/
void publish_update(const BinanceDiffDepthUpdate &diff_update) {
diff_update.apply_to_book(book);
listener.on_book(book);
}
};
BinanceDepthBootstrapper::BinanceDepthBootstrapper()
BinanceDepthListenerAdaptor::BinanceDepthListenerAdaptor()
: me_(std::make_unique<PImpl>()) {}
BinanceDepthBootstrapper::~BinanceDepthBootstrapper() = default;
BinanceDepthListenerAdaptor::~BinanceDepthListenerAdaptor() = default;
BinanceDepthBootstrapper::OnDepthChanged &
BinanceDepthBootstrapper::on_depth_changed() {
return me_->on_depth_changed;
void BinanceDepthListenerAdaptor::add_listener(IMarketDataListener *listener) {
me_->listener.add(listener);
}
void BinanceDepthBootstrapper::on_snapshot(
void BinanceDepthListenerAdaptor::remove_listener(
IMarketDataListener *listener) {
me_->listener.remove(listener);
}
std::size_t BinanceDepthListenerAdaptor::num_listeners() const {
return me_->listener.listeners.size();
}
void BinanceDepthListenerAdaptor::on_snapshot(
const BinanceDepthSnapshot &snapshot) {
me_->on_snapshot(snapshot);
}
void BinanceDepthBootstrapper::on_diff_update(
void BinanceDepthListenerAdaptor::on_diff_update(
const BinanceDiffDepthUpdate &diff_update) {
me_->on_diff_update(diff_update);
}
@@ -0,0 +1,39 @@
#pragma once
#include "lib/domain/Book.h"
#include "lib/domain/SecurityResolver.h"
#include "lib/domain/Venue.h"
#include "lib/logging/logging.h"
#include "lib/md/CompositeMarketDataListener.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/binance/BinanceDiffDepthUpdate.h"
#include <memory>
#include <unordered_map>
namespace bts::md::binance {
class BinanceDepthListenerAdaptor {
public:
BinanceDepthListenerAdaptor();
BinanceDepthListenerAdaptor(const BinanceDepthListenerAdaptor &) = delete;
BinanceDepthListenerAdaptor(BinanceDepthListenerAdaptor &&) noexcept =
default;
BinanceDepthListenerAdaptor &operator=(const BinanceDepthListenerAdaptor &) =
delete;
BinanceDepthListenerAdaptor &operator=(
BinanceDepthListenerAdaptor &&) noexcept = default;
~BinanceDepthListenerAdaptor();
void on_snapshot(const BinanceDepthSnapshot &snapshot);
void on_diff_update(const BinanceDiffDepthUpdate &diff_update);
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
std::size_t num_listeners() const;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md::binance
@@ -1,45 +0,0 @@
#include "BinanceDepthStreamSite.h"
#include "lib/md/binance/BinanceDepthStreamSiteKey.h"
#include "lib/md/binance/BinanceDiffDepthSubscription.h"
#include "lib/md/binance/BinanceDiffDepthUpdate.h"
namespace bts::md::binance {
BinanceDepthStreamSite::BinanceDepthStreamSite(
BinanceDepthStreamSiteKey site_key)
: site_key_(std::move(site_key)) {}
void BinanceDepthStreamSite::connect() {
auto &b = binance_.emplace(site_key_.params());
// (re)bind the callback
b.on_depth_changed().bind<&BinanceDepthStreamSite::fwd_on_depth_changed_>(
*this);
b.connect();
}
void BinanceDepthStreamSite::add_symbol(const std::string &symbol) {
binance_->add_symbol(symbol);
}
void BinanceDepthStreamSite::remove_symbol(const std::string &symbol) {
binance_->remove_symbol(symbol);
}
void BinanceDepthStreamSite::ensure_connect_() {
if (!binance_) {
connect();
}
}
BinanceDiffDepthSubscription::OnDepthChanged &
BinanceDepthStreamSite::on_depth_changed() {
return on_depth_changed_proxy_;
}
void BinanceDepthStreamSite::fwd_on_depth_changed_(
const std::string &symbol,
bts::md::binance::BinanceDiffDepthUpdate update) {
on_depth_changed_proxy_(symbol, update);
}
} // namespace bts::md::binance
@@ -1,60 +0,0 @@
#pragma once
#include "BinanceDepthStreamSiteKey.h"
#include "lib/md/binance/BinanceDiffDepthSubscription.h"
#include <optional>
namespace bts::md::binance {
class BinanceDiffDepthUpdate;
/**
* @brief Facilitates the flow of Binance market data updates (from a particular
* site) to subscribed listeners (of type IMarketDataListener)
*
* @details For a given symbol, one internal subscription serves all listeners
* subscribed to that symbol
*/
class BinanceDepthStreamSite {
public:
BinanceDepthStreamSite() = default;
explicit BinanceDepthStreamSite(BinanceDepthStreamSiteKey site_key);
/**
* @brief Connects to the market data site, and begins the flow of updates to
* subscribed listeners
*
* @note This only needs to be called once; subsequent calls reset the
* connection
*/
void connect();
/**
* @brief Adds a symbol to the depth subscription
*
* @note Updates won't flow until BinanceDepthStreamSite::connect() has been
* called (if it has already been called, they will begin flowing
* immediately)
*/
void add_symbol(const std::string &symbol);
/**
* @brief Removes a symbol from the depth subscription
*/
void remove_symbol(const std::string &symbol);
BinanceDiffDepthSubscription::OnDepthChanged &on_depth_changed();
private:
BinanceDepthStreamSiteKey site_key_;
std::optional<binance::BinanceDiffDepthSubscription> binance_ = std::nullopt;
// Use a proxy so the internal connection, `binance_`, can be changed
// (disposed and recreated) without unbinding the existing callback.
BinanceDiffDepthSubscription::OnDepthChanged on_depth_changed_proxy_;
void ensure_connect_();
void fwd_on_depth_changed_(
const std::string &symbol,
bts::md::binance::BinanceDiffDepthUpdate update);
};
} // namespace bts::md::binance
@@ -1,4 +1,5 @@
#include "BinanceDepthStreamSiteKey.h"
#include "lib/api/md/binance/BinanceDepthStreamParams.pb.h"
#include "lib/core/HashPhaser.h"
namespace bts::md::binance {
@@ -1,14 +1,18 @@
#pragma once
#include "BinanceDepthStreamParams.h"
#include "lib/api/md/binance/BinanceMarketDataSubscriptionParams.pb.h"
#include "lib/domain/SecurityID.h"
#include "lib/io/URL.h"
#include <utility>
namespace bts::api::md::binance {
class BinanceDepthStreamParams;
}
namespace bts::md::binance {
/**
* @brief Represents a Binance market data site
* @brief Identifies a Binance market data site as a pair of stream/snapshot
* urls, and some primitive subscription params
*/
class BinanceDepthStreamSiteKey {
public:
@@ -1,5 +1,5 @@
#include "BinanceDiffDepthSubscription.h"
#include "BinanceDepthBootstrapper.h"
#include "BinanceDepthListenerAdaptor.h"
#include "BinanceDepthSnapshot.h"
#include "BinanceDepthStreamParams.h"
#include "BinanceDiffDepthUpdate.h"
@@ -10,7 +10,9 @@
#include "lib/io/websocket/WebsocketClient.h"
#include "lib/io/websocket/WebsocketClientConfiguration.h"
#include "lib/logging/logging.h"
#include "lib/md/CompositeMarketDataListener.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/binance/BinanceDepthListenerAdaptor.h"
#include "lib/md/binance/BinanceDepthStreamParams.h"
#include "server/sparkd/cmds/spartan/system/Delegate.h"
#include <fmt/format.h>
@@ -25,35 +27,12 @@ static std::string get_stream_name(const std::string &symbol) {
return fmt::format("{}@depth", symbol);
}
struct DiffDepthReq {
std::string symbol;
BinanceDepthBootstrapper bootstrapper;
DiffDepthReq() {
bootstrapper.on_depth_changed().bind<&DiffDepthReq::on_depth_changed>(
*this);
}
DiffDepthReq(const DiffDepthReq &) = delete;
DiffDepthReq(DiffDepthReq &&) noexcept = default;
DiffDepthReq &operator=(const DiffDepthReq &) = delete;
DiffDepthReq &operator=(DiffDepthReq &&) noexcept = default;
~DiffDepthReq() = default;
BinanceDiffDepthSubscription::OnDepthChanged *mux_callback;
void on_depth_changed(const BinanceDiffDepthUpdate &upd) {
if (mux_callback) {
(*mux_callback)(symbol, upd);
}
}
};
struct BinanceDiffDepthSubscription::PImpl {
BinanceDepthStreamParams params;
bts::io::http::HTTPClient http_client;
std::optional<io::websocket::WebsocketClient> socket;
std::unordered_map<std::string, DiffDepthReq> depth_reqs;
std::unordered_map<std::string, BinanceDepthListenerAdaptor> listeners;
// Listens directly to the socket, and demuxes book updates by symbol.
// (invokes a delegate with args for the symbol and book-update itself).
BinanceWebsocketStreamDemuxer demuxer;
@@ -71,59 +50,7 @@ struct BinanceDiffDepthSubscription::PImpl {
// }
std::uint64_t send_id_counter;
OnDepthChanged on_depth_changed;
bool is_connected = false;
DiffDepthReq &subscribe_symbol(const std::string &symbol) {
auto [req_it, is_new] = depth_reqs.try_emplace(symbol);
auto &req = req_it->second;
if (!is_new) {
return req;
}
auto &cb =
demuxer.add_handler<BinanceDiffDepthUpdate>(get_stream_name(symbol));
cb.bind<&BinanceDepthBootstrapper::on_diff_update>(req.bootstrapper);
req.symbol = symbol;
req.mux_callback = &on_depth_changed;
if (!socket.has_value()) {
return req;
}
// live subscribe
auto send_id = send_id_counter++;
auto req_body = BinanceDepthStreamParams::live_subscribe_request_body(
send_id, {symbol});
LOG(md, Info, "Subscribing {} depth", symbol);
socket.value().send_message(io::websocket::WebsocketMessage{
send_id, req_body.c_str(), req_body.size()});
return req;
}
void unsubscribe_symbol(const std::string &symbol) {
if (!socket.has_value()) {
return;
}
auto it = depth_reqs.find(symbol);
if (it == depth_reqs.end()) {
return;
}
depth_reqs.erase(it);
// live unsubscribe
auto stream_name = get_stream_name(symbol);
nlohmann::json j;
auto send_id = send_id_counter++;
j["method"] = "UNSUBSCRIBE";
j["params"] = std::vector<std::string>{stream_name};
j["id"] = send_id;
std::stringstream payload_ss;
payload_ss >> j;
auto payload = payload_ss.str();
LOG(md, Info, "Unsubscribing {} depth", symbol);
socket.value().send_message(io::websocket::WebsocketMessage{
send_id, payload.c_str(), payload.size()});
}
bool is_connected() const { return socket.has_value(); }
};
BinanceDiffDepthSubscription::BinanceDiffDepthSubscription()
@@ -142,22 +69,18 @@ BinanceDiffDepthSubscription &BinanceDiffDepthSubscription::operator=(
BinanceDiffDepthSubscription &&) noexcept = default;
BinanceDiffDepthSubscription::~BinanceDiffDepthSubscription() {
if (me_->socket) {
if (me_->is_connected()) {
LOG(md, bts::Info, "Disconnecting socket. url: {}",
me_->params.stream_url());
// Don't need to do anything to unsubscribe (the socket disconnects itself
// in its destructor)
}
}
BinanceDiffDepthSubscription::OnDepthChanged &
BinanceDiffDepthSubscription::on_depth_changed() {
return me_->on_depth_changed;
}
void BinanceDiffDepthSubscription::connect() {
if (me_->is_connected) {
if (me_->is_connected()) {
return;
}
me_->is_connected = true;
// This implementation is a little tricky. Multiple depth update streams (1
// per symbol) come in on a single websocket, and each stream must be
// bootstrapped independently. Market data flow is as follows
@@ -176,18 +99,13 @@ void BinanceDiffDepthSubscription::connect() {
// stream from its bootstrapper, and forwards them to this class's
// outgoing delegate along with its symbol.
// Initialize the per-symbol DepthReq's
for (const auto &sym : me_->params.symbols()) {
me_->subscribe_symbol(sym);
}
// Now that the DepthReq's are bound to the demuxer, connect the demuxer to
// the websocket.
// First, connect the demuxer to the websocket (need an incremental update
// at or before the snapshot to bootstrap the sequence)
auto conf = io::websocket::WebsocketClientConfiguration::from_url(
me_->params.stream_url());
me_->socket.emplace(conf, &me_->demuxer).start();
// Finally, give the per-symbol bootstrappers snapshots of their respective
// Next, give the per-symbol bootstrappers snapshots of their respective
// symbol's depth. This is deliberately done after the bootstrappers are
// connected to the websocket, so the snapshots are less likely to be stale.
//
@@ -216,16 +134,65 @@ void BinanceDiffDepthSubscription::connect() {
nlohmann::json j = nlohmann::json::parse(res.body());
auto snap = j.get<BinanceDepthSnapshot>();
me_->depth_reqs[sym].bootstrapper.on_snapshot(snap);
me_->listeners[sym].on_snapshot(snap);
}
}
void BinanceDiffDepthSubscription::add_symbol(const std::string &symbol) {
me_->subscribe_symbol(symbol);
void BinanceDiffDepthSubscription::add_listener(
const std::string &symbol, IMarketDataListener *listener) {
auto [it, is_new_sym] = me_->listeners.try_emplace(symbol);
it->second.add_listener(listener);
if (!is_new_sym) {
return;
}
auto &req = it->second;
auto &cb = me_->demuxer.add_handler<BinanceDiffDepthUpdate>(
get_stream_name(symbol));
cb.bind<&BinanceDepthListenerAdaptor::on_diff_update>(req);
if (!me_->is_connected()) {
return;
}
// live subscribe
auto send_id = me_->send_id_counter++;
auto req_body =
BinanceDepthStreamParams::live_subscribe_request_body(send_id, {symbol});
LOG(md, Info, "Subscribing {} depth", symbol);
me_->socket->send_message(io::websocket::WebsocketMessage{
send_id, req_body.c_str(), req_body.size()});
}
void BinanceDiffDepthSubscription::remove_symbol(const std::string &symbol) {
me_->unsubscribe_symbol(symbol);
bool BinanceDiffDepthSubscription::remove_listener(
const std::string &symbol, IMarketDataListener *listener) {
bts_assert(listener);
auto it = me_->listeners.find(symbol);
if (it == me_->listeners.end()) {
LOG(md, Error, "Failed to remove Binance '{}' depth listener for",
symbol);
return false;
}
it->second.remove_listener(listener);
if (it->second.num_listeners() > 0 || !me_->is_connected()) {
return true;
}
// live unsubscribe
LOG(md, Info, "Unsubscribing from Binance '{}' depth", symbol);
auto stream_name = get_stream_name(symbol);
nlohmann::json j;
auto send_id = me_->send_id_counter++;
j["method"] = "UNSUBSCRIBE";
j["params"] = std::vector<std::string>{stream_name};
j["id"] = send_id;
std::stringstream payload_ss;
payload_ss >> j;
auto payload = payload_ss.str();
LOG(md, Info, "Unsubscribing {} depth", symbol);
me_->socket->send_message(io::websocket::WebsocketMessage{
send_id, payload.c_str(), payload.size()});
return true;
}
} // namespace bts::md::binance
@@ -1,5 +1,6 @@
#pragma once
#include "lib/io/URL.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/binance/BinanceDiffDepthUpdate.h"
#include "server/sparkd/cmds/spartan/system/Delegate.h"
#include <memory>
@@ -21,8 +22,7 @@ class BinanceDepthStreamParams;
class BinanceDiffDepthSubscription {
public:
BinanceDiffDepthSubscription();
explicit BinanceDiffDepthSubscription(
bts::md::binance::BinanceDepthStreamParams params);
explicit BinanceDiffDepthSubscription(BinanceDepthStreamParams params);
BinanceDiffDepthSubscription(const BinanceDiffDepthSubscription &) = delete;
BinanceDiffDepthSubscription(BinanceDiffDepthSubscription &&) noexcept;
@@ -39,20 +39,8 @@ class BinanceDiffDepthSubscription {
*/
void connect();
/**
* @brief Adds a symbol to the market data subscription
*
* @note Idempotent
* @note Can be called before or after `connect` has been called
*/
void add_symbol(const std::string &symbol);
/**
* @brief Removes a symbol from the market data subscription
*
* @note Idempotent
*/
void remove_symbol(const std::string &symbol);
void add_listener(const std::string &symbol, IMarketDataListener *listener);
bool remove_listener(const std::string &symbol, IMarketDataListener *listener);
using OnDepthChanged = spartan::Delegate<void(
const std::string &, bts::md::binance::BinanceDiffDepthUpdate)>;