#include "BinanceDiffDepthSubscription.h" #include "BinanceDepthBootstrapper.h" #include "BinanceDepthSnapshot.h" #include "BinanceDepthStreamParams.h" #include "BinanceDiffDepthUpdate.h" #include "BinanceWebsocketStreamDemuxer.h" #include "lib/io/http/HTTPClient.h" #include "lib/io/http/HTTPRequest.h" #include "lib/io/http/HTTPResponse.h" #include "lib/io/websocket/WebsocketClient.h" #include "lib/io/websocket/WebsocketClientConfiguration.h" #include "lib/logging/logging.h" #include "lib/md/IMarketDataListener.h" #include "lib/md/binance/BinanceDepthStreamParams.h" #include "server/sparkd/cmds/spartan/system/Delegate.h" #include #include #include #include #include namespace bts::md::binance { static std::string get_stream_name(const std::string &symbol) { return fmt::format("{}@depth", symbol); } struct DiffDepthReq { std::string symbol; BinanceDepthBootstrapper bootstrapper; DiffDepthReq() { bootstrapper.on_depth_changed().bind<&DiffDepthReq::on_depth_changed>( *this); } DiffDepthReq(const DiffDepthReq &) = delete; DiffDepthReq(DiffDepthReq &&) noexcept = default; DiffDepthReq &operator=(const DiffDepthReq &) = delete; DiffDepthReq &operator=(DiffDepthReq &&) noexcept = default; ~DiffDepthReq() = default; BinanceDiffDepthSubscription::OnDepthChanged *mux_callback; void on_depth_changed(const BinanceDiffDepthUpdate &upd) { if (mux_callback) { (*mux_callback)(symbol, upd); } } }; struct BinanceDiffDepthSubscription::PImpl { BinanceDepthStreamParams params; bts::io::http::HTTPClient http_client; std::optional socket; std::unordered_map depth_reqs; // Listens directly to the socket, and demuxes book updates by symbol. // (invokes a delegate with args for the symbol and book-update itself). BinanceWebsocketStreamDemuxer demuxer; // Once connected, the websocket accepts live subscribe/unsubscribe requests. // Requests are json strings with 3 params, one of which is the send_id. // It's only purpose is to associate a response with a request. // { // "method": <"SUBSCRIBE" | "UNSUBSCRIBE">, // "params": [ // , // , // // ], // "send_id": // } std::uint64_t send_id_counter; OnDepthChanged on_depth_changed; bool is_connected = false; DiffDepthReq &subscribe_symbol(const std::string &symbol) { auto [req_it, is_new] = depth_reqs.try_emplace(symbol); auto &req = req_it->second; if (!is_new) { return req; } auto &cb = demuxer.add_handler(get_stream_name(symbol)); cb.bind<&BinanceDepthBootstrapper::on_diff_update>(req.bootstrapper); req.symbol = symbol; req.mux_callback = &on_depth_changed; if (!socket.has_value()) { return req; } // live subscribe auto send_id = send_id_counter++; auto req_body = BinanceDepthStreamParams::live_subscribe_request_body( send_id, {symbol}); LOG(md, Info, "Subscribing {} depth", symbol); socket.value().send_message(io::websocket::WebsocketMessage{ send_id, req_body.c_str(), req_body.size()}); return req; } void unsubscribe_symbol(const std::string &symbol) { if (!socket.has_value()) { return; } auto it = depth_reqs.find(symbol); if (it == depth_reqs.end()) { return; } depth_reqs.erase(it); // live unsubscribe auto stream_name = get_stream_name(symbol); nlohmann::json j; auto send_id = send_id_counter++; j["method"] = "UNSUBSCRIBE"; j["params"] = std::vector{stream_name}; j["id"] = send_id; std::stringstream payload_ss; payload_ss >> j; auto payload = payload_ss.str(); LOG(md, Info, "Unsubscribing {} depth", symbol); socket.value().send_message(io::websocket::WebsocketMessage{ send_id, payload.c_str(), payload.size()}); } }; BinanceDiffDepthSubscription::BinanceDiffDepthSubscription() : me_(std::make_unique()) {} BinanceDiffDepthSubscription::BinanceDiffDepthSubscription( BinanceDepthStreamParams params) : me_(std::make_unique()) { me_->params = std::move(params); } BinanceDiffDepthSubscription::BinanceDiffDepthSubscription( BinanceDiffDepthSubscription &&) noexcept = default; BinanceDiffDepthSubscription &BinanceDiffDepthSubscription::operator=( BinanceDiffDepthSubscription &&) noexcept = default; BinanceDiffDepthSubscription::~BinanceDiffDepthSubscription() { if (me_->socket) { LOG(md, bts::Info, "Disconnecting socket. url: {}", me_->params.stream_url()); } } BinanceDiffDepthSubscription::OnDepthChanged & BinanceDiffDepthSubscription::on_depth_changed() { return me_->on_depth_changed; } void BinanceDiffDepthSubscription::connect() { if (me_->is_connected) { return; } me_->is_connected = true; // This implementation is a little tricky. Multiple depth update streams (1 // per symbol) come in on a single websocket, and each stream must be // bootstrapped independently. Market data flow is as follows // // 1) demux updates by symbol: // * The demuxer gets data directly from the websocket, and invokes // per-symbol callbacks. // // 2) bootstrap incremental stream for each symbol: // * The bootstrappers bind to the demuxed callbacks, and buffer the // updates until it can synchronize them with a depth snapshot (it gets // the snapshot via http get req). // // 3) send output: // * Each DepthReq (one per symbol) gets a seamless incremental update // stream from its bootstrapper, and forwards them to this class's // outgoing delegate along with its symbol. // Initialize the per-symbol DepthReq's for (const auto &sym : me_->params.symbols()) { me_->subscribe_symbol(sym); } // Now that the DepthReq's are bound to the demuxer, connect the demuxer to // the websocket. auto conf = io::websocket::WebsocketClientConfiguration::from_url( me_->params.stream_url()); me_->socket.emplace(conf, &me_->demuxer).start(); // Finally, give the per-symbol bootstrappers snapshots of their respective // symbol's depth. This is deliberately done after the bootstrappers are // connected to the websocket, so the snapshots are less likely to be stale. // // TODO: The proper way to bootstrap each stream would be to pass the // listeners the snapshot as soon as it is received from the exchange, // then search the buffered updates for the first update after the snapshot. // // If this update is found, merge it with all subsequent buffered updates, // feed the listener the result, then continue to feed the listener with // updates as they arrive. // // If this update is not found, get a new snapshot (if the // buffered updates are ahead of the current snapshot), and try to bootstrap // the sequence again. In the unlikely event that the snapshot is ahead of // the buffer, simply wait for the next update before attempting to bootstrap // the sequence again. for (const auto &sym : me_->params.symbols()) { io::http::HTTPRequest req; req.url() = bts::io::URL(*me_->params.snapshot_url(sym)); io::http::HTTPResponse res = me_->http_client.get(req); std::stringstream res_str; res_str << res; LOG(md, bts::Info, "Received depth snapshot from {}:\n\n{}\n", req.url(), res_str.str()); nlohmann::json j = nlohmann::json::parse(res.body()); auto snap = j.get(); me_->depth_reqs[sym].bootstrapper.on_snapshot(snap); } } void BinanceDiffDepthSubscription::add_symbol(const std::string &symbol) { me_->subscribe_symbol(symbol); } void BinanceDiffDepthSubscription::remove_symbol(const std::string &symbol) { me_->unsubscribe_symbol(symbol); } } // namespace bts::md::binance