From 9930b65ddcf9504e7fd9f63672eeea5e0843e77c Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 22 Aug 2024 05:33:28 +0000 Subject: [PATCH] refactor: Split MarketDataGateway up into providers/sources MarketDataGateway still exists, but is greatly simplified. It creates several `IMarketDataProvider`s, each of which generates a list of `IMarketDataSource`s. The market data sources provide market data to an `IMarketDataListener`. * EmumbaMarketDataProvider * CCApiMarketDataProvider * PyMarketDataProvider * OnixsMarketDataProvider * ReplayMarketDataProvider As a side effect, individual py market data sources are now explicitly limited to a single venue (it probably should have been this way all along, since the shared memory writers do not expect updates to involve multiple venues). --- src/lib/core/LOCK_GUARD.h | 7 + src/lib/md/GeneralMarketDataListener.cpp | 22 +- src/lib/md/GeneralMarketDataListener.h | 6 +- src/lib/md/IMarketDataProvider.cpp | 7 + src/lib/md/IMarketDataProvider.h | 25 + src/lib/md/IMarketDataSource.cpp | 9 + src/lib/md/IMarketDataSource.h | 20 + .../sparkd/cmds/crypto/LiveLadderDemo.cpp | 15 +- src/server/sparkd/cmds/md/cmds/Run.cpp | 14 +- src/server/sparkd/md/MarketDataGateway.cpp | 502 +++--------------- src/server/sparkd/md/MarketDataGateway.h | 38 +- .../sparkd/md/MarketDataSubscription.cpp | 68 --- src/server/sparkd/md/MarketDataSubscription.h | 64 --- .../sparkd/md/OnixsMarketDataProvider.cpp | 44 ++ .../sparkd/md/OnixsMarketDataProvider.h | 26 + .../sparkd/md/ReplayMarketDataProvider.cpp | 52 ++ .../sparkd/md/ReplayMarketDataProvider.h | 28 + .../md/ReplayMarketDataSubscription.cpp | 47 ++ .../sparkd/md/ReplayMarketDataSubscription.h | 25 + .../sparkd/md/binance/BinanceSubscription.cpp | 34 +- .../sparkd/md/binance/BinanceSubscription.h | 12 +- .../binance_coin_futures_subscription.cpp | 33 +- .../binance_coin_futures_subscription.h | 10 +- .../md/binance_us/BinanceUsSubscription.cpp | 31 +- .../md/binance_us/BinanceUsSubscription.h | 10 +- .../BinanceUsdsFuturesSubscription.cpp | 45 +- .../BinanceUsdsFuturesSubscription.h | 10 +- .../md/ccapi/CCApiMarketDataProvider.cpp | 139 +++++ .../sparkd/md/ccapi/CCApiMarketDataProvider.h | 27 + .../md/coinbase/CoinbaseSubscription.cpp | 30 +- .../sparkd/md/coinbase/CoinbaseSubscription.h | 10 +- .../databento/DatabentoMarketDataProvider.cpp | 52 ++ .../databento/DatabentoMarketDataProvider.h | 33 ++ .../sparkd/md/deribit/DeribitSubscription.cpp | 26 +- .../sparkd/md/deribit/DeribitSubscription.h | 12 +- .../md/emumba/EmumbaDeribitSubscription.cpp | 78 +++ .../md/emumba/EmumbaDeribitSubscription.h | 24 + .../md/emumba/EmumbaMarketDataProvider.cpp | 40 ++ .../md/emumba/EmumbaMarketDataProvider.h | 25 + src/server/sparkd/md/ftx/FTXSubscription.cpp | 30 +- src/server/sparkd/md/ftx/FTXSubscription.h | 11 +- .../md/glbx/DatabentoGlbxSubscription.cpp | 41 ++ .../md/glbx/DatabentoGlbxSubscription.h | 47 ++ .../sparkd/md/glbx/OnixsGlbxSubscription.cpp | 118 ++++ .../sparkd/md/glbx/OnixsGlbxSubscription.h | 48 ++ src/server/sparkd/md/okx/OkxSubscription.cpp | 27 +- src/server/sparkd/md/okx/OkxSubscription.h | 15 +- src/server/sparkd/md/recording/MDReplayer.cpp | 50 +- src/server/sparkd/md/recording/MDReplayer.h | 17 +- .../md/taifex/TaifexMarketDataProvider.cpp | 47 ++ .../md/taifex/TaifexMarketDataProvider.h | 33 ++ .../sparkd/md/taifex/TaifexSubscription.cpp | 40 ++ .../sparkd/md/taifex/TaifexSubscription.h | 43 ++ 53 files changed, 1482 insertions(+), 785 deletions(-) create mode 100644 src/lib/core/LOCK_GUARD.h create mode 100644 src/lib/md/IMarketDataProvider.cpp create mode 100644 src/lib/md/IMarketDataProvider.h create mode 100644 src/lib/md/IMarketDataSource.cpp create mode 100644 src/lib/md/IMarketDataSource.h delete mode 100644 src/server/sparkd/md/MarketDataSubscription.cpp delete mode 100644 src/server/sparkd/md/MarketDataSubscription.h create mode 100644 src/server/sparkd/md/OnixsMarketDataProvider.cpp create mode 100644 src/server/sparkd/md/OnixsMarketDataProvider.h create mode 100644 src/server/sparkd/md/ReplayMarketDataProvider.cpp create mode 100644 src/server/sparkd/md/ReplayMarketDataProvider.h create mode 100644 src/server/sparkd/md/ReplayMarketDataSubscription.cpp create mode 100644 src/server/sparkd/md/ReplayMarketDataSubscription.h create mode 100644 src/server/sparkd/md/ccapi/CCApiMarketDataProvider.cpp create mode 100644 src/server/sparkd/md/ccapi/CCApiMarketDataProvider.h create mode 100644 src/server/sparkd/md/databento/DatabentoMarketDataProvider.cpp create mode 100644 src/server/sparkd/md/databento/DatabentoMarketDataProvider.h create mode 100644 src/server/sparkd/md/emumba/EmumbaDeribitSubscription.cpp create mode 100644 src/server/sparkd/md/emumba/EmumbaDeribitSubscription.h create mode 100644 src/server/sparkd/md/emumba/EmumbaMarketDataProvider.cpp create mode 100644 src/server/sparkd/md/emumba/EmumbaMarketDataProvider.h create mode 100644 src/server/sparkd/md/glbx/DatabentoGlbxSubscription.cpp create mode 100644 src/server/sparkd/md/glbx/DatabentoGlbxSubscription.h create mode 100644 src/server/sparkd/md/glbx/OnixsGlbxSubscription.cpp create mode 100644 src/server/sparkd/md/glbx/OnixsGlbxSubscription.h create mode 100644 src/server/sparkd/md/taifex/TaifexMarketDataProvider.cpp create mode 100644 src/server/sparkd/md/taifex/TaifexMarketDataProvider.h create mode 100644 src/server/sparkd/md/taifex/TaifexSubscription.cpp create mode 100644 src/server/sparkd/md/taifex/TaifexSubscription.h diff --git a/src/lib/core/LOCK_GUARD.h b/src/lib/core/LOCK_GUARD.h new file mode 100644 index 000000000..c90b94b94 --- /dev/null +++ b/src/lib/core/LOCK_GUARD.h @@ -0,0 +1,7 @@ +#pragma once + +#define BTS_LOCK_GUARD(lockable) \ + std::lock_guard __lock_guard { lockable } + +#define BTS_LOCK_GUARD_AS(lockable, name) \ + std::lock_guard __lock_guard_##name { lockable } diff --git a/src/lib/md/GeneralMarketDataListener.cpp b/src/lib/md/GeneralMarketDataListener.cpp index 3e29730db..91a87b934 100644 --- a/src/lib/md/GeneralMarketDataListener.cpp +++ b/src/lib/md/GeneralMarketDataListener.cpp @@ -315,8 +315,7 @@ struct GeneralMarketDataListener::PImpl { true; // with true, replay works, with orders goign in ((did not push it in, since it was always false - need to test)) const std::string cc_name_prefix = "cc_security"; size_t n_bk_updates = 0; - bts::persistence::IDGenerator &id_gen = - bts::persistence::IDGenerator::instance(); + bts::persistence::IDGenerator *id_gen; GeneralSecurityIndex &index; MDProducer &md_producer; const std::shared_ptr sim_ex_gwy = @@ -334,9 +333,10 @@ struct GeneralMarketDataListener::PImpl { api::domain::Venue venue, GeneralSecurityIndex &index, MDProducer &md_producer, - std::shared_ptr sim_ex_gwy) - : venue(venue), index(index), md_producer(md_producer), - sim_ex_gwy(std::move(sim_ex_gwy)) { + std::shared_ptr sim_ex_gwy, + bts::persistence::IDGenerator *sec_id_store) + : venue(venue), id_gen(sec_id_store), index(index), + md_producer(md_producer), sim_ex_gwy(std::move(sim_ex_gwy)) { if (this->sim_ex_gwy != nullptr) { this->sim_ex_gwy->set_producers(&this->index); } @@ -398,13 +398,17 @@ void copy_books( // interface functions //////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// GeneralMarketDataListener::GeneralMarketDataListener( - GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue) + GeneralSecurityIndex &index, + MDProducer &md_producer, + api::Venue venue, + persistence::IDGenerator *sec_id_store) : me(std::make_unique( domain::DomainSecurityUtils::convert(venue), index, md_producer, bts::orderman::SimExecutionGatewayFactory::get()->get_getayeway( - venue))) { + venue), + sec_id_store)) { LOG(md, Info, "GeneralMarketDataListener::GeneralMarketDataListener venue: {}", bts::api::Venue_Name(venue)); @@ -677,7 +681,7 @@ CCSecSearchResult GeneralMarketDataListener::_find_sec( if (add_if_absent) { uint64_t bts_id = cc_id.value(); if (!me->use_given_sec_key) { - bts_id = me->id_gen.get_or_add(me->venue, sec.type(), sec.desc()); + bts_id = me->id_gen->get_or_add(me->venue, sec.type(), sec.desc()); } me->cc_sec_id_map.insert({cc_id, {bts_id}}); LOG(md, Info, @@ -929,7 +933,7 @@ uint64_t GeneralMarketDataListener::_generate_security_key( auto logical_key = get_logical_key(sec); // if (me->use_given_sec_key) // return sec.security_common.id.value(); - return me->id_gen.get_or_add(me->venue, sec.type(), sec.desc()); + return me->id_gen->get_or_add(me->venue, sec.type(), sec.desc()); } void GeneralMarketDataListener::_possibly_match_with_sim_orders( diff --git a/src/lib/md/GeneralMarketDataListener.h b/src/lib/md/GeneralMarketDataListener.h index d14af272e..290e35ef3 100644 --- a/src/lib/md/GeneralMarketDataListener.h +++ b/src/lib/md/GeneralMarketDataListener.h @@ -5,6 +5,7 @@ #include "lib/md/GeneralSecurityIndex.h" #include "lib/md/IMarketDataListener.h" #include "lib/md/Shm/FixedSecurity.h" +#include "lib/persistence/IDGenerator.h" namespace bts::md { class MDProducer; @@ -16,7 +17,10 @@ namespace bts::md { class GeneralMarketDataListener : public IMarketDataListener { public: GeneralMarketDataListener( - GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue); + GeneralSecurityIndex &index, + MDProducer &md_producer, + api::Venue venue, + persistence::IDGenerator *sec_id_store); virtual ~GeneralMarketDataListener(); void on_securities(const std::vector &complete_security_list) override; diff --git a/src/lib/md/IMarketDataProvider.cpp b/src/lib/md/IMarketDataProvider.cpp new file mode 100644 index 000000000..fe2bbe22c --- /dev/null +++ b/src/lib/md/IMarketDataProvider.cpp @@ -0,0 +1,7 @@ +#include "IMarketDataProvider.h" + +namespace bts::md { + +IMarketDataProvider::~IMarketDataProvider() = default; + +} // namespace bts::md diff --git a/src/lib/md/IMarketDataProvider.h b/src/lib/md/IMarketDataProvider.h new file mode 100644 index 000000000..a561477ab --- /dev/null +++ b/src/lib/md/IMarketDataProvider.h @@ -0,0 +1,25 @@ +#pragma once + +#include "lib/api/domain/Venue.pb.h" +#include "lib/md/IMarketDataSource.h" + +#include + +namespace bts::md { + +/** + * @brief This is a single function interface, whose sole purpose is to provide a list of market + * data sources (per venue). A good example implementation is the `CCApiMarketDataProvider`, which + * reads `MarketDataConfig` and returns an `IMarketDataSource` for each venue configuration (e.g., + * `bts::api::md::BinanceMarketDataConfig`, `bts::api::md::CoinbaseMarketDataConfig`, etc.). + */ +class IMarketDataProvider { + public: + virtual ~IMarketDataProvider(); + + virtual std::vector< + std::pair>> + get_sources() = 0; +}; + +} // namespace bts::md diff --git a/src/lib/md/IMarketDataSource.cpp b/src/lib/md/IMarketDataSource.cpp new file mode 100644 index 000000000..b1b24bfc8 --- /dev/null +++ b/src/lib/md/IMarketDataSource.cpp @@ -0,0 +1,9 @@ +#include "IMarketDataSource.h" + +namespace bts::md { + +IMarketDataSource::~IMarketDataSource() = default; + +void IMarketDataSource::stop() {} + +} // namespace bts::md diff --git a/src/lib/md/IMarketDataSource.h b/src/lib/md/IMarketDataSource.h new file mode 100644 index 000000000..303e69084 --- /dev/null +++ b/src/lib/md/IMarketDataSource.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace bts::md { + +class IMarketDataListener; + +/** + * @brief This interface represents a source of market data. Call `IMarketDataSource::start` to + * connect an `IMarketDataListener` to the source and begin receiving market data. + */ +class IMarketDataSource { + public: + virtual ~IMarketDataSource(); + virtual void start(std::shared_ptr listener) = 0; + virtual void stop(); +}; + +} // namespace bts::md diff --git a/src/server/sparkd/cmds/crypto/LiveLadderDemo.cpp b/src/server/sparkd/cmds/crypto/LiveLadderDemo.cpp index 18cab31a5..b5d6585fa 100644 --- a/src/server/sparkd/cmds/crypto/LiveLadderDemo.cpp +++ b/src/server/sparkd/cmds/crypto/LiveLadderDemo.cpp @@ -1,19 +1,24 @@ #include "LiveLadderDemo.h" #include "lib/api/md/MarketDataConfig.pb.h" +#include "lib/config/config.h" +#include "lib/persistence/IDGenerator.h" #include "server/sparkd/md/MarketDataGateway.h" -#include "server/sparkd/md/MarketDataRegistration.h" #include "server/sparkd/md/listeners/LiveLadderPrinter.h" -#include + +#include namespace bts::cmds { -LiveLadderDemo::LiveLadderDemo(const api::md::MarketDataConfig &) { +using bts::api::md::MarketDataConfig; +using bts::persistence::IDGenerator; + +LiveLadderDemo::LiveLadderDemo(const MarketDataConfig &config) { char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt"; putenv(ssl_cert_file); - md::MarketDataGateway gw; + md::MarketDataGateway gw{config, IDGenerator::instance_ptr()}; md::listeners::LiveLadderPrinter ladder_printer; - gw.run(); + gw.start(); const std::size_t n_loops = 16; for (std::size_t n = 0; n < n_loops; n++) { diff --git a/src/server/sparkd/cmds/md/cmds/Run.cpp b/src/server/sparkd/cmds/md/cmds/Run.cpp index 2da9eca3a..d11b0e1ad 100644 --- a/src/server/sparkd/cmds/md/cmds/Run.cpp +++ b/src/server/sparkd/cmds/md/cmds/Run.cpp @@ -1,5 +1,6 @@ #include "Run.h" #include "Cleanup.h" +#include "lib/api/md/MarketDataConfig.pb.h" #include "lib/config/protos/market_data.pb.h" #include "lib/domain/SecurityCriteria.h" #include "lib/logging/logging.h" @@ -7,6 +8,7 @@ #include "lib/md/UserSymbol.h" #include "lib/md/relay/RelayRecorder.h" #include "lib/os/os.h" +#include "lib/persistence/IDGenerator.h" #include "lib/utils/bts_assert.h" #include "lib/utils/thread_manager.h" #include "server/sparkd/md/MarketDataGateway.h" @@ -14,6 +16,7 @@ #include "server/sparkd/md/platforms/ice_onixs/ICEMain.h" #include "server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h" #include +#include namespace bts::cmds::md::cmds { @@ -91,9 +94,14 @@ static void run_onixs( static void run_md_gwy() { LOG(md, Info, "Run::run_md_gwy t_id:{} t_name:{}", os::get_thread_id(), os::get_thread_name()); - bts::md::MarketDataGateway gwy = {}; - // this keeps going in a loop - gwy.run(); + bts::md::MarketDataGateway gwy{ + config::get(), + persistence::IDGenerator::instance_ptr()}; + gwy.start(); + while (true) { + using namespace std::chrono_literals; + std::this_thread::sleep_for(2s); + } } void Run::run() { diff --git a/src/server/sparkd/md/MarketDataGateway.cpp b/src/server/sparkd/md/MarketDataGateway.cpp index 27a85a90e..dbb4e4a17 100644 --- a/src/server/sparkd/md/MarketDataGateway.cpp +++ b/src/server/sparkd/md/MarketDataGateway.cpp @@ -1,70 +1,44 @@ #include "MarketDataGateway.h" -#include "lib/api/domain/Venue.pb.h" -#include "lib/api/md/EmumbaDeribitMarketDataConfig.pb.h" +#include "lib/api/domain/Venue.h" #include "lib/api/md/MarketDataConfig.pb.h" #include "lib/api/md/MarketDataConfigUtils.h" -#include "lib/api/md/MarketDataReplayConfig.pb.h" -#include "lib/collections/Contains.h" -#include "lib/config/protos/system_config.pb.h" #include "lib/core/FMT.h" +#include "lib/core/TimeSpan.h" +#include "lib/domain/DomainSecurityUtils.h" #include "lib/domain/SecurityID.h" -#include "lib/domain_model/Security.pb.h" -#include "lib/io/ProtoPrinter.h" +#include "lib/domain_model/Venue.h" #include "lib/logging/logging.h" #include "lib/md/GeneralMarketDataListener.h" #include "lib/md/GeneralSecurityIndex.h" +#include "lib/md/IMarketDataListener.h" +#include "lib/md/IMarketDataProvider.h" +#include "lib/md/IMarketDataSource.h" #include "lib/md/MDProducer.h" +#include "lib/os/os.h" #include "lib/persistence/IDGenerator.h" -#include "lib/utils/Dispatcher.h" -#include "lib/utils/bts_assert.h" #include "lib/utils/bts_fatal.h" -#include "lib/utils/thread_manager.h" -#include "server/sparkd/md/MarketDataListenerID.h" -#include "server/sparkd/md/MarketDataRegistration.h" -#include "server/sparkd/md/MarketDataRegistrationError.h" -#include "server/sparkd/md/MarketDataRegistrationFactory.h" -#include "server/sparkd/md/MarketDataSubscription.h" -#include "server/sparkd/md/binance/BinanceSubscription.h" -#include "server/sparkd/md/binance_coin_futures/binance_coin_futures_subscription.h" -#include "server/sparkd/md/binance_us/BinanceUsSubscription.h" -#include "server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h" -#include "server/sparkd/md/cboe/CboeMarketDataHandler.h" -#include "server/sparkd/md/coinbase/CoinbaseSubscription.h" -#include "server/sparkd/md/deribit/DeribitSubscription.h" -#include "server/sparkd/md/emumba/EmumbaDeribitData.h" -#include "server/sparkd/md/ftx/FTXSubscription.h" -#include "server/sparkd/md/glbx/GlbxMarketDataHandler.h" -#include "server/sparkd/md/okx/OkxSubscription.h" -#include "server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h" -#include "server/sparkd/md/recording/MDReplayer.h" -#include "server/sparkd/md/taifex/TaifexDataGateway.h" -#include "server/sparkd/md/twse/TwseDataGateway.h" +#include "server/sparkd/md/OnixsMarketDataProvider.h" +#include "server/sparkd/md/ReplayMarketDataProvider.h" +#include "server/sparkd/md/ccapi/CCApiMarketDataProvider.h" +#include "server/sparkd/md/databento/DatabentoMarketDataProvider.h" +#include "server/sparkd/md/emumba/EmumbaMarketDataProvider.h" +#include "server/sparkd/md/taifex/TaifexMarketDataProvider.h" #include -#include -#include -#include #include -#include -#include -#include #include namespace bts::md { using bts::api::md::MarketDataConfig; -using bts::config::SystemConfig; -//using bts::domain::Security; +using bts::domain::DomainSecurityUtils; using bts::domain::SecurityID; -using bts::io::ProtoPrinter; using bts::persistence::IDGenerator; +using std::shared_ptr; using std::string; using std::unique_ptr; -using std::unordered_map; using std::vector; -using Path = boost::filesystem::path; - static bool validate(const MarketDataConfig &cfg, vector &errs) { auto orig_n_errs = errs.size(); if (!cfg.deribit().empty() && !cfg.emumba_deribit().empty()) { @@ -74,393 +48,73 @@ static bool validate(const MarketDataConfig &cfg, vector &errs) { return errs.size() == orig_n_errs; } -struct MarketDataListenerData { - MarketDataListenerData() = default; - MarketDataListenerData(const MarketDataListenerData &) noexcept = delete; - MarketDataListenerData(MarketDataListenerData &&) noexcept = default; - MarketDataListenerData &operator=(const MarketDataListenerData &) noexcept = - delete; - MarketDataListenerData &operator=(MarketDataListenerData &&) noexcept = - default; - ~MarketDataListenerData() = default; - - MarketDataListenerID id; - MarketDataSubscription subscription; -}; - -struct MDReplayData { - bts::api::md::MarketDataReplayConfig replay_conf; - bts::api::Venue venue; - bts::md::GeneralSecurityIndex sec_index; - bts::md::MDProducer md_producer; - bts::md::recording::MDReplayer md_replayer; - - MDReplayData(const bts::api::md::MarketDataReplayConfig &replay_conf) - : replay_conf(replay_conf), venue(replay_conf.venue()), sec_index(), - md_replayer( - replay_conf.venue(), - sec_index, - md_producer, - replay_conf.publish_boostrap_books_only(), - replay_conf.replay_speed()) {} -}; - -struct VenueListenerData { - bts::md::GeneralSecurityIndex sec_index; - bts::md::MDProducer md_producer; - std::shared_ptr listener; - std::optional ccapi_dispatcher; - utils::DispatcherTimer timer; - - VenueListenerData(bts::api::Venue venue) - : sec_index(), - listener(std::make_shared( - sec_index, md_producer, venue)), - timer(utils::Dispatcher::background(), 200, [&]() { - md_producer.heartbeat(); - }) {} -}; - struct MarketDataGateway::PImpl { - api::md::MarketDataConfig md_config = - config::get(); - MarketDataRegistrationFactory registration_factory; - std::shared_ptr sec_id_store = IDGenerator::instance_ptr(); - unordered_map listeners; - std::vector> replays = {}; - // we create these and pass them to the cc md sources - std::vector> venue_listeners = {}; + struct MdSink { + GeneralSecurityIndex sec_index; + MDProducer md_producer; + shared_ptr listener; + + MdSink(api::Venue venue, IDGenerator *sec_id_store) + : sec_index(), listener(std::make_shared( + sec_index, md_producer, venue, sec_id_store)) {} + }; + + MarketDataConfig config; + shared_ptr sec_id_store; + vector> providers; + vector> sinks; + vector> sources; + + PImpl(MarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) { + providers.push_back(std::make_unique( + this->config, this->sec_id_store)); + providers.push_back( + std::make_unique(this->config)); + providers.push_back( + std::make_unique(this->config)); + providers.push_back(std::make_unique( + this->config, this->sec_id_store)); + providers.push_back(std::make_unique( + this->config, this->sec_id_store)); + providers.push_back(std::make_unique( + this->config, this->sec_id_store)); + } + + void start() { + bts_assert(sources.empty()); + vector cfg_errs; + if (!validate(config, cfg_errs)) { + BTS_FATAL_MSGF("Invalid MarketDataConfig:", FMT_JOIN(cfg_errs, "\n")); + return; + } + LOG(md, Info, "MarketDataGateway::run starting t_id:{} t_name:{}", + os::get_thread_id(), os::get_thread_name()); + for (auto &provider : providers) { + for (auto &[venue, source] : provider->get_sources()) { + sources.push_back(source); + auto &sink = sinks.emplace_back(std::make_unique( + DomainSecurityUtils::convert(venue), sec_id_store.get())); + LOG(md, Info, "MarketDataGateway::run. Starting {} md", venue); + source->start(sink->listener); + } + } + } + + void stop() { + for (auto &source : sources) { + source->stop(); + } + } }; -MarketDataGateway::MarketDataGateway() - : me_(std::make_unique()) {} +MarketDataGateway::MarketDataGateway( + MarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique( + std::move(config), std::move(sec_id_store))) {} MarketDataGateway::~MarketDataGateway() = default; -// starts the market data sources and then keeps the function going -void MarketDataGateway::run() { - vector cfg_errs; - if (!validate(me_->md_config, cfg_errs)) { - BTS_FATAL_MSGF("Invalid MarketDataConfig:", FMT_JOIN(cfg_errs, "\n")); - return; - } - LOG(md, Info, "MarketDataGateway::run starting t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); - _start_cc(me_->md_config); - if (!me_->md_config.emumba_deribit().empty()) { - bts_assert(me_->md_config.emumba_deribit().size() == 1); - _start_emumba_deribit(me_->md_config.emumba_deribit().at(0)); - } - start_glbx(me_->md_config); - start_taifex(me_->md_config); - start_twse(me_->md_config); - start_cboe(me_->md_config); - _start_replays(me_->md_config); - while (true) { - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - LOG(md, Info, "MarketDataGateway::run ending t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); -} - -// CCAPI ////////////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////////////////////// -MarketDataRegistration MarketDataGateway::_start_cc( - const MarketDataConfig &cfg) noexcept { - LOG(md, Info, "MarketDataGateway::_start_cc starting t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); - auto reg = me_->registration_factory.create(); - bts_assert(reg.listener_id.has_value()); - bts_assert(!CONTAINS_KEY(me_->listeners, *reg.listener_id)); - LOG(md, Info, "MarketDataGateway subscribing to data:\n{}", - ProtoPrinter::to_string(cfg)); - - vector cfg_errs; - if (!validate(cfg, cfg_errs)) { - return MarketDataRegistration{ - {MarketDataRegistrationError::Type::CONFIG_ERROR, - fmt::to_string(fmt::join(cfg_errs, "\n"))}}; - } - - MarketDataListenerData &listener_data = me_->listeners[*reg.listener_id]; - listener_data.id = *reg.listener_id; - - // FTX - for (const auto &ftx : cfg.ftx()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_FTX)); - listener_data.subscription.add(ftx::FTXSubscription{ - &p->ccapi_dispatcher.emplace(), ftx, me_->sec_id_store.get(), - p->listener.get()}); - } - - // Binance - for (const auto &binance : cfg.binance()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_BINANCE)); - listener_data.subscription.add(binance::BinanceSubscription{ - &p->ccapi_dispatcher.emplace(), binance, me_->sec_id_store.get(), - p->listener.get()}); - } - - // Binance Coin Futures - for (const auto &binance_cfs : cfg.binance_coin_futures()) { - auto p = - me_->venue_listeners.emplace_back(std::make_shared( - bts::api::VENUE_BINANCE_COIN_FUTURES)); - listener_data.subscription.add( - binance_coin_futures::BinanceCoinFuturesSubscription{ - &p->ccapi_dispatcher.emplace(), binance_cfs, - me_->sec_id_store.get(), p->listener.get()}); - } - - // Binance US - for (const auto &binance_us : cfg.binance_us()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_BINANCE_US)); - listener_data.subscription.add(binance_us::BinanceUsSubscription{ - &p->ccapi_dispatcher.emplace(), binance_us, me_->sec_id_store.get(), - p->listener.get()}); - } - // Coinbase - for (const auto &coinbase : cfg.coinbase()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_COINBASE)); - listener_data.subscription.add(coinbase::CoinbaseSubscription{ - &p->ccapi_dispatcher.emplace(), coinbase, me_->sec_id_store.get(), - p->listener.get()}); - } - // Binance USDS Futures - for (const auto &binance_ufs : cfg.binance_usds_futures()) { - auto p = - me_->venue_listeners.emplace_back(std::make_shared( - bts::api::VENUE_BINANCE_USDS_FUTURES)); - listener_data.subscription.add( - binance_usds_futures::BinanceUsdsFuturesSubscription{ - &p->ccapi_dispatcher.emplace(), binance_ufs, - me_->sec_id_store.get(), p->listener.get()}); - } - - // Deribit - for (const auto &deribit : cfg.deribit()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_DERIBIT)); - listener_data.subscription.add(deribit::DeribitSubscription{ - &p->ccapi_dispatcher.emplace(), deribit, me_->sec_id_store.get(), - p->listener.get()}); - } - - // Okx - for (const auto &okx : cfg.okx()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_OKX)); - listener_data.subscription.add(okx::OkxSubscription{ - &p->ccapi_dispatcher.emplace(), okx, me_->sec_id_store.get(), - p->listener.get()}); - } - - LOG(md, Info, "MarketDataGateway::_start_cc ending t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); - return reg; -} - -void MarketDataGateway::unsubscribe(MarketDataListenerID id) { - LOG(md, Info, "Unsubscribing listener {}", id); - me_->listeners.erase(id); -} -/// glbx ////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////////// -void MarketDataGateway::start_glbx(const bts::api::md::MarketDataConfig &cfg) { - for (const auto &glbx : cfg.glbx()) { - if (!glbx.use_databento()) { - _start_onix_glbx(glbx); - return; - } - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_GLBX)); - // Capture `p->listener` and `me_->sec_id_store` by value - auto listener = p->listener; - auto sec_id_store = me_->sec_id_store; - std::string tid_name = "glbx"; - bts::threading::make_thread(tid_name, [listener, sec_id_store, glbx]() { - api::md::glbx::GlbxMarketDatahandler::run( - listener, sec_id_store, glbx); - }).detach(); - } -} -static void run_onixs() { - LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name", - os::get_thread_id(), os::get_thread_name()); - bts::md::MBOWriter *mbo = nullptr; - bts::md::TradeSummarySignalDispatcher *tssd = nullptr; - std::vector> platforms; - // get all glbx symbols - auto cme_symbols = SymbolLoader::load( - bts::domain::SecurityCriteria::create({api::domain::VENUE_GLBX})); - if (!cme_symbols.empty()) { - auto *main = new bts::md::MDP3Main(); - main->start({cme_symbols.begin(), cme_symbols.end()}, mbo, tssd); - platforms.emplace_back(main); - } - // deribit - // auto *d_platform = new bts::md::EmMDListener() - // Timers - auto &disp = utils::Dispatcher::current(); - utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() { - for (auto &p : platforms) - p->send_heartbeats(); - }); - heartbeat_timer.start(); - - disp.run(); -} - -void MarketDataGateway::_start_onix_glbx( - bts::api::md::GlbxMarketDataConfig cfg) { - LOG(md, Info, - "MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. " - "cfg:\n{}", - os::get_thread_id(), os::get_thread_name(), cfg.DebugString()); - bts::threading::make_thread("Onix CME", [&]() { - bts::md::run_onixs(); - }).detach(); -} - -// deribit //////////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////////////////////// - -void MarketDataGateway::_start_emumba_deribit( - bts::api::md::EmumbaDeribitMarketDataConfig cfg) { - LOG(md, Info, - "MarketDataGateway::_start_emumba_deribit starting t_id:{} t_name:{}. " - "cfg:\n{}", - os::get_thread_id(), os::get_thread_name(), cfg.DebugString()); - auto start_deribit_thr = [cfg = std::move(cfg)]() { - const auto config_path = Path{config::get().config_path()} / - cfg.json_conf_file_name(); - const auto conf = - bts::cmds::DeribitDemoConfiguration::read_file(config_path.string()); - bts_assert_msgf( - conf.has_value(), "Failed to parse deribit config '{}'", - config_path.string()); - // log - for (const auto &s : conf.value().symbol_config_list) { - LOG(md, Info, "sub.symbol: {}", s.symbol_name); - LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future, - s.interested_in_option); - for (const auto &i : s.interested_instruments) { - LOG(md, Info, "sub.instrument: {}", i); - } - } - EmumbaDeribitData ddata_obj; - EmumbaDeribitData *ddata = &ddata_obj; - // end log - bts_assert(conf.has_value()); - LOG(md, Info, "Deribit conf: {}", conf.value()); - ddata->real_handler = - std::make_shared( - conf.value().symbol_config_list, conf.value().trading_config, - conf.value().credential_map, conf.value().logging_configuration, - conf.value().deribit_url, &ddata->listener); - // since the trading interface is housed inside the EmMDListener, we need to - // create and send emumba's trading_session_interface - for (const auto &p : conf.value().credential_map) { - const auto &sess = p.second; - auto ts_ptr = std::make_shared< - emumba::deribit_connector::trading_session_interface>( - ddata->real_handler->create_new_trading_session(sess)); - ddata->trading_session_vec.push_back({sess, ts_ptr}); - } - bts_assert(!ddata->trading_session_vec.empty()); - ddata->listener.set_handler( - ddata->real_handler.get(), ddata->trading_session_vec); - ddata->real_handler->run(); - }; - threading::make_thread("Emumba", start_deribit_thr).detach(); - LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); -} - -// replays //////////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////////////////////// -void MarketDataGateway::_start_replays( - const bts::api::md::MarketDataConfig &cfg) { - LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); - for (const auto &r : - MarketDataConfigUtils::get_effective_replay_config(cfg)) { - me_->replays.emplace_back(std::make_unique(r)); - } - for (const auto &replayer : me_->replays) { - std::string tid_name = "MDReplayer_" + Venue_Name(replayer->venue); - bts::threading::make_thread(tid_name, [&]() { - replayer->md_replayer.run(); - }).detach(); - } - LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); -} - -void MarketDataGateway::start_twse(const bts::api::md::MarketDataConfig &cfg) { - // twse -// int counter = 0; -// LOG(md, Info, "In start, mdgtway.cpp"); - -// for (const auto &twse : cfg.twse()) { -// counter++; -// auto p = me_->venue_listeners.emplace_back( -// std::make_shared(bts::api::VENUE_TWSE)); -// std::string tid_name = "twse" + std::to_string(counter); -// bts::threading::make_thread(tid_name, [&]() { -// bts::md::twse::TwseDataGateway::run( -// twse, p->listener, me_->sec_id_store); -// }).detach(); -// } - for (const auto &twse : cfg.twse()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_TWSE)); - // Capture `p->listener` and `me_->sec_id_store` by value - auto listener = p->listener; - auto sec_id_store = me_->sec_id_store; - std::string tid_name = "twse"; - bts::threading::make_thread(tid_name, [listener, sec_id_store, twse]() { - bts::md::twse::TwseDataGateway::run( - twse, listener, sec_id_store); - }).detach(); - - } -} - -void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) { - // taifex - for (const auto &taifex : cfg.taifex()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_TAIFEX)); - // Capture `p->listener` and `me_->sec_id_store` by value - auto listener = p->listener; - auto sec_id_store = me_->sec_id_store; - std::string tid_name = "taifex"; - bts::threading::make_thread(tid_name, [listener, sec_id_store, taifex]() { - bts::md::taifex::TaifexDataGateway::run( - taifex, listener, sec_id_store); - }).detach(); - } -} -void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg) { - for (const api::md::CboeMarketDataConfig &cboe : cfg.cboe()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_CBOE)); - std::string tid_name = "CBOE"; - - // Capture `p->listener` and `me_->sec_id_store` by value - auto listener = p->listener; - auto sec_id_store = me_->sec_id_store; - - bts::threading::make_thread(tid_name, [listener, sec_id_store, cboe]() { - api::md::cboe::CboeMarketDatahandler::run( - listener, sec_id_store, cboe); - }).detach(); - } -} +void MarketDataGateway::start() { me_->start(); } } // namespace bts::md diff --git a/src/server/sparkd/md/MarketDataGateway.h b/src/server/sparkd/md/MarketDataGateway.h index 473b063c5..11e2bc3a0 100644 --- a/src/server/sparkd/md/MarketDataGateway.h +++ b/src/server/sparkd/md/MarketDataGateway.h @@ -1,45 +1,33 @@ #pragma once -#include "lib/api/md/GlbxMarketDataConfig.pb.h" -#include "server/sparkd/md/MarketDataListenerID.h" - #include namespace bts::api::md { -class EmumbaDeribitMarketDataConfig; class MarketDataConfig; -} // namespace bts::api::md +} + +namespace bts::persistence { +class IDGenerator; +} namespace bts::md { -class IMarketDataListener; -class MarketDataRegistration; - +/** + * @brief The `MarketDataGateway` is responsible for starting all configured market data sources. + * It connects each source to a special `IMarketDataListener` that writes the data to shared memory. + */ class MarketDataGateway { public: - MarketDataGateway(); + explicit MarketDataGateway( + api::md::MarketDataConfig config, + std::shared_ptr sec_id_store); ~MarketDataGateway(); - /** - * @brief Connects a market data listener to a market data source - * @return MarketDataRegistration - */ - void run(); - void unsubscribe(MarketDataListenerID id); + void start(); private: struct PImpl; std::unique_ptr me_; - MarketDataRegistration _start_cc(const api::md::MarketDataConfig &) noexcept; - static void _start_emumba_deribit( - bts::api::md::EmumbaDeribitMarketDataConfig cfg); - static void _start_onix_glbx( - bts::api::md::GlbxMarketDataConfig cfg); - void _start_replays(const bts::api::md::MarketDataConfig &cfg); - void start_glbx(const bts::api::md::MarketDataConfig &cfg); - void start_cboe(const bts::api::md::MarketDataConfig &cfg); - void start_taifex(const bts::api::md::MarketDataConfig &cfg); - void start_twse(const bts::api::md::MarketDataConfig &cfg); }; } // namespace bts::md diff --git a/src/server/sparkd/md/MarketDataSubscription.cpp b/src/server/sparkd/md/MarketDataSubscription.cpp deleted file mode 100644 index 3e1d97ad9..000000000 --- a/src/server/sparkd/md/MarketDataSubscription.cpp +++ /dev/null @@ -1,68 +0,0 @@ -#include "MarketDataSubscription.h" -#include "server/sparkd/md/binance/BinanceSubscription.h" -#include "server/sparkd/md/binance_coin_futures/binance_coin_futures_subscription.h" -#include "server/sparkd/md/binance_us/BinanceUsSubscription.h" -#include "server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h" -#include "server/sparkd/md/coinbase/CoinbaseSubscription.h" -#include "server/sparkd/md/deribit/DeribitSubscription.h" -#include "server/sparkd/md/ftx/FTXSubscription.h" -#include "server/sparkd/md/okx/OkxSubscription.h" - -#include - -namespace bts::md { - -using std::vector; - -struct MarketDataSubscription::PImpl { - vector ftx_subs; - vector binance_subs; - vector - binance_coin_futures_subs; - vector binance_us_subs; - vector coinbase_subs; - vector - binance_usds_futures_subs; - vector deribit_subs; - vector okx_subs; - // TODO: add more exchanges -}; - -MarketDataSubscription::MarketDataSubscription() - : me_(std::make_unique()) {} - -MarketDataSubscription::~MarketDataSubscription() = default; - -void MarketDataSubscription::add(binance::BinanceSubscription sub) { - me_->binance_subs.push_back(std::move(sub)); -} - -void MarketDataSubscription::add( - binance_coin_futures::BinanceCoinFuturesSubscription sub) { - me_->binance_coin_futures_subs.push_back(std::move(sub)); -} - -void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) { - me_->binance_us_subs.push_back(std::move(sub)); -} -void MarketDataSubscription::add(coinbase::CoinbaseSubscription sub) { - me_->coinbase_subs.push_back(std::move(sub)); -} -void MarketDataSubscription::add( - binance_usds_futures::BinanceUsdsFuturesSubscription sub) { - me_->binance_usds_futures_subs.push_back(std::move(sub)); -} - -void MarketDataSubscription::add(deribit::DeribitSubscription sub) { - me_->deribit_subs.push_back(std::move(sub)); -} - -void MarketDataSubscription::add(ftx::FTXSubscription sub) { - me_->ftx_subs.push_back(std::move(sub)); -} - -void MarketDataSubscription::add(okx::OkxSubscription sub) { - me_->okx_subs.push_back(std::move(sub)); -} - -} // namespace bts::md diff --git a/src/server/sparkd/md/MarketDataSubscription.h b/src/server/sparkd/md/MarketDataSubscription.h deleted file mode 100644 index 5fcd2f95d..000000000 --- a/src/server/sparkd/md/MarketDataSubscription.h +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once - -#include - -namespace bts::md::binance { -class BinanceSubscription; -} - -namespace bts::md::binance_coin_futures { -class BinanceCoinFuturesSubscription; -} - -namespace bts::md::binance_us { -class BinanceUsSubscription; -} -namespace bts::md::coinbase { -class CoinbaseSubscription; -} -namespace bts::md::binance_usds_futures { -class BinanceUsdsFuturesSubscription; -} - -namespace bts::md::deribit { -class DeribitSubscription; -} - -namespace bts::md::ftx { -class FTXSubscription; -} - -namespace bts::md::okx { -class OkxSubscription; -} - -namespace bts::md { - -class MarketDataSubscription { - public: - MarketDataSubscription(); - - MarketDataSubscription(const MarketDataSubscription &) noexcept = delete; - MarketDataSubscription(MarketDataSubscription &&) noexcept = default; - MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept = - delete; - MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept = - default; - - ~MarketDataSubscription(); - - void add(binance::BinanceSubscription); - void add(binance_coin_futures::BinanceCoinFuturesSubscription); - void add(binance_us::BinanceUsSubscription); - void add(coinbase::CoinbaseSubscription); - void add(binance_usds_futures::BinanceUsdsFuturesSubscription); - void add(deribit::DeribitSubscription); - void add(ftx::FTXSubscription); - void add(okx::OkxSubscription); - - private: - struct PImpl; - std::unique_ptr me_; -}; - -} // namespace bts::md diff --git a/src/server/sparkd/md/OnixsMarketDataProvider.cpp b/src/server/sparkd/md/OnixsMarketDataProvider.cpp new file mode 100644 index 000000000..7e960bd26 --- /dev/null +++ b/src/server/sparkd/md/OnixsMarketDataProvider.cpp @@ -0,0 +1,44 @@ +#include "OnixsMarketDataProvider.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/persistence/IDGenerator.h" +#include "server/sparkd/md/glbx/OnixsGlbxSubscription.h" + +#include +#include + +namespace bts::md { + +using bts::api::domain::Venue; +using bts::api::md::MarketDataConfig; +using bts::md::OnixsMarketDataProvider; +using bts::persistence::IDGenerator; +using std::pair; +using std::shared_ptr; +using std::string; +using std::vector; + +struct OnixsMarketDataProvider::PImpl { + MarketDataConfig config; + std::shared_ptr sec_id_store; + vector> sources; +}; + +OnixsMarketDataProvider::OnixsMarketDataProvider(MarketDataConfig config) + : me_(std::make_unique()) { + me_->config = std::move(config); +} + +OnixsMarketDataProvider::~OnixsMarketDataProvider() = default; + +vector>> +OnixsMarketDataProvider::get_sources() { + vector>> rv; + for (const auto &glbx : me_->config.glbx()) { + me_->sources.push_back( + std::make_shared(me_->sec_id_store, glbx)); + rv.push_back({Venue::VENUE_GLBX, me_->sources.back()}); + } + return rv; +} + +} // namespace bts::md diff --git a/src/server/sparkd/md/OnixsMarketDataProvider.h b/src/server/sparkd/md/OnixsMarketDataProvider.h new file mode 100644 index 000000000..283114937 --- /dev/null +++ b/src/server/sparkd/md/OnixsMarketDataProvider.h @@ -0,0 +1,26 @@ +#pragma once + +#include "lib/api/md/MarketDataConfig.pb.h" +#include "lib/md/IMarketDataProvider.h" +#include "lib/md/IMarketDataSource.h" + +#include + +namespace bts::md { + +class OnixsMarketDataProvider : public IMarketDataProvider { + public: + OnixsMarketDataProvider(); + explicit OnixsMarketDataProvider(api::md::MarketDataConfig config); + ~OnixsMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/server/sparkd/md/ReplayMarketDataProvider.cpp b/src/server/sparkd/md/ReplayMarketDataProvider.cpp new file mode 100644 index 000000000..64f12b464 --- /dev/null +++ b/src/server/sparkd/md/ReplayMarketDataProvider.cpp @@ -0,0 +1,52 @@ +#include "ReplayMarketDataProvider.h" +#include "lib/api/md/MarketDataConfig.pb.h" +#include "lib/api/md/MarketDataConfigUtils.h" +#include "lib/config/config.h" +#include "lib/domain/DomainSecurityUtils.h" +#include "lib/md/IMarketDataSource.h" +#include "lib/persistence/IDGenerator.h" +#include "server/sparkd/md/recording/MDReplayer.h" + +namespace bts::md { + +using bts::api::md::MarketDataConfig; +using bts::domain::DomainSecurityUtils; +using bts::md::recording::MDReplayer; +using bts::persistence::IDGenerator; +using std::pair; +using std::shared_ptr; +using std::string; +using std::vector; + +struct ReplayMarketDataProvider::PImpl { + MarketDataConfig config; + shared_ptr sec_id_store; + vector>> sources; + + PImpl(MarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} +}; + +ReplayMarketDataProvider::ReplayMarketDataProvider() + : ReplayMarketDataProvider( + config::get(), IDGenerator::instance_ptr()) {} + +ReplayMarketDataProvider::ReplayMarketDataProvider( + MarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { +} + +ReplayMarketDataProvider::~ReplayMarketDataProvider() = default; + +vector>> +ReplayMarketDataProvider::get_sources() { + const auto replay_cfgs = + MarketDataConfigUtils::get_effective_replay_config(me_->config); + for (const auto &r : replay_cfgs) { + const auto v = DomainSecurityUtils::convert(r.venue()); + me_->sources.push_back({v, std::make_shared(r.venue())}); + } + return me_->sources; +} + +} // namespace bts::md diff --git a/src/server/sparkd/md/ReplayMarketDataProvider.h b/src/server/sparkd/md/ReplayMarketDataProvider.h new file mode 100644 index 000000000..534fc7fe3 --- /dev/null +++ b/src/server/sparkd/md/ReplayMarketDataProvider.h @@ -0,0 +1,28 @@ +#pragma once + +#include "lib/api/md/MarketDataConfig.pb.h" +#include "lib/md/IMarketDataProvider.h" +#include "lib/persistence/IDGenerator.h" + +#include + +namespace bts::md { + +class ReplayMarketDataProvider : public IMarketDataProvider { + public: + ReplayMarketDataProvider(); + explicit ReplayMarketDataProvider( + api::md::MarketDataConfig config, + std::shared_ptr sec_id_store); + ~ReplayMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/server/sparkd/md/ReplayMarketDataSubscription.cpp b/src/server/sparkd/md/ReplayMarketDataSubscription.cpp new file mode 100644 index 000000000..31b765c4d --- /dev/null +++ b/src/server/sparkd/md/ReplayMarketDataSubscription.cpp @@ -0,0 +1,47 @@ +#include "ReplayMarketDataSubscription.h" +#include "lib/api/md/MarketDataReplayConfig.pb.h" +#include "lib/logging/logging.h" +#include "lib/md/GeneralMarketDataListener.h" +#include "lib/md/IMarketDataListener.h" +#include "lib/md/MDProducer.h" +#include "lib/utils/thread_manager.h" +#include "server/sparkd/md/recording/MDReplayer.h" + +#include +#include + +namespace bts::md { + +using bts::api::md::MarketDataReplayConfig; +using bts::md::recording::MDReplayer; +using std::string; + +struct ReplayMarketDataSource::PImpl { + MarketDataReplayConfig config; + + PImpl(const MarketDataReplayConfig &config) : config(config) {} +}; + +ReplayMarketDataSource::ReplayMarketDataSource(MarketDataReplayConfig config) + : me_(std::make_unique(config)) { + me_->config = std::move(config); +} + +ReplayMarketDataSource::~ReplayMarketDataSource() = default; + +void ReplayMarketDataSource::start( + std::shared_ptr listener) { + LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}", + os::get_thread_id(), os::get_thread_name()); + auto md_replayer = std::make_shared(me_->config.venue()); + std::string tid_name = "MDReplayer_" + Venue_Name(me_->config.venue()); + bts::threading::make_thread(tid_name, [&, listener, md_replayer]() { + md_replayer->start(listener); + }).detach(); + LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}", + os::get_thread_id(), os::get_thread_name()); +} + +void ReplayMarketDataSource::stop() {} + +} // namespace bts::md diff --git a/src/server/sparkd/md/ReplayMarketDataSubscription.h b/src/server/sparkd/md/ReplayMarketDataSubscription.h new file mode 100644 index 000000000..e4bf9d104 --- /dev/null +++ b/src/server/sparkd/md/ReplayMarketDataSubscription.h @@ -0,0 +1,25 @@ +#pragma once + +#include "lib/api/md/MarketDataReplayConfig.pb.h" +#include "lib/md/IMarketDataListener.h" +#include "lib/md/IMarketDataSource.h" + +#include + +namespace bts::md { + +class ReplayMarketDataSource : public IMarketDataSource { + public: + ReplayMarketDataSource(); + explicit ReplayMarketDataSource(api::md::MarketDataReplayConfig config); + ~ReplayMarketDataSource(); + + void start(std::shared_ptr listener) override; + void stop() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/server/sparkd/md/binance/BinanceSubscription.cpp b/src/server/sparkd/md/binance/BinanceSubscription.cpp index 93d9b0cbf..6c1a9ad0d 100644 --- a/src/server/sparkd/md/binance/BinanceSubscription.cpp +++ b/src/server/sparkd/md/binance/BinanceSubscription.cpp @@ -193,7 +193,7 @@ struct BinanceSubscription::PImpl { // The session automatically starts/stops in its ctr/dtr CCApiSession session; - IMarketDataListener *listener; + std::shared_ptr listener; // These aliases are useful for quickly switching the handler types (e.g., // a special handler that logs messages for testing/diagnostic purposes) @@ -216,20 +216,23 @@ struct BinanceSubscription::PImpl { explicit PImpl( ccapi::EventDispatcher *dispatcher, BinanceMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : sec_id_store(sec_id_store), cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), session( dispatcher, mk_session_cfg(this->cfg), mk_session_opts(), - {ccapi::BtsVenues::BINANCE}), - listener(listener) { + {ccapi::BtsVenues::BINANCE}) { LOG(md, Info, "Initializing BinanceSubscription:\n{}", ProtoPrinter::to_string(this->cfg)); - // Request the list of all spots at Binance. Match the results against our SecurityCriteria, - // and subscribe to md for each matching spot. + } + + void start(std::shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); + // Request the list of all spots at Binance. Match the results against our + // SecurityCriteria, and subscribe to md for each matching spot. start_thread = std::thread{[this]() { LOG(md, Info, "Requesting Binance securities on background thread"); std::string exch_info_url = this->cfg.url_rest_base(); @@ -266,7 +269,7 @@ struct BinanceSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - depth_handlers.try_emplace(sec_desc, sec_id, listener); + depth_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert(is_new); return it->second; } @@ -275,7 +278,7 @@ struct BinanceSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - ticker_handlers.try_emplace(sec_desc, sec_id, listener); + ticker_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Ticker handler already created for '{}'", sec_desc); return it->second; @@ -291,7 +294,8 @@ struct BinanceSubscription::PImpl { sec_desc_id_tuples.emplace_back(type, std::move(name), id); } auto [it, is_new] = stats_handlers.try_emplace( - hnd_id, std::make_unique(sec_desc_id_tuples, listener)); + hnd_id, + std::make_unique(sec_desc_id_tuples, listener.get())); bts_assert_msgf( is_new, "StatsHandler already created with id '{}'", hnd_id); return *it->second; @@ -401,10 +405,8 @@ struct BinanceSubscription::PImpl { BinanceSubscription::BinanceSubscription( ccapi::EventDispatcher *dispatcher, BinanceMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} BinanceSubscription::BinanceSubscription(BinanceSubscription &&) noexcept = default; @@ -413,4 +415,8 @@ BinanceSubscription &BinanceSubscription::operator=( BinanceSubscription::~BinanceSubscription() = default; +void BinanceSubscription::start(std::shared_ptr listener) { + me_->start(listener); +} + } // namespace bts::md::binance diff --git a/src/server/sparkd/md/binance/BinanceSubscription.h b/src/server/sparkd/md/binance/BinanceSubscription.h index be94be9a4..541305580 100644 --- a/src/server/sparkd/md/binance/BinanceSubscription.h +++ b/src/server/sparkd/md/binance/BinanceSubscription.h @@ -1,5 +1,7 @@ #pragma once +#include "lib/md/IMarketDataSource.h" + #include namespace ccapi { @@ -25,13 +27,12 @@ namespace bts::md::binance { * * @a https://binance-docs.github.io/apidocs/spot/en/#exchange-information */ -class BinanceSubscription { +class BinanceSubscription : public IMarketDataSource { public: BinanceSubscription( ccapi::EventDispatcher *dispatcher, api::md::BinanceMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); BinanceSubscription(const BinanceSubscription &) noexcept = delete; BinanceSubscription(BinanceSubscription &&) noexcept; @@ -39,10 +40,9 @@ class BinanceSubscription { delete; BinanceSubscription &operator=(BinanceSubscription &&) noexcept; - ~BinanceSubscription(); + ~BinanceSubscription() override; - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/server/sparkd/md/binance_coin_futures/binance_coin_futures_subscription.cpp b/src/server/sparkd/md/binance_coin_futures/binance_coin_futures_subscription.cpp index 34dfa7e39..5fc0a4011 100644 --- a/src/server/sparkd/md/binance_coin_futures/binance_coin_futures_subscription.cpp +++ b/src/server/sparkd/md/binance_coin_futures/binance_coin_futures_subscription.cpp @@ -18,6 +18,7 @@ #include "server/sparkd/md/ccapi/CCApiSession.h" #include "server/sparkd/md/ccapi/CCApiTradeAdaptor.h" +#include #include #include #include @@ -41,6 +42,7 @@ using bts::io::ProtoPrinter; using bts::io::QueryParams; using bts::persistence::IDGenerator; using std::pair; +using std::shared_ptr; using std::string; using std::uint32_t; using std::unique_ptr; @@ -169,7 +171,7 @@ struct BinanceCoinFuturesSubscription::PImpl { SecurityCriteria criteria; // The session automatically starts/stops in its ctr/dtr CCApiSession session; - IMarketDataListener *listener; + std::shared_ptr listener; BinanceCoinFuturesSecurityRequestor sec_requestor; // These aliases are useful for quickly switching the handler types (e.g., @@ -185,8 +187,7 @@ struct BinanceCoinFuturesSubscription::PImpl { explicit PImpl( ccapi::EventDispatcher *dispatcher, BinanceCoinFuturesMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : sec_id_store(sec_id_store), cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), session( @@ -194,10 +195,15 @@ struct BinanceCoinFuturesSubscription::PImpl { mk_session_cfg(this->cfg), {}, {ccapi::BtsVenues::BINANCE_COIN_FUTURES}), - listener(listener), sec_requestor(&session, this->cfg, sec_id_store) { - LOG(md, Info, "Initializing BinanceCoinFuturesSubscription:\n{}", + sec_requestor(&session, this->cfg, sec_id_store) { + LOG(md, Info, "Initializing BinanceUsdsFuturesSubscription:\n{}", ProtoPrinter::to_string(this->cfg)); sec_requestor.callback().bind<&PImpl::on_securities>(*this); + } + + void start(shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); sec_requestor.request(); } @@ -224,7 +230,7 @@ struct BinanceCoinFuturesSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = - depth_handlers.try_emplace(sec_desc, sec_id, listener); + depth_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Depth handler already created for '{}'", sec_desc); return it->second; @@ -234,7 +240,7 @@ struct BinanceCoinFuturesSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = - ticker_handlers.try_emplace(sec_desc, sec_id, listener); + ticker_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Ticker handler already created for '{}'", sec_desc); return it->second; @@ -244,7 +250,7 @@ struct BinanceCoinFuturesSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = - stats_handlers.try_emplace(sec_desc, sec_id, listener); + stats_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Stats handler already created for '{}'", sec_desc); return it->second; @@ -316,10 +322,8 @@ struct BinanceCoinFuturesSubscription::PImpl { BinanceCoinFuturesSubscription::BinanceCoinFuturesSubscription( ccapi::EventDispatcher *dispatcher, BinanceCoinFuturesMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} BinanceCoinFuturesSubscription::BinanceCoinFuturesSubscription( BinanceCoinFuturesSubscription &&) noexcept = default; @@ -328,4 +332,9 @@ BinanceCoinFuturesSubscription &BinanceCoinFuturesSubscription::operator=( BinanceCoinFuturesSubscription::~BinanceCoinFuturesSubscription() = default; +void BinanceCoinFuturesSubscription::start( + shared_ptr listener) { + me_->start(std::move(listener)); +} + } // namespace bts::md::binance_coin_futures diff --git a/src/server/sparkd/md/binance_coin_futures/binance_coin_futures_subscription.h b/src/server/sparkd/md/binance_coin_futures/binance_coin_futures_subscription.h index c06fd4812..f65ba80b8 100644 --- a/src/server/sparkd/md/binance_coin_futures/binance_coin_futures_subscription.h +++ b/src/server/sparkd/md/binance_coin_futures/binance_coin_futures_subscription.h @@ -1,5 +1,7 @@ #pragma once +#include "lib/md/IMarketDataSource.h" + #include namespace ccapi { @@ -25,13 +27,12 @@ namespace bts::md::binance_coin_futures { * * @a https://developers.binance.com/docs/derivatives/coin-margined-futures/market-data/Exchange-Information */ -class BinanceCoinFuturesSubscription { +class BinanceCoinFuturesSubscription : public IMarketDataSource { public: BinanceCoinFuturesSubscription( ccapi::EventDispatcher *dispatcher, api::md::BinanceCoinFuturesMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); BinanceCoinFuturesSubscription( const BinanceCoinFuturesSubscription &) noexcept = delete; @@ -43,8 +44,7 @@ class BinanceCoinFuturesSubscription { ~BinanceCoinFuturesSubscription(); - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp b/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp index c4e11330c..49cc4f1dd 100644 --- a/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp +++ b/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp @@ -47,6 +47,7 @@ using bts::io::ProtoPrinter; using bts::io::QueryParams; using bts::persistence::IDGenerator; using std::pair; +using std::shared_ptr; using std::string; using std::tuple; using std::uint32_t; @@ -174,7 +175,7 @@ struct BinanceUsSubscription::PImpl { // The session automatically starts/stops in its ctr/dtr CCApiSession session; - IMarketDataListener *listener; + shared_ptr listener; BinanceUsSecurityRequestor sec_requestor; // These aliases are useful for quickly switching the handler types (e.g., @@ -191,14 +192,13 @@ struct BinanceUsSubscription::PImpl { // different map, 'stats_handlers' (unique ids are generated w/ a counter, // 'stats_handlers_id_counter'). unordered_map sec_stats_handler_map; - unordered_map> stats_handlers; + unordered_map> stats_handlers; uint32_t stats_handlers_id_counter = 0; explicit PImpl( ccapi::EventDispatcher *dispatcher, BinanceUsMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : sec_id_store(sec_id_store), cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), session( @@ -206,7 +206,6 @@ struct BinanceUsSubscription::PImpl { mk_session_cfg(this->cfg), {}, {ccapi::BtsVenues::BINANCE_US}), - listener(listener), sec_requestor( &session, this->criteria, @@ -217,6 +216,11 @@ struct BinanceUsSubscription::PImpl { LOG(md, Info, "Initializing BinanceUsSubscription:\n{}", ProtoPrinter::to_string(this->cfg)); sec_requestor.callback().bind<&PImpl::on_securities>(*this); + } + + void start(std::shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); sec_requestor.request(); } @@ -224,7 +228,7 @@ struct BinanceUsSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - depth_handlers.try_emplace(sec_desc, sec_id, listener); + depth_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert(is_new); return it->second; } @@ -233,7 +237,7 @@ struct BinanceUsSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - ticker_handlers.try_emplace(sec_desc, sec_id, listener); + ticker_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Ticker handler already created for '{}'", sec_desc); return it->second; @@ -249,7 +253,8 @@ struct BinanceUsSubscription::PImpl { sec_desc_id_tuples.emplace_back(type, std::move(name), id); } auto [it, is_new] = stats_handlers.try_emplace( - hnd_id, std::make_unique(sec_desc_id_tuples, listener)); + hnd_id, + std::make_unique(sec_desc_id_tuples, listener.get())); bts_assert_msgf( is_new, "StatsHandler already created with id '{}'", hnd_id); return *it->second; @@ -339,10 +344,8 @@ struct BinanceUsSubscription::PImpl { BinanceUsSubscription::BinanceUsSubscription( ccapi::EventDispatcher *dispatcher, BinanceUsMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} BinanceUsSubscription::BinanceUsSubscription( BinanceUsSubscription &&) noexcept = default; @@ -351,4 +354,8 @@ BinanceUsSubscription &BinanceUsSubscription::operator=( BinanceUsSubscription::~BinanceUsSubscription() = default; +void BinanceUsSubscription::start(shared_ptr listener) { + me_->start(std::move(listener)); +} + } // namespace bts::md::binance_us diff --git a/src/server/sparkd/md/binance_us/BinanceUsSubscription.h b/src/server/sparkd/md/binance_us/BinanceUsSubscription.h index 12755cb82..4c19a4ca2 100644 --- a/src/server/sparkd/md/binance_us/BinanceUsSubscription.h +++ b/src/server/sparkd/md/binance_us/BinanceUsSubscription.h @@ -1,5 +1,7 @@ #pragma once +#include "lib/md/IMarketDataSource.h" + #include namespace ccapi { @@ -25,13 +27,12 @@ namespace bts::md::binance_us { * * @a https://docs.binance.us/#get-exchange-information */ -class BinanceUsSubscription { +class BinanceUsSubscription : public IMarketDataSource { public: BinanceUsSubscription( ccapi::EventDispatcher *dispatcher, api::md::BinanceUsMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); BinanceUsSubscription(const BinanceUsSubscription &) noexcept = delete; BinanceUsSubscription(BinanceUsSubscription &&) noexcept; @@ -41,8 +42,7 @@ class BinanceUsSubscription { ~BinanceUsSubscription(); - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp b/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp index 327c6671d..af1ef17c7 100644 --- a/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp +++ b/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp @@ -47,6 +47,7 @@ using bts::io::ProtoPrinter; using bts::io::QueryParams; using bts::persistence::IDGenerator; using std::pair; +using std::shared_ptr; using std::string; using std::uint32_t; using std::unique_ptr; @@ -187,9 +188,10 @@ struct BinanceUsdsFuturesSubscription::PImpl { // (i.e., inspection in debugger, temp logs, etc.) BinanceUsdsFuturesMarketDataConfig cfg; SecurityCriteria criteria; + ccapi::EventDispatcher *dispatcher; // The session automatically starts/stops in its ctr/dtr CCApiSession session; - IMarketDataListener *listener; + shared_ptr listener; // These aliases are useful for quickly switching the handler types (e.g., // a special handler that logs messages for testing/diagnostic purposes) @@ -205,31 +207,29 @@ struct BinanceUsdsFuturesSubscription::PImpl { explicit PImpl( ccapi::EventDispatcher *dispatcher, BinanceUsdsFuturesMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : sec_id_store(sec_id_store), cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), - session( - dispatcher, - mk_session_cfg(this->cfg), - mk_session_opts(), - {ccapi::BtsVenues::BINANCE_USDS_FUTURES}), - listener(listener) { + dispatcher(dispatcher), session( + dispatcher, + mk_session_cfg(this->cfg), + mk_session_opts(), + {ccapi::BtsVenues::BINANCE_USDS_FUTURES}) { LOG(md, Info, "Initializing BinanceUsdsFuturesSubscription:\n{}", ProtoPrinter::to_string(this->cfg)); + } + void start(shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); // Request the list of all spots at Binance. Match the results against our SecurityCriteria, // and subscribe to md for each matching spot. - start_thread = std::thread{[this, dispatcher]() { + start_thread = std::thread{[this]() { std::string exch_info_url = this->cfg.url_rest_base(); exch_info_url.append("/fapi/v1/exchangeInfo"); bts::http::HttpResponse exch_info_res; bts::http::HttpClient::get(exch_info_url, {}, &exch_info_res); std::vector secs; - // LOG(md, Info, - // "BinanceUsdsFuturesSubscription::start_thread. Received " - // "securities.\n{}", - // exch_info_res); BinanceUsdsFuture::parse(exch_info_res.body, secs); // Exclude secs that don't satisfy the SecurityCriteria auto is_sec_excluded = [this](const Security &sec) { @@ -267,7 +267,7 @@ struct BinanceUsdsFuturesSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = - depth_handlers.try_emplace(sec_desc, sec_id, listener); + depth_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Depth handler already created for '{}'", sec_desc); return it->second; @@ -277,7 +277,7 @@ struct BinanceUsdsFuturesSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = - ticker_handlers.try_emplace(sec_desc, sec_id, listener); + ticker_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Ticker handler already created for '{}'", sec_desc); return it->second; @@ -287,7 +287,7 @@ struct BinanceUsdsFuturesSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = - stats_handlers.try_emplace(sec_desc, sec_id, listener); + stats_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Stats handler already created for '{}'", sec_desc); return it->second; @@ -365,10 +365,8 @@ struct BinanceUsdsFuturesSubscription::PImpl { BinanceUsdsFuturesSubscription::BinanceUsdsFuturesSubscription( ccapi::EventDispatcher *dispatcher, BinanceUsdsFuturesMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} BinanceUsdsFuturesSubscription::BinanceUsdsFuturesSubscription( BinanceUsdsFuturesSubscription &&) noexcept = default; @@ -377,4 +375,9 @@ BinanceUsdsFuturesSubscription &BinanceUsdsFuturesSubscription::operator=( BinanceUsdsFuturesSubscription::~BinanceUsdsFuturesSubscription() = default; +void BinanceUsdsFuturesSubscription::start( + std::shared_ptr listener) { + me_->start(listener); +} + } // namespace bts::md::binance_usds_futures diff --git a/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h b/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h index 519e56383..7486da7f9 100644 --- a/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h +++ b/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h @@ -1,5 +1,7 @@ #pragma once +#include "lib/md/IMarketDataSource.h" + #include namespace ccapi { @@ -25,13 +27,12 @@ namespace bts::md::binance_usds_futures { * * @a https://binance-docs.github.io/apidocs/futures/en/#exchange-information */ -class BinanceUsdsFuturesSubscription { +class BinanceUsdsFuturesSubscription : public IMarketDataSource { public: BinanceUsdsFuturesSubscription( ccapi::EventDispatcher *dispatcher, api::md::BinanceUsdsFuturesMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); BinanceUsdsFuturesSubscription( const BinanceUsdsFuturesSubscription &) noexcept = delete; @@ -43,8 +44,7 @@ class BinanceUsdsFuturesSubscription { ~BinanceUsdsFuturesSubscription(); - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/server/sparkd/md/ccapi/CCApiMarketDataProvider.cpp b/src/server/sparkd/md/ccapi/CCApiMarketDataProvider.cpp new file mode 100644 index 000000000..882ecdd0d --- /dev/null +++ b/src/server/sparkd/md/ccapi/CCApiMarketDataProvider.cpp @@ -0,0 +1,139 @@ +#include "CCApiMarketDataProvider.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/api/md/MarketDataConfig.pb.h" +#include "lib/config/config.h" +#include "lib/io/ProtoPrinter.h" +#include "lib/logging/logging.h" +#include "lib/md/GeneralMarketDataListener.h" +#include "lib/md/GeneralSecurityIndex.h" +#include "lib/md/MDProducer.h" +#include "lib/persistence/IDGenerator.h" +#include "server/sparkd/md/binance/BinanceSubscription.h" +#include "server/sparkd/md/binance_us/BinanceUsSubscription.h" +#include "server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h" +#include "server/sparkd/md/coinbase/CoinbaseSubscription.h" +#include "server/sparkd/md/deribit/DeribitSubscription.h" +#include "server/sparkd/md/ftx/FTXSubscription.h" +#include "server/sparkd/md/okx/OkxSubscription.h" + +#include + +namespace bts::md { + +using bts::api::md::MarketDataConfig; +using bts::persistence::IDGenerator; +using std::pair; +using std::shared_ptr; +using std::string; +using std::unique_ptr; +using std::unordered_map; +using std::vector; + +static bool validate( + const MarketDataConfig & /*cfg*/, vector & /*errs*/) { + return true; +} + +struct CCApiMarketDataProvider::PImpl { + struct VenueSink { + GeneralSecurityIndex sec_index; + MDProducer md_producer; + GeneralMarketDataListener listener; + + VenueSink(api::Venue venue, shared_ptr sec_id_store) + : sec_index(), + listener(sec_index, md_producer, venue, sec_id_store.get()) {} + }; + + MarketDataConfig config; + ccapi::EventDispatcher dispatcher; + shared_ptr sec_id_store; + unordered_map> venue_sinks; + vector>> + sources; + + PImpl(MarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} + + VenueSink *get_venue_sink(api::Venue venue) { + auto it = venue_sinks.find(venue); + if (it != venue_sinks.end()) { + return it->second.get(); + } + return venue_sinks + .try_emplace(venue, std::make_unique(venue, sec_id_store)) + .first->second.get(); + } +}; + +CCApiMarketDataProvider::CCApiMarketDataProvider( + MarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { +} + +CCApiMarketDataProvider::~CCApiMarketDataProvider() = default; + +vector>> +CCApiMarketDataProvider::get_sources() { + auto &cfg = me_->config; + vector cfg_errs; + if (!validate(cfg, cfg_errs)) { + LOG(md, Error, + "CCApiMarketDataProvider::get_sources failed. Invalid config: {}", + fmt::to_string(fmt::join(cfg_errs, "\n"))); + } + + // Binance + for (const auto &binance : cfg.binance()) { + me_->sources.emplace_back( + api::domain::VENUE_BINANCE, + std::make_shared( + &me_->dispatcher, binance, me_->sec_id_store.get())); + } + // Binance US + for (const auto &binance_us : cfg.binance_us()) { + me_->sources.emplace_back( + api::domain::VENUE_BINANCE_US, + std::make_shared( + &me_->dispatcher, binance_us, me_->sec_id_store.get())); + } + // Binance USDS Futures + for (const auto &binance_ufs : cfg.binance_usds_futures()) { + me_->sources.emplace_back( + api::domain::VENUE_BINANCE_USDS_FUTURES, + std::make_shared< + binance_usds_futures::BinanceUsdsFuturesSubscription>( + &me_->dispatcher, binance_ufs, me_->sec_id_store.get())); + } + // Coinbase + for (const auto &coinbase : cfg.coinbase()) { + me_->sources.emplace_back( + api::domain::VENUE_COINBASE, + std::make_shared( + &me_->dispatcher, coinbase, me_->sec_id_store.get())); + } + // Deribit + for (const auto &deribit : cfg.deribit()) { + me_->sources.emplace_back( + api::domain::VENUE_DERIBIT, + std::make_shared( + &me_->dispatcher, deribit, me_->sec_id_store.get())); + } + // FTX + for (const auto &ftx : cfg.ftx()) { + me_->sources.emplace_back( + api::domain::VENUE_FTX, + std::make_shared( + &me_->dispatcher, ftx, me_->sec_id_store.get())); + } + // Okx + for (const auto &okx : cfg.okx()) { + me_->sources.emplace_back( + api::domain::VENUE_OKX, + std::make_shared( + &me_->dispatcher, okx, me_->sec_id_store.get())); + } + return me_->sources; +} + +} // namespace bts::md diff --git a/src/server/sparkd/md/ccapi/CCApiMarketDataProvider.h b/src/server/sparkd/md/ccapi/CCApiMarketDataProvider.h new file mode 100644 index 000000000..34ba7fb23 --- /dev/null +++ b/src/server/sparkd/md/ccapi/CCApiMarketDataProvider.h @@ -0,0 +1,27 @@ +#pragma once + +#include "lib/api/md/MarketDataConfig.pb.h" +#include "lib/md/IMarketDataProvider.h" +#include "lib/persistence/IDGenerator.h" + +#include + +namespace bts::md { + +class CCApiMarketDataProvider : public IMarketDataProvider { + public: + explicit CCApiMarketDataProvider( + api::md::MarketDataConfig config, + std::shared_ptr sec_id_store); + ~CCApiMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/server/sparkd/md/coinbase/CoinbaseSubscription.cpp b/src/server/sparkd/md/coinbase/CoinbaseSubscription.cpp index 6e9ceeb9a..cba166c29 100644 --- a/src/server/sparkd/md/coinbase/CoinbaseSubscription.cpp +++ b/src/server/sparkd/md/coinbase/CoinbaseSubscription.cpp @@ -5,7 +5,9 @@ #include "lib/domain/SecurityCriteria.h" #include "lib/io/ProtoPrinter.h" #include "lib/io/QueryParams.h" +#include "lib/md/IMarketDataListener.h" #include "lib/persistence/IDGenerator.h" +#include "lib/utils/bts_assert.h" #include "server/sparkd/md/ccapi/CCApiMarketDepthAdaptor.h" #include "server/sparkd/md/ccapi/CCApiSession.h" #include "server/sparkd/md/ccapi/CCApiTradeAdaptor.h" @@ -35,6 +37,7 @@ using bts::io::ProtoPrinter; using bts::io::QueryParams; using bts::persistence::IDGenerator; using std::pair; +using std::shared_ptr; using std::string; using std::unique_ptr; using std::unordered_map; @@ -150,7 +153,7 @@ struct CoinbaseSubscription::PImpl { // The session automatically starts/stops in its ctr/dtr CCApiSession session; - IMarketDataListener *listener; + shared_ptr listener; CoinbaseSecurityRequestor sec_requestor; // These aliases are useful for quickly switching the handler types (e.g., @@ -171,8 +174,7 @@ struct CoinbaseSubscription::PImpl { explicit PImpl( ccapi::EventDispatcher *dispatcher, CoinbaseMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : sec_id_store(sec_id_store), cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), session( @@ -180,7 +182,6 @@ struct CoinbaseSubscription::PImpl { mk_session_cfg(this->cfg), {}, {ccapi::BtsVenues::COINBASE}), - listener(listener), sec_requestor( &session, this->criteria, @@ -191,6 +192,11 @@ struct CoinbaseSubscription::PImpl { LOG(md, Info, "Initializing CoinbaseSubscription:\n{}", ProtoPrinter::to_string(this->cfg)); sec_requestor.callback().bind<&PImpl::on_securities>(*this); + } + + void start(shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); sec_requestor.request(); } @@ -199,7 +205,7 @@ struct CoinbaseSubscription::PImpl { auto sec_id = SecurityID{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - stats_handlers.try_emplace(sec_desc, sec_id, listener); + stats_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "StatsHandler already created with id '{}'", sec_id); return it->second; @@ -208,7 +214,7 @@ struct CoinbaseSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - depth_handlers.try_emplace(sec_desc, sec_id, listener); + depth_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert(is_new); return it->second; } @@ -217,7 +223,7 @@ struct CoinbaseSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - ticker_handlers.try_emplace(sec_desc, sec_id, listener); + ticker_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Ticker handler already created for '{}'", sec_desc); return it->second; @@ -289,10 +295,8 @@ struct CoinbaseSubscription::PImpl { CoinbaseSubscription::CoinbaseSubscription( ccapi::EventDispatcher *dispatcher, CoinbaseMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} CoinbaseSubscription::CoinbaseSubscription(CoinbaseSubscription &&) noexcept = default; @@ -301,4 +305,8 @@ CoinbaseSubscription &CoinbaseSubscription::operator=( CoinbaseSubscription::~CoinbaseSubscription() = default; +void CoinbaseSubscription::start(shared_ptr listener) { + me_->start(std::move(listener)); +} + } // namespace bts::md::coinbase diff --git a/src/server/sparkd/md/coinbase/CoinbaseSubscription.h b/src/server/sparkd/md/coinbase/CoinbaseSubscription.h index 999a55cab..c8e76ea01 100644 --- a/src/server/sparkd/md/coinbase/CoinbaseSubscription.h +++ b/src/server/sparkd/md/coinbase/CoinbaseSubscription.h @@ -1,5 +1,7 @@ #pragma once +#include "lib/md/IMarketDataSource.h" + #include namespace ccapi { @@ -20,13 +22,12 @@ class IDGenerator; namespace bts::md::coinbase { -class CoinbaseSubscription { +class CoinbaseSubscription : public IMarketDataSource { public: CoinbaseSubscription( ccapi::EventDispatcher *dispatcher, api::md::CoinbaseMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); CoinbaseSubscription(const CoinbaseSubscription &) noexcept = delete; CoinbaseSubscription(CoinbaseSubscription &&) noexcept; @@ -36,8 +37,7 @@ class CoinbaseSubscription { ~CoinbaseSubscription(); - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/server/sparkd/md/databento/DatabentoMarketDataProvider.cpp b/src/server/sparkd/md/databento/DatabentoMarketDataProvider.cpp new file mode 100644 index 000000000..66c2d1294 --- /dev/null +++ b/src/server/sparkd/md/databento/DatabentoMarketDataProvider.cpp @@ -0,0 +1,52 @@ +#include "DatabentoMarketDataProvider.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/api/md/MarketDataConfig.pb.h" +#include "lib/md/IMarketDataSource.h" +#include "lib/persistence/IDGenerator.h" +#include "server/sparkd/md/databento/DatabentoDataHandler.h" +#include "server/sparkd/md/glbx/DatabentoGlbxSubscription.h" + +#include +#include + +namespace bts::md { + +using bts::api::md::MarketDataConfig; +using bts::persistence::IDGenerator; +using std::pair; +using std::shared_ptr; +using std::string; +using std::vector; + +struct DatabentoMarketDataProvider::PImpl { + MarketDataConfig config; + shared_ptr sec_id_store; + vector> glbx_sources; + + PImpl(MarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} +}; + +DatabentoMarketDataProvider::DatabentoMarketDataProvider( + MarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { + me_->config = std::move(config); +} + +DatabentoMarketDataProvider::~DatabentoMarketDataProvider() = default; + +vector>> +DatabentoMarketDataProvider::get_sources() { + vector>> rv; + for (const auto &glbx : me_->config.glbx()) { + if (glbx.use_databento()) { + me_->glbx_sources.push_back( + std::make_shared( + glbx, me_->sec_id_store)); + rv.push_back({api::domain::VENUE_GLBX, me_->glbx_sources.back()}); + } + } + return rv; +} + +} // namespace bts::md diff --git a/src/server/sparkd/md/databento/DatabentoMarketDataProvider.h b/src/server/sparkd/md/databento/DatabentoMarketDataProvider.h new file mode 100644 index 000000000..41275ce30 --- /dev/null +++ b/src/server/sparkd/md/databento/DatabentoMarketDataProvider.h @@ -0,0 +1,33 @@ +#pragma once + +#include "lib/md/IMarketDataProvider.h" + +#include + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::api::md { +class MarketDataConfig; +} + +namespace bts::md { + +class DatabentoMarketDataProvider : public IMarketDataProvider { + public: + explicit DatabentoMarketDataProvider( + api::md::MarketDataConfig config, + std::shared_ptr sec_id_store); + ~DatabentoMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/server/sparkd/md/deribit/DeribitSubscription.cpp b/src/server/sparkd/md/deribit/DeribitSubscription.cpp index bbfe0008d..7619c7249 100644 --- a/src/server/sparkd/md/deribit/DeribitSubscription.cpp +++ b/src/server/sparkd/md/deribit/DeribitSubscription.cpp @@ -52,6 +52,7 @@ using bts::io::websocket::JsonRpcId; using bts::io::websocket::JsonRpcRequest; using bts::persistence::IDGenerator; using nlohmann::json; +using std::shared_ptr; using std::string; using std::uint32_t; using std::unique_ptr; @@ -134,14 +135,13 @@ struct DeribitSubscription::PImpl { const bool subscribe_trades = true; size_t sub_id_counter = 0; CCApiSession session; - IMarketDataListener *listener; + shared_ptr listener; DeribitChannelDemuxer channel_demuxer; explicit PImpl( ccapi::EventDispatcher *dispatcher, DeribitMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), excluded_secs( @@ -163,11 +163,15 @@ struct DeribitSubscription::PImpl { dispatcher, mk_session_cfg(this->cfg), {}, - {ccapi::BtsVenues::DERIBIT}), - listener(listener) { + {ccapi::BtsVenues::DERIBIT}) { // DON'T reference the ctr arg 'cfg' (it has been moved to 'this->cfg') _INFO("ctr. config:\n{}", ProtoPrinter::to_string(this->cfg)); sec_requestor.callback().bind<&PImpl::on_securities>(*this); + } + + void start(std::shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); sec_requestor.request_all(); subscribe_instrument_state(); } @@ -176,7 +180,7 @@ struct DeribitSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = security_data_map.try_emplace( - sec_desc, std::make_unique(sid, listener)); + sec_desc, std::make_unique(sid, listener.get())); _ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc); _INFO("add_security_data. sec: '{}'", sec_desc); return *it->second; @@ -307,10 +311,8 @@ struct DeribitSubscription::PImpl { DeribitSubscription::DeribitSubscription( ccapi::EventDispatcher *dispatcher, DeribitMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} DeribitSubscription::DeribitSubscription(DeribitSubscription &&) noexcept = default; @@ -319,4 +321,8 @@ DeribitSubscription &DeribitSubscription::operator=( DeribitSubscription::~DeribitSubscription() = default; +void DeribitSubscription::start(shared_ptr listener) { + me_->start(std::move(listener)); +} + } // namespace bts::md::deribit diff --git a/src/server/sparkd/md/deribit/DeribitSubscription.h b/src/server/sparkd/md/deribit/DeribitSubscription.h index c8f41ec6e..aeabe6cca 100644 --- a/src/server/sparkd/md/deribit/DeribitSubscription.h +++ b/src/server/sparkd/md/deribit/DeribitSubscription.h @@ -1,5 +1,7 @@ #pragma once +#include "lib/md/IMarketDataSource.h" + #include namespace ccapi { @@ -24,13 +26,12 @@ namespace bts::md::deribit { * @brief Subscribes to market data at Deribit * @a https://docs.deribit.com/?#market-data */ -class DeribitSubscription { +class DeribitSubscription : public IMarketDataSource { public: DeribitSubscription( ccapi::EventDispatcher *dispatcher, api::md::DeribitMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); DeribitSubscription(const DeribitSubscription &) noexcept = delete; DeribitSubscription(DeribitSubscription &&) noexcept; @@ -40,9 +41,8 @@ class DeribitSubscription { ~DeribitSubscription(); - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); - + void start(std::shared_ptr listener) override; + private: struct PImpl; std::unique_ptr me_; diff --git a/src/server/sparkd/md/emumba/EmumbaDeribitSubscription.cpp b/src/server/sparkd/md/emumba/EmumbaDeribitSubscription.cpp new file mode 100644 index 000000000..5facdd62e --- /dev/null +++ b/src/server/sparkd/md/emumba/EmumbaDeribitSubscription.cpp @@ -0,0 +1,78 @@ +#include "EmumbaDeribitSubscription.h" +#include "lib/api/md/EmumbaDeribitMarketDataConfig.pb.h" +#include "lib/config/protos/system_config.pb.h" +#include "lib/utils/thread_manager.h" +#include "server/sparkd/md/emumba/EmumbaDeribitData.h" + +namespace bts::md { + +using bts::config::SystemConfig; + +using Path = boost::filesystem::path; + +struct EmumbaDeribitSubscription::PImpl { + api::md::EmumbaDeribitMarketDataConfig config; +}; + +EmumbaDeribitSubscription::EmumbaDeribitSubscription( + api::md::EmumbaDeribitMarketDataConfig config) + : me_(std::make_unique()) { + me_->config = std::move(config); +} + +EmumbaDeribitSubscription::~EmumbaDeribitSubscription() = default; + +void EmumbaDeribitSubscription::start( + std::shared_ptr /*listener*/) { + auto &cfg = me_->config; + LOG(md, Info, "EmumbaMarketDataProvider::start t_id:{} t_name:{}. cfg:\n{}", + os::get_thread_id(), os::get_thread_name(), cfg.DebugString()); + auto start_deribit_thr = [=]() { + const auto config_path = Path{config::get().config_path()} / + cfg.json_conf_file_name(); + const auto conf = + bts::cmds::DeribitDemoConfiguration::read_file(config_path.string()); + bts_assert_msg(conf.has_value(), "Failed to parse deribit config"); + // log + for (const auto &s : conf.value().symbol_config_list) { + LOG(md, Info, "sub.symbol: {}", s.symbol_name); + LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future, + s.interested_in_option); + for (const auto &i : s.interested_instruments) { + LOG(md, Info, "sub.instrument: {}", i); + } + } + EmumbaDeribitData ddata_obj; + EmumbaDeribitData *ddata = &ddata_obj; + // end log + bts_assert(conf.has_value()); + LOG(md, Info, "Deribit conf: {}", conf.value()); + ddata->real_handler = + std::make_shared( + conf.value().symbol_config_list, conf.value().trading_config, + conf.value().credential_map, conf.value().logging_configuration, + conf.value().deribit_url, &ddata->listener); + // since the trading interface is housed inside the EmMDListener, we need to + // create and send emumba's trading_session_interface + for (const auto &p : conf.value().credential_map) { + const auto &sess = p.second; + auto ts_ptr = std::make_shared< + emumba::deribit_connector::trading_session_interface>( + ddata->real_handler->create_new_trading_session(sess)); + ddata->trading_session_vec.push_back({sess, ts_ptr}); + } + bts_assert(!ddata->trading_session_vec.empty()); + ddata->listener.set_handler( + ddata->real_handler.get(), ddata->trading_session_vec); + ddata->real_handler->run(); + }; + threading::make_thread("Emumba", [&]() { start_deribit_thr(); }).detach(); + LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}", + os::get_thread_id(), os::get_thread_name()); +} + +void EmumbaDeribitSubscription::stop() { + // +} + +} // namespace bts::md diff --git a/src/server/sparkd/md/emumba/EmumbaDeribitSubscription.h b/src/server/sparkd/md/emumba/EmumbaDeribitSubscription.h new file mode 100644 index 000000000..133697e58 --- /dev/null +++ b/src/server/sparkd/md/emumba/EmumbaDeribitSubscription.h @@ -0,0 +1,24 @@ +#pragma once + +#include "lib/api/md/EmumbaDeribitMarketDataConfig.pb.h" +#include "lib/md/IMarketDataSource.h" + +#include + +namespace bts::md { + +class EmumbaDeribitSubscription : public IMarketDataSource { + public: + explicit EmumbaDeribitSubscription( + api::md::EmumbaDeribitMarketDataConfig config); + ~EmumbaDeribitSubscription() override; + + void start(std::shared_ptr listener) override; + void stop() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/server/sparkd/md/emumba/EmumbaMarketDataProvider.cpp b/src/server/sparkd/md/emumba/EmumbaMarketDataProvider.cpp new file mode 100644 index 000000000..22d8bae61 --- /dev/null +++ b/src/server/sparkd/md/emumba/EmumbaMarketDataProvider.cpp @@ -0,0 +1,40 @@ +#include "EmumbaMarketDataProvider.h" +#include "lib/api/md/MarketDataConfig.pb.h" +#include "lib/api/md/MarketDataConfigUtils.h" +#include "lib/md/IMarketDataSource.h" +#include "server/sparkd/md/emumba/EmumbaDeribitSubscription.h" + +#include +#include + +namespace bts::md { + +using bts::api::md::MarketDataConfig; +using std::pair; +using std::shared_ptr; +using std::string; +using std::vector; + +struct EmumbaMarketDataProvider::PImpl { + MarketDataConfig config; + vector>> deribit_subs; + + PImpl(MarketDataConfig config) : config(std::move(config)) {} +}; + +EmumbaMarketDataProvider::EmumbaMarketDataProvider(MarketDataConfig config) + : me_(std::make_unique(std::move(config))) {} + +EmumbaMarketDataProvider::~EmumbaMarketDataProvider() = default; + +vector>> +EmumbaMarketDataProvider::get_sources() { + for (const auto &cfg : me_->config.emumba_deribit()) { + me_->deribit_subs.push_back( + {api::domain::VENUE_DERIBIT, + std::make_shared(cfg)}); + } + return me_->deribit_subs; +} + +} // namespace bts::md diff --git a/src/server/sparkd/md/emumba/EmumbaMarketDataProvider.h b/src/server/sparkd/md/emumba/EmumbaMarketDataProvider.h new file mode 100644 index 000000000..b49f93fa2 --- /dev/null +++ b/src/server/sparkd/md/emumba/EmumbaMarketDataProvider.h @@ -0,0 +1,25 @@ +#pragma once + +#include "lib/api/md/MarketDataConfig.pb.h" +#include "lib/md/IMarketDataProvider.h" + +#include + +namespace bts::md { + +class EmumbaMarketDataProvider : public IMarketDataProvider { + public: + EmumbaMarketDataProvider(); + explicit EmumbaMarketDataProvider(api::md::MarketDataConfig config); + ~EmumbaMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/server/sparkd/md/ftx/FTXSubscription.cpp b/src/server/sparkd/md/ftx/FTXSubscription.cpp index ba95e25fa..0a29419dc 100644 --- a/src/server/sparkd/md/ftx/FTXSubscription.cpp +++ b/src/server/sparkd/md/ftx/FTXSubscription.cpp @@ -39,6 +39,7 @@ using bts::domain::SecurityID; using bts::io::ProtoPrinter; using bts::persistence::IDGenerator; using std::pair; +using std::shared_ptr; using std::string; using std::unique_ptr; using std::unordered_map; @@ -106,7 +107,7 @@ struct FTXSubscription::PImpl { FTXSecurityRequestor sec_requestor; // The session automatically starts/stops in its ctr/dtr CCApiSession session; - IMarketDataListener *listener; + shared_ptr listener; // These aliases are useful for quickly switching the handler types (e.g., // a special handler that logs messages for testing/diagnostic purposes) @@ -121,18 +122,21 @@ struct FTXSubscription::PImpl { explicit PImpl( ccapi::EventDispatcher *dispatcher, FTXMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener) + persistence::IDGenerator *sec_id_store) : sec_id_store(sec_id_store), cfg(std::move(cfg)), criteria(domain::SecurityCriteria::from_proto( this->cfg.security_criteria())), sec_requestor(dispatcher, criteria, sec_id_store), session( - dispatcher, ccapi::SessionConfigs{}, {}, {ccapi::BtsVenues::FTX}), - listener(listener) { + dispatcher, ccapi::SessionConfigs{}, {}, {ccapi::BtsVenues::FTX}) { LOG(md, Info, "Initializing FTXSubscription:\n{}", ProtoPrinter::to_string(this->cfg)); sec_requestor.callback().bind<&PImpl::on_securities>(*this); + } + + void start(shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); sec_requestor.request(); } @@ -140,7 +144,7 @@ struct FTXSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - depth_handlers.try_emplace(sec_desc, sec_id, listener); + depth_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert(is_new); return it->second; } @@ -149,7 +153,7 @@ struct FTXSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - ticker_handlers.try_emplace(sec_desc, sec_id, listener); + ticker_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Ticker handler already created for '{}'", sec_desc); return it->second; @@ -159,7 +163,7 @@ struct FTXSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - stats_handlers.try_emplace(sec_desc, sec_id, listener); + stats_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Statistics handler already created for '{}'", sec_desc); return it->second; @@ -231,10 +235,8 @@ struct FTXSubscription::PImpl { FTXSubscription::FTXSubscription( ccapi::EventDispatcher *dispatcher, FTXMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} FTXSubscription::FTXSubscription(FTXSubscription &&) noexcept = default; FTXSubscription &FTXSubscription::operator=(FTXSubscription &&) noexcept = @@ -242,4 +244,8 @@ FTXSubscription &FTXSubscription::operator=(FTXSubscription &&) noexcept = FTXSubscription::~FTXSubscription() = default; +void FTXSubscription::start(shared_ptr listener) { + me_->start(std::move(listener)); +} + } // namespace bts::md::ftx diff --git a/src/server/sparkd/md/ftx/FTXSubscription.h b/src/server/sparkd/md/ftx/FTXSubscription.h index a9d3d58cd..88a0b03e8 100644 --- a/src/server/sparkd/md/ftx/FTXSubscription.h +++ b/src/server/sparkd/md/ftx/FTXSubscription.h @@ -1,5 +1,7 @@ #pragma once +#include "lib/md/IMarketDataSource.h" + #include namespace ccapi { @@ -25,20 +27,21 @@ namespace bts::md::ftx { * * @details Uses ccapi infrastructure to request/subscribe data */ -class FTXSubscription { +class FTXSubscription : public IMarketDataSource { public: FTXSubscription( ccapi::EventDispatcher *dispatcher, api::md::FTXMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); FTXSubscription(const FTXSubscription &) noexcept = delete; FTXSubscription(FTXSubscription &&) noexcept; FTXSubscription &operator=(const FTXSubscription &) noexcept = delete; FTXSubscription &operator=(FTXSubscription &&) noexcept; - ~FTXSubscription(); + ~FTXSubscription() override; + + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/server/sparkd/md/glbx/DatabentoGlbxSubscription.cpp b/src/server/sparkd/md/glbx/DatabentoGlbxSubscription.cpp new file mode 100644 index 000000000..b18818b98 --- /dev/null +++ b/src/server/sparkd/md/glbx/DatabentoGlbxSubscription.cpp @@ -0,0 +1,41 @@ +#include "DatabentoGlbxSubscription.h" +#include "lib/api/md/GlbxMarketDataConfig.pb.h" +#include "lib/utils/thread_manager.h" +#include "server/sparkd/md/databento/DatabentoDataHandler.h" +#include "server/sparkd/md/glbx/GlbxMarketDataHandler.h" + +namespace bts::md { + +using bts::api::md::GlbxMarketDataConfig; +using bts::persistence::IDGenerator; +using std::shared_ptr; + +struct DatabentoGlbxSubscription::PImpl { + GlbxMarketDataConfig config; + shared_ptr sec_id_store; + + PImpl(GlbxMarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} +}; + +DatabentoGlbxSubscription::DatabentoGlbxSubscription( + GlbxMarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { +} + +DatabentoGlbxSubscription::~DatabentoGlbxSubscription() = default; + +void DatabentoGlbxSubscription::start( + std::shared_ptr listener) { + bts::threading::make_thread( + "glbx", + [config = me_->config, sec_id_store = me_->sec_id_store, + listener = std::move(listener)]() { + api::md::glbx::GlbxMarketDatahandler::run( + std::move(listener), std::move(sec_id_store), config); + }) + .detach(); + // TODO: implement 'stop' (signal the thread above to terminate, then join it). +} + +} // namespace bts::md diff --git a/src/server/sparkd/md/glbx/DatabentoGlbxSubscription.h b/src/server/sparkd/md/glbx/DatabentoGlbxSubscription.h new file mode 100644 index 000000000..b28755418 --- /dev/null +++ b/src/server/sparkd/md/glbx/DatabentoGlbxSubscription.h @@ -0,0 +1,47 @@ +#pragma once + +#include "lib/md/IMarketDataSource.h" + +#include + +namespace bts::api::md { +class GlbxMarketDataConfig; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::md { + +/** + * @brief Subscribes to market data at Glbx via Databento + * + * @a https://www.Databento.biz/ + */ +class DatabentoGlbxSubscription : public IMarketDataSource { + public: + DatabentoGlbxSubscription( + api::md::GlbxMarketDataConfig config, + std::shared_ptr sec_id_store); + DatabentoGlbxSubscription(const DatabentoGlbxSubscription &) noexcept = + delete; + DatabentoGlbxSubscription(DatabentoGlbxSubscription &&) noexcept; + DatabentoGlbxSubscription &operator=( + const DatabentoGlbxSubscription &) noexcept = delete; + DatabentoGlbxSubscription &operator=(DatabentoGlbxSubscription &&) noexcept; + ~DatabentoGlbxSubscription() override; + + /** + * Starts market data subscription on a background thread. + * + * @note Does not block. + */ + void start(std::shared_ptr listener) override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/server/sparkd/md/glbx/OnixsGlbxSubscription.cpp b/src/server/sparkd/md/glbx/OnixsGlbxSubscription.cpp new file mode 100644 index 000000000..0f69ed0c1 --- /dev/null +++ b/src/server/sparkd/md/glbx/OnixsGlbxSubscription.cpp @@ -0,0 +1,118 @@ +#include "OnixsGlbxSubscription.h" +#include "lib/api/md/GlbxMarketDataConfig.pb.h" +#include "lib/core/LOCK_GUARD.h" +#include "lib/logging/logging.h" +#include "lib/md/IMarketDataListener.h" +#include "lib/md/SymbolLoader.h" +#include "lib/os/os.h" +#include "lib/persistence/IDGenerator.h" +#include "lib/utils/Dispatcher.h" +#include "server/sparkd/md/platforms/IMDPlatform.h" +#include "server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h" + +#include +#include +#include +#include + +namespace bts::md { + +using bts::api::md::GlbxMarketDataConfig; +using bts::domain::SecurityCriteria; +using bts::md::IMDPlatform; +using bts::persistence::IDGenerator; +using std::shared_ptr; +using std::string; +using std::vector; + +struct OnixsGlbxSubscription::PImpl { + GlbxMarketDataConfig config; + shared_ptr sec_id_store; + + PImpl(shared_ptr sec_id_store, GlbxMarketDataConfig config) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} + + std::thread dispatcher_thread; + utils::Dispatcher *dispatcher{nullptr}; + std::optional heartbeat_timer; + bool is_running = false; + std::mutex mx; + + ~PImpl() { stop(); } + + /** + * Starts market data subscription on this thread. + * + * @note Designed to be called from a thread callback inside `OnixsGlbxSubscription::start`. + * @note Blocks until subscription stops. + */ + void run() { + { + BTS_LOCK_GUARD(mx); + if (!is_running) { + return; + } + const auto symbols = SymbolLoader::load( + SecurityCriteria::create({api::domain::VENUE_GLBX})); + if (symbols.empty()) { + LOG(md, Info, "OnixsGlbxSubscription aborting. No symbols loaded"); + return; + } + LOG(md, Info, + "OnixsGlbxSubscription starting. t_id: {}; t_name: {}; config: " + "{}; symbols: {}", + os::get_thread_id(), os::get_thread_name(), config.DebugString(), + FMT_JOIN(symbols, ", ")); + vector> platforms; + auto main = shared_ptr(); + main->start({symbols.begin(), symbols.end()}, nullptr, nullptr); + platforms.emplace_back(std::move(main)); + dispatcher = &utils::Dispatcher::current(); + heartbeat_timer.emplace(*dispatcher, 250, [&]() { + for (auto &p : platforms) { + p->send_heartbeats(); + } + }); + heartbeat_timer->start(); + is_running = true; + } + // This will block until dispatcher->terminate() is called. + // There is a slim, but non-zero chance that this is executed after + // 'stop' is called, since we 'run' outside of the lock. + dispatcher->run(); + } + + void stop() { + BTS_LOCK_GUARD(mx); + if (!is_running) { + return; + } + is_running = false; + if (dispatcher != nullptr) { + LOG(md, Diag, "OnixsGlbxSubscription stopping."); + dispatcher->io_service()->stop(); + } + if (dispatcher_thread.joinable()) { + LOG(md, Diag, "OnixsGlbxSubscription joining dispatcher thread."); + dispatcher_thread.join(); + } + } +}; + +OnixsGlbxSubscription::OnixsGlbxSubscription( + shared_ptr sec_id_store, GlbxMarketDataConfig config) + : me_(std::make_unique(std::move(sec_id_store), std::move(config))) { +} + +OnixsGlbxSubscription::~OnixsGlbxSubscription() = default; + +void OnixsGlbxSubscription::start( + shared_ptr /*listener*/) { + std::atomic_thread_fence(std::memory_order_release); + bts_assert(!me_->dispatcher_thread.joinable()); + me_->dispatcher_thread = std::thread{[&]() { me_->run(); }}; +} + +void OnixsGlbxSubscription::stop() { me_->stop(); } + +} // namespace bts::md diff --git a/src/server/sparkd/md/glbx/OnixsGlbxSubscription.h b/src/server/sparkd/md/glbx/OnixsGlbxSubscription.h new file mode 100644 index 000000000..70e6a8e3f --- /dev/null +++ b/src/server/sparkd/md/glbx/OnixsGlbxSubscription.h @@ -0,0 +1,48 @@ +#pragma once + +#include "lib/md/IMarketDataSource.h" + +#include + +namespace bts::api::md { +class GlbxMarketDataConfig; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::md { + +/** + * @brief Subscribes to market data at Glbx via Onixs + * + * @a https://www.onixs.biz/ + */ +class OnixsGlbxSubscription : public IMarketDataSource { + public: + OnixsGlbxSubscription( + std::shared_ptr sec_id_store, + api::md::GlbxMarketDataConfig config); + OnixsGlbxSubscription(const OnixsGlbxSubscription &) noexcept = delete; + OnixsGlbxSubscription(OnixsGlbxSubscription &&) noexcept; + OnixsGlbxSubscription &operator=(const OnixsGlbxSubscription &) noexcept = + delete; + OnixsGlbxSubscription &operator=(OnixsGlbxSubscription &&) noexcept; + ~OnixsGlbxSubscription() override; + + /** + * Starts market data subscription on a background thread. + * + * @note Does not block. + */ + void start(std::shared_ptr listener) override; + + void stop() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/server/sparkd/md/okx/OkxSubscription.cpp b/src/server/sparkd/md/okx/OkxSubscription.cpp index aa02de00c..bcd1e5edf 100644 --- a/src/server/sparkd/md/okx/OkxSubscription.cpp +++ b/src/server/sparkd/md/okx/OkxSubscription.cpp @@ -47,6 +47,7 @@ using bts::domain::SecurityID; using bts::io::ProtoPrinter; using bts::persistence::IDGenerator; using std::optional; +using std::shared_ptr; using std::size_t; using std::string; using std::unique_ptr; @@ -137,14 +138,13 @@ struct OkxSubscription::PImpl { const bool subscribe_trades = true; size_t sub_id_counter = 0; CCApiSession session; - IMarketDataListener *listener; + shared_ptr listener; OkxChannelDemuxer channel_demuxer; explicit PImpl( ccapi::EventDispatcher *dispatcher, OkxMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), excluded_secs( @@ -161,10 +161,14 @@ struct OkxSubscription::PImpl { dispatcher, mk_session_cfg(this->cfg), mk_session_opts(this->cfg), - {ccapi::BtsVenues::OKX}), - listener(listener) { + {ccapi::BtsVenues::OKX}) { // DON'T reference the ctr arg 'cfg' (it has been moved to 'this->cfg') _INFO("ctr. config:\n{}", ProtoPrinter::to_string(this->cfg)); + } + + void start(std::shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = listener; sec_sub.emplace( &session, &instrument_store, this->criteria, [this](const auto &secs) { on_securities(secs); }); @@ -174,7 +178,7 @@ struct OkxSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = security_data_map.try_emplace( - sec_desc, std::make_unique(sid, listener)); + sec_desc, std::make_unique(sid, listener.get())); _ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc); _INFO("add_security_data. sec: '{}'", sec_desc); return *it->second; @@ -257,10 +261,8 @@ struct OkxSubscription::PImpl { OkxSubscription::OkxSubscription( ccapi::EventDispatcher *dispatcher, OkxMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} OkxSubscription::OkxSubscription(OkxSubscription &&) noexcept = default; OkxSubscription &OkxSubscription::operator=(OkxSubscription &&) noexcept = @@ -268,4 +270,9 @@ OkxSubscription &OkxSubscription::operator=(OkxSubscription &&) noexcept = OkxSubscription::~OkxSubscription() = default; +void OkxSubscription::start( + std::shared_ptr listener) { + me_->start(listener); +} + } // namespace bts::md::okx diff --git a/src/server/sparkd/md/okx/OkxSubscription.h b/src/server/sparkd/md/okx/OkxSubscription.h index df5128c3c..9842b3f8c 100644 --- a/src/server/sparkd/md/okx/OkxSubscription.h +++ b/src/server/sparkd/md/okx/OkxSubscription.h @@ -1,5 +1,7 @@ #pragma once +#include "lib/md/IMarketDataSource.h" + #include namespace ccapi { @@ -25,25 +27,22 @@ namespace bts::md::okx { * * @a https://www.okx.com/docs-v5/en/#websocket-api-public-channel */ -class OkxSubscription { +class OkxSubscription : public IMarketDataSource { public: OkxSubscription( ccapi::EventDispatcher *dispatcher, api::md::OkxMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); OkxSubscription(const OkxSubscription &) noexcept = delete; OkxSubscription(OkxSubscription &&) noexcept; - OkxSubscription &operator=(const OkxSubscription &) noexcept = - delete; + OkxSubscription &operator=(const OkxSubscription &) noexcept = delete; OkxSubscription &operator=(OkxSubscription &&) noexcept; ~OkxSubscription(); - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); - + void start(std::shared_ptr listener) override; + private: struct PImpl; std::unique_ptr me_; diff --git a/src/server/sparkd/md/recording/MDReplayer.cpp b/src/server/sparkd/md/recording/MDReplayer.cpp index 776a2414f..9c5abe249 100644 --- a/src/server/sparkd/md/recording/MDReplayer.cpp +++ b/src/server/sparkd/md/recording/MDReplayer.cpp @@ -1,10 +1,10 @@ #include "MDReplayer.h" #include "lib/core/TimeStamp.h" #include "lib/domain/Book.h" -#include "lib/domain/DomainSecurityUtils.h" #include "lib/domain/Price.h" #include "lib/domain/Quantity.h" #include "lib/domain/Security.h" +#include "lib/domain/SecurityCommon.h" #include "lib/domain/SecurityID.h" #include "lib/domain/SecurityMarketDataStatistics.h" #include "lib/domain/Side.h" @@ -12,10 +12,11 @@ #include "lib/domain_model/Security.pb.h" #include "lib/logging/logging.h" #include "lib/md/GeneralMarketDataListener.h" +#include "lib/md/IMarketDataListener.h" #include "lib/md/MDProducer.h" -#include "lib/md/Shm/FixedSecurity.h" #include "lib/md/protos/co2.pb.h" #include "lib/os/os.h" +#include "lib/persistence/IDGenerator.h" #include "lib/utils/PBUtils.h" #include "lib/utils/bts_assert.h" #include "server/sparkd/md/recording/MDRecorder.h" @@ -55,38 +56,27 @@ struct MDReplayer ::PImpl { std::map books = {}; bts::api::Venue venue; std::set sec_keys = {}; - int64_t start_time = bts::core::TimeStamp::now_ns(); - GeneralMarketDataListener listener; + uint64_t start_time = 0; + std::shared_ptr listener{nullptr}; bool publish_boostrap_books_only; double replay_speed; - PImpl( - bts::api::Venue venue, - bts::md::GeneralSecurityIndex &index, - bts::md::MDProducer &md_producer, - bool publish_boostrap_books_only_, - double replay_speed) - : venue(venue), listener(index, md_producer, venue), - publish_boostrap_books_only(publish_boostrap_books_only_), - replay_speed(replay_speed) {} + + PImpl(bts::api::Venue venue, double replay_speed) + : venue(venue), replay_speed(replay_speed) {} }; -MDReplayer::MDReplayer( - bts::api::Venue venue, - bts::md::GeneralSecurityIndex &index, - bts::md::MDProducer &md_producer, - bool publish_boostrap_books_only, - double replay_speed) - : me(new PImpl( - venue, - index, - md_producer, - publish_boostrap_books_only, - replay_speed)) { + +MDReplayer::MDReplayer(bts::api::Venue venue, double replay_speed) + : me(new PImpl(venue, replay_speed)) { LOG(md, Info, "MDReplayer::MDReplayer venue: {}", Venue_Name(venue)); } MDReplayer::~MDReplayer() {} -void MDReplayer::run() { +void MDReplayer::start(std::shared_ptr listener) { + LOG(md, Info, "MDReplayer::start venue: {}", Venue_Name(me->venue)); + bts_assert(listener != nullptr); + me->listener = std::move(listener); + me->start_time = bts::core::TimeStamp::now_ns(); std::string directory = MDRecorder::recording_path(); auto include_file = [&](const std::string &file_name) { return MDRecorder::is_md_recording_file(file_name, me->venue); @@ -179,7 +169,7 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) { me->sec_keys.insert(m.security().id()); std::vector sec_vec = {}; sec_vec.push_back(&new_sec); - me->listener.on_securities(sec_vec); + me->listener->on_securities(sec_vec); break; } case (co2::RecordedMDMessage::MDRecordCase::kTotalBook): { @@ -194,19 +184,19 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) { bts::domain::Book book = {}; copy_books(tot_book.direct_book(), book); book.set_security_id(bts::domain::SecurityID(m.sec_key())); - me->listener.on_book(book); + me->listener->on_book(book); } break; } case (co2::RecordedMDMessage::MDRecordCase::kMisc): { bts::domain::SecurityMarketDataStatistics stats(m.sec_key(), m.misc()); - me->listener.on_security_statistics(stats); + me->listener->on_security_statistics(stats); break; } case (co2::RecordedMDMessage::MDRecordCase::kTrade): { domain::TradeRecord tr; domain::DomainSecurityUtils::convert(m.trade(), tr); - me->listener.on_trades({&tr}); + me->listener->on_trades({&tr}); break; } default: { diff --git a/src/server/sparkd/md/recording/MDReplayer.h b/src/server/sparkd/md/recording/MDReplayer.h index b678e770e..e88540ad1 100644 --- a/src/server/sparkd/md/recording/MDReplayer.h +++ b/src/server/sparkd/md/recording/MDReplayer.h @@ -1,7 +1,8 @@ #pragma once #include "lib/domain_model/Security.pb.h" -#include "lib/md/GeneralSecurityIndex.h" +#include "lib/md/IMarketDataListener.h" +#include "lib/md/IMarketDataSource.h" namespace bts::md { class MDProducer; @@ -14,16 +15,12 @@ class RecordedMDMessage; namespace bts::md::recording { -class MDReplayer { +class MDReplayer : public IMarketDataSource { public: - MDReplayer( - bts::api::Venue venue, - bts::md::GeneralSecurityIndex &index, - bts::md::MDProducer &md_producer, - bool publish_boostrap_books_only, - double replay_speed = 1); - ~MDReplayer(); - void run(); + explicit MDReplayer(bts::api::Venue venue, double replay_speed = 1); + ~MDReplayer() override; + + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/server/sparkd/md/taifex/TaifexMarketDataProvider.cpp b/src/server/sparkd/md/taifex/TaifexMarketDataProvider.cpp new file mode 100644 index 000000000..1ad70c088 --- /dev/null +++ b/src/server/sparkd/md/taifex/TaifexMarketDataProvider.cpp @@ -0,0 +1,47 @@ +#include "TaifexMarketDataProvider.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/api/md/MarketDataConfig.pb.h" +#include "lib/persistence/IDGenerator.h" +#include "server/sparkd/md/taifex/TaifexSubscription.h" + +#include +#include + +namespace bts::md { + +using bts::api::domain::Venue; +using bts::api::md::MarketDataConfig; +using bts::persistence::IDGenerator; +using std::pair; +using std::shared_ptr; +using std::string; +using std::vector; + +struct TaifexMarketDataProvider::PImpl { + MarketDataConfig config; + shared_ptr sec_id_store; + vector> sources; + + PImpl(MarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} +}; + +TaifexMarketDataProvider::TaifexMarketDataProvider( + MarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { +} + +TaifexMarketDataProvider::~TaifexMarketDataProvider() = default; + +vector>> +TaifexMarketDataProvider::get_sources() { + vector>> rv; + for (const auto &taifex : me_->config.taifex()) { + me_->sources.push_back( + std::make_shared(taifex, me_->sec_id_store)); + rv.push_back({Venue::VENUE_TAIFEX, me_->sources.back()}); + } + return rv; +} + +} // namespace bts::md diff --git a/src/server/sparkd/md/taifex/TaifexMarketDataProvider.h b/src/server/sparkd/md/taifex/TaifexMarketDataProvider.h new file mode 100644 index 000000000..4706eb5b9 --- /dev/null +++ b/src/server/sparkd/md/taifex/TaifexMarketDataProvider.h @@ -0,0 +1,33 @@ +#pragma once + +#include "lib/md/IMarketDataProvider.h" + +#include + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::api::md { +class MarketDataConfig; +} + +namespace bts::md { + +class TaifexMarketDataProvider : public IMarketDataProvider { + public: + explicit TaifexMarketDataProvider( + api::md::MarketDataConfig config, + std::shared_ptr sec_id_store); + ~TaifexMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/server/sparkd/md/taifex/TaifexSubscription.cpp b/src/server/sparkd/md/taifex/TaifexSubscription.cpp new file mode 100644 index 000000000..11a7e04f5 --- /dev/null +++ b/src/server/sparkd/md/taifex/TaifexSubscription.cpp @@ -0,0 +1,40 @@ +#include "TaifexSubscription.h" +#include "lib/api/md/TaifexMarketDataConfig.pb.h" +#include "lib/utils/thread_manager.h" +#include "server/sparkd/md/taifex/TaifexDataGateway.h" + +namespace bts::md { + +using bts::api::md::TaifexMarketDataConfig; +using bts::md::taifex::TaifexDataGateway; +using bts::persistence::IDGenerator; +using std::shared_ptr; + +struct TaifexSubscription::PImpl { + TaifexMarketDataConfig config; + shared_ptr sec_id_store; + + PImpl(TaifexMarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} +}; + +TaifexSubscription::TaifexSubscription( + TaifexMarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { +} + +TaifexSubscription::~TaifexSubscription() = default; + +void TaifexSubscription::start(shared_ptr listener) { + bts::threading::make_thread( + "taifex", + [config = me_->config, sec_id_store = me_->sec_id_store, + listener = std::move(listener)]() { + TaifexDataGateway::run( + config, std::move(listener), std::move(sec_id_store)); + }) + .detach(); + // TODO: implement 'stop' (signal the thread above to terminate, then join it). +} + +} // namespace bts::md diff --git a/src/server/sparkd/md/taifex/TaifexSubscription.h b/src/server/sparkd/md/taifex/TaifexSubscription.h new file mode 100644 index 000000000..19a55ec2d --- /dev/null +++ b/src/server/sparkd/md/taifex/TaifexSubscription.h @@ -0,0 +1,43 @@ +#pragma once + +#include "lib/md/IMarketDataSource.h" + +#include + +namespace bts::api::md { +class TaifexMarketDataConfig; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::md { + +/** + * @brief Subscribes to market data at Taifex + */ +class TaifexSubscription : public IMarketDataSource { + public: + TaifexSubscription( + api::md::TaifexMarketDataConfig config, + std::shared_ptr sec_id_store); + TaifexSubscription(const TaifexSubscription &) noexcept = delete; + TaifexSubscription(TaifexSubscription &&) noexcept; + TaifexSubscription &operator=(const TaifexSubscription &) noexcept = delete; + TaifexSubscription &operator=(TaifexSubscription &&) noexcept; + ~TaifexSubscription() override; + + /** + * Starts market data subscription on a background thread. + * + * @note Does not block. + */ + void start(std::shared_ptr listener) override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md