diff --git a/src/app_sparkd/sparkd/cmds/Run.cpp b/src/app_sparkd/sparkd/cmds/Run.cpp index 7ec9617b7..53eb19859 100644 --- a/src/app_sparkd/sparkd/cmds/Run.cpp +++ b/src/app_sparkd/sparkd/cmds/Run.cpp @@ -1,4 +1,5 @@ #include "Run.h" +#include "spark/api/md/MarketDataConfig.pb.h" #include "spark/api/public/ServiceContext.h" #include "spark/config/config.h" #include "spark/config/protos/ports.pb.h" @@ -15,6 +16,7 @@ #include "spark/orderman/OrderServiceServer.h" #include "spark/orderman/events/TradeSummarySignalDispatcher.h" #include "spark/orderman/order_factory/DynamicOrderRegistry.h" +#include "spark/persistence/IDGenerator.h" #include "spark/persistence/PersistenceServiceServer.h" #include "spark/plugin/SparkServices.h" #include "spark/plugin/plugin.h" @@ -24,8 +26,10 @@ #include "md/cmds/Run.h" #include "sparkd/algo/AlgoDefinitionServiceServer.h" #include "sparkd/analytics/AnalyticsServiceServer.h" +#include "sparkd/cmds/md/cmds/Cleanup.h" #include "sparkd/http/WebServer.h" #include "sparkd/md/MarketDataAuxiliaryServiceServer.h" +#include "sparkd/md/MarketDataGateway.h" #include "sparkd/md/recording/MDRecorder.h" #include "sparkd/position/MarginServiceServer.h" #include "sparkd/position/PositionServiceServer.h" @@ -63,7 +67,11 @@ static void execute() { bts::md::MBOProcessor mbo_processor; bts::md::TradeSummarySignalDispatcher tss_dispatcher; - md::cmds::Run::run(); + md::cmds::Cleanup::execute(); + bts::md::MarketDataGateway md_gateway{ + config::get(), + persistence::IDGenerator::instance_ptr()}; + md_gateway.start(); api::synthetic::SyntheticSecurityService synthetic_service(sc); bts::md::MDClient mdclient( diff --git a/src/app_sparkd/sparkd/cmds/crypto/LiveLadderDemo.cpp b/src/app_sparkd/sparkd/cmds/crypto/LiveLadderDemo.cpp index 3bbb73646..255f9bb75 100644 --- a/src/app_sparkd/sparkd/cmds/crypto/LiveLadderDemo.cpp +++ b/src/app_sparkd/sparkd/cmds/crypto/LiveLadderDemo.cpp @@ -1,5 +1,7 @@ #include "LiveLadderDemo.h" #include "spark/api/md/MarketDataConfig.pb.h" +#include "spark/config/config.h" +#include "spark/persistence/IDGenerator.h" #include "sparkd/md/MarketDataGateway.h" #include "sparkd/md/listeners/LiveLadderPrinter.h" @@ -7,13 +9,16 @@ namespace bts::cmds { -LiveLadderDemo::LiveLadderDemo(const api::md::MarketDataConfig &) { +using bts::api::md::MarketDataConfig; +using bts::persistence::IDGenerator; + +LiveLadderDemo::LiveLadderDemo(const MarketDataConfig &config) { char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt"; putenv(ssl_cert_file); - md::MarketDataGateway gw; + md::MarketDataGateway gw{config, IDGenerator::instance_ptr()}; md::listeners::LiveLadderPrinter ladder_printer; - gw.run(); + gw.start(); const std::size_t n_loops = 16; for (std::size_t n = 0; n < n_loops; n++) { diff --git a/src/app_sparkd/sparkd/cmds/md/cmds/Run.cpp b/src/app_sparkd/sparkd/cmds/md/cmds/Run.cpp index 3717dd89a..dd5f4d9c4 100644 --- a/src/app_sparkd/sparkd/cmds/md/cmds/Run.cpp +++ b/src/app_sparkd/sparkd/cmds/md/cmds/Run.cpp @@ -1,5 +1,6 @@ #include "Run.h" #include "Cleanup.h" +#include "spark/api/md/MarketDataConfig.pb.h" #include "spark/config/protos/market_data.pb.h" #include "spark/domain/SecurityCriteria.h" #include "spark/logging/logging.h" @@ -7,6 +8,7 @@ #include "spark/md/UserSymbol.h" #include "spark/md/relay/RelayRecorder.h" #include "spark/os/os.h" +#include "spark/persistence/IDGenerator.h" #include "spark/utils/bts_assert.h" #include "spark/utils/thread_manager.h" #include "sparkd/md/MarketDataGateway.h" @@ -14,6 +16,7 @@ #include "sparkd/md/platforms/ice_onixs/ICEMain.h" #include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h" #include +#include namespace bts::cmds::md::cmds { @@ -91,9 +94,14 @@ static void run_onixs( static void run_md_gwy() { LOG(md, Info, "Run::run_md_gwy t_id:{} t_name:{}", os::get_thread_id(), os::get_thread_name()); - bts::md::MarketDataGateway gwy = {}; - // this keeps going in a loop - gwy.run(); + bts::md::MarketDataGateway gwy{ + config::get(), + persistence::IDGenerator::instance_ptr()}; + gwy.start(); + while (true) { + using namespace std::chrono_literals; + std::this_thread::sleep_for(2s); + } } void Run::run() { diff --git a/src/app_sparkd/sparkd/md/IMarketDataProvider.cpp b/src/app_sparkd/sparkd/md/IMarketDataProvider.cpp new file mode 100644 index 000000000..fe2bbe22c --- /dev/null +++ b/src/app_sparkd/sparkd/md/IMarketDataProvider.cpp @@ -0,0 +1,7 @@ +#include "IMarketDataProvider.h" + +namespace bts::md { + +IMarketDataProvider::~IMarketDataProvider() = default; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/IMarketDataProvider.h b/src/app_sparkd/sparkd/md/IMarketDataProvider.h new file mode 100644 index 000000000..7d7bb1072 --- /dev/null +++ b/src/app_sparkd/sparkd/md/IMarketDataProvider.h @@ -0,0 +1,25 @@ +#pragma once + +#include "spark/api/domain/Venue.pb.h" +#include "sparkd/md/IMarketDataSource.h" + +#include + +namespace bts::md { + +/** + * @brief This is a single function interface, whose sole purpose is to provide a list of market + * data sources (per venue). A good example implementation is the `CCApiMarketDataProvider`, which + * reads `MarketDataConfig` and returns an `IMarketDataSource` for each venue configuration (e.g., + * `bts::api::md::BinanceMarketDataConfig`, `bts::api::md::CoinbaseMarketDataConfig`, etc.). + */ +class IMarketDataProvider { + public: + virtual ~IMarketDataProvider(); + + virtual std::vector< + std::pair>> + get_sources() = 0; +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/IMarketDataSource.cpp b/src/app_sparkd/sparkd/md/IMarketDataSource.cpp new file mode 100644 index 000000000..b1b24bfc8 --- /dev/null +++ b/src/app_sparkd/sparkd/md/IMarketDataSource.cpp @@ -0,0 +1,9 @@ +#include "IMarketDataSource.h" + +namespace bts::md { + +IMarketDataSource::~IMarketDataSource() = default; + +void IMarketDataSource::stop() {} + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/IMarketDataSource.h b/src/app_sparkd/sparkd/md/IMarketDataSource.h new file mode 100644 index 000000000..303e69084 --- /dev/null +++ b/src/app_sparkd/sparkd/md/IMarketDataSource.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace bts::md { + +class IMarketDataListener; + +/** + * @brief This interface represents a source of market data. Call `IMarketDataSource::start` to + * connect an `IMarketDataListener` to the source and begin receiving market data. + */ +class IMarketDataSource { + public: + virtual ~IMarketDataSource(); + virtual void start(std::shared_ptr listener) = 0; + virtual void stop(); +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/MarketDataGateway.cpp b/src/app_sparkd/sparkd/md/MarketDataGateway.cpp index 7c90fb6bd..e7c4429bd 100644 --- a/src/app_sparkd/sparkd/md/MarketDataGateway.cpp +++ b/src/app_sparkd/sparkd/md/MarketDataGateway.cpp @@ -1,75 +1,41 @@ #include "MarketDataGateway.h" -#include "spark/api/md/CboeMarketDataConfig.pb.h" -#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h" #include "spark/api/md/MarketDataConfig.pb.h" #include "spark/api/md/MarketDataConfigUtils.h" -#include "spark/api/md/MarketDataReplayConfig.pb.h" -#include "spark/collections/Contains.h" -#include "spark/config/protos/system_config.pb.h" #include "spark/core/FMT.h" +#include "spark/domain/DomainSecurityUtils.h" #include "spark/domain/SecurityID.h" -#include "spark/domain_model/Security.pb.h" -#include "spark/io/ProtoPrinter.h" #include "spark/logging/logging.h" #include "spark/md/GeneralMarketDataListener.h" #include "spark/md/GeneralSecurityIndex.h" +#include "spark/md/IMarketDataListener.h" #include "spark/md/MDProducer.h" #include "spark/os/os.h" #include "spark/persistence/IDGenerator.h" -#include "spark/utils/bts_assert.h" #include "spark/utils/bts_fatal.h" -#include "spark/utils/thread_manager.h" -#include "sparkd/md/MarketDataListenerID.h" -#include "sparkd/md/MarketDataRegistration.h" -#include "sparkd/md/MarketDataRegistrationError.h" -#include "sparkd/md/MarketDataRegistrationFactory.h" -#include "sparkd/md/MarketDataSubscription.h" +#include "sparkd/md/IMarketDataProvider.h" +#include "sparkd/md/OnixsMarketDataProvider.h" +#include "sparkd/md/PyMarketDataProvider.h" #include "sparkd/md/PyMarketDataSubscription.h" -#include "sparkd/md/binance/BinanceSubscription.h" -#include "sparkd/md/binance_us/BinanceUsSubscription.h" -#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h" -#include "sparkd/md/cboe/CboeMarketDataHandler.h" -#include "sparkd/md/coinbase/CoinbaseSubscription.h" -#include "sparkd/md/deribit/DeribitSubscription.h" -#include "sparkd/md/emumba/EmumbaDeribitData.h" -#include "sparkd/md/ftx/FTXSubscription.h" -#include "sparkd/md/glbx/GlbxMarketDataHandler.h" -#include "sparkd/md/okx/OkxSubscription.h" -#include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h" -#include "sparkd/md/recording/MDReplayer.h" -#include "sparkd/md/taifex/TaifexDataGateway.h" +#include "sparkd/md/ReplayMarketDataProvider.h" +#include "sparkd/md/ccapi/CCApiMarketDataProvider.h" +#include "sparkd/md/databento/DatabentoMarketDataProvider.h" +#include "sparkd/md/emumba/EmumbaMarketDataProvider.h" +#include "sparkd/md/taifex/TaifexMarketDataProvider.h" #include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include #include -namespace py = pybind11; - namespace bts::md { using bts::api::md::MarketDataConfig; -using bts::config::SystemConfig; -//using bts::domain::Security; using bts::domain::SecurityID; -using bts::io::ProtoPrinter; using bts::persistence::IDGenerator; -using os::sparkd_wd; +using std::shared_ptr; using std::string; using std::unique_ptr; -using std::unordered_map; using std::vector; -using Path = boost::filesystem::path; - static bool validate(const MarketDataConfig &cfg, vector &errs) { auto orig_n_errs = errs.size(); if (!cfg.deribit().empty() && !cfg.emumba_deribit().empty()) { @@ -79,361 +45,66 @@ static bool validate(const MarketDataConfig &cfg, vector &errs) { return errs.size() == orig_n_errs; } -struct MarketDataListenerData { - MarketDataListenerData() = default; - MarketDataListenerData(const MarketDataListenerData &) noexcept = delete; - MarketDataListenerData(MarketDataListenerData &&) noexcept = default; - MarketDataListenerData &operator=(const MarketDataListenerData &) noexcept = - delete; - MarketDataListenerData &operator=(MarketDataListenerData &&) noexcept = - default; - ~MarketDataListenerData() = default; - - MarketDataListenerID id; - MarketDataSubscription subscription; -}; - -struct MDReplayData { - bts::api::md::MarketDataReplayConfig replay_conf; - bts::api::Venue venue; - bts::md::GeneralSecurityIndex sec_index; - bts::md::MDProducer md_producer; - bts::md::recording::MDReplayer md_replayer; - - MDReplayData(const bts::api::md::MarketDataReplayConfig &replay_conf) - : replay_conf(replay_conf), venue(replay_conf.venue()), sec_index(), - md_replayer(replay_conf.venue(), sec_index, md_producer, replay_conf.replay_speed()) {} -}; - -struct VenueListenerData { - bts::md::GeneralSecurityIndex sec_index; - bts::md::MDProducer md_producer; - std::shared_ptr listener; - - VenueListenerData(bts::api::Venue venue) - : sec_index(), - listener(std::make_shared( - sec_index, md_producer, venue)) {} -}; - struct MarketDataGateway::PImpl { - api::md::MarketDataConfig md_config = - config::get(); - ccapi::EventDispatcher dispatcher; - MarketDataRegistrationFactory registration_factory; - std::shared_ptr sec_id_store = IDGenerator::instance_ptr(); - unordered_map listeners; - std::vector> replays = {}; - // we create these and pass them to the cc md sources - std::vector> venue_listeners = {}; - std::vector> py_md_subs = {}; + struct MdSink { + GeneralSecurityIndex sec_index; + MDProducer md_producer; + shared_ptr listener; + + MdSink(api::Venue venue, IDGenerator *sec_id_store) + : sec_index(), listener(std::make_shared( + sec_index, md_producer, venue, sec_id_store)) {} + }; + + MarketDataConfig config; + shared_ptr sec_id_store; + vector> providers; + vector> sinks; + vector> sources; + + PImpl(MarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) { + providers.emplace_back(std::make_unique( + this->config, this->sec_id_store)); + providers.emplace_back( + std::make_unique(this->config)); + providers.emplace_back( + std::make_unique(this->config)); + providers.emplace_back(std::make_unique( + this->config, this->sec_id_store)); + providers.emplace_back(std::make_unique( + this->config, this->sec_id_store)); + providers.emplace_back(std::make_unique( + this->config, this->sec_id_store)); + providers.emplace_back(std::make_unique( + this->config, this->sec_id_store)); + } }; -MarketDataGateway::MarketDataGateway() - : me_(std::make_unique()) {} +MarketDataGateway::MarketDataGateway( + MarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique( + std::move(config), std::move(sec_id_store))) {} MarketDataGateway::~MarketDataGateway() = default; // starts the market data sources and then keeps the function going -void MarketDataGateway::run() { +void MarketDataGateway::start() { vector cfg_errs; - if (!validate(me_->md_config, cfg_errs)) { + if (!validate(me_->config, cfg_errs)) { BTS_FATAL_MSGF("Invalid MarketDataConfig:", FMT_JOIN(cfg_errs, "\n")); return; } LOG(md, Info, "MarketDataGateway::run starting t_id:{} t_name:{}", os::get_thread_id(), os::get_thread_name()); - _start_cc(me_->md_config); - if (!me_->md_config.emumba_deribit().empty()) { - bts_assert(me_->md_config.emumba_deribit().size() == 1); - _start_emumba_deribit(me_->md_config.emumba_deribit().at(0)); - } - start_glbx(me_->md_config); - start_taifex(me_->md_config); - start_cboe(me_->md_config); - _start_replays(me_->md_config); - start_py_(); - while (true) { - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - LOG(md, Info, "MarketDataGateway::run ending t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); -} - -void MarketDataGateway::start_py_() { - for (const auto &py_plugin : me_->md_config.python_md_providers()) { - try { - me_->py_md_subs.push_back(std::make_shared( - py_plugin, me_->sec_id_store.get())); - } catch (const std::exception &ex) { - bts_assert_msgf( - false, "PyMarketDataSubscription threw: {}", - std::string{ex.what()}); + for (auto &provider : me_->providers) { + for (auto &[venue, source] : provider->get_sources()) { + const auto v = domain::DomainSecurityUtils::convert(venue); + auto &sink = me_->sinks.emplace_back( + std::make_unique(v, me_->sec_id_store.get())); + source->start(sink->listener); } } } -// CCAPI ////////////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////////////////////// -MarketDataRegistration MarketDataGateway::_start_cc( - const MarketDataConfig &cfg) noexcept { - LOG(md, Info, "MarketDataGateway::_start_cc starting t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); - auto reg = me_->registration_factory.create(); - bts_assert(reg.listener_id.has_value()); - bts_assert(!CONTAINS_KEY(me_->listeners, *reg.listener_id)); - LOG(md, Info, "MarketDataGateway subscribing to data:\n{}", - ProtoPrinter::to_string(cfg)); - - vector cfg_errs; - if (!validate(cfg, cfg_errs)) { - return MarketDataRegistration{ - {MarketDataRegistrationError::Type::CONFIG_ERROR, - fmt::to_string(fmt::join(cfg_errs, "\n"))}}; - } - - MarketDataListenerData &listener_data = me_->listeners[*reg.listener_id]; - listener_data.id = *reg.listener_id; - - // FTX - for (const auto &ftx : cfg.ftx()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_FTX)); - listener_data.subscription.add(ftx::FTXSubscription{ - &me_->dispatcher, ftx, me_->sec_id_store.get(), p->listener.get()}); - } - - // Binance - for (const auto &binance : cfg.binance()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_BINANCE)); - listener_data.subscription.add(binance::BinanceSubscription{ - &me_->dispatcher, binance, me_->sec_id_store.get(), - p->listener.get()}); - } - - // Binance US - for (const auto &binance_us : cfg.binance_us()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_BINANCE_US)); - listener_data.subscription.add(binance_us::BinanceUsSubscription{ - &me_->dispatcher, binance_us, me_->sec_id_store.get(), - p->listener.get()}); - } - // Coinbase - for (const auto &coinbase : cfg.coinbase()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_COINBASE)); - listener_data.subscription.add(coinbase::CoinbaseSubscription{ - &me_->dispatcher, coinbase, me_->sec_id_store.get(), - p->listener.get()}); - } - // Binance USDS Futures - for (const auto &binance_ufs : cfg.binance_usds_futures()) { - auto p = - me_->venue_listeners.emplace_back(std::make_shared( - bts::api::VENUE_BINANCE_USDS_FUTURES)); - listener_data.subscription.add( - binance_usds_futures::BinanceUsdsFuturesSubscription{ - &me_->dispatcher, binance_ufs, me_->sec_id_store.get(), - p->listener.get()}); - } - - // Deribit - for (const auto &deribit : cfg.deribit()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_DERIBIT)); - listener_data.subscription.add(deribit::DeribitSubscription{ - &me_->dispatcher, deribit, me_->sec_id_store.get(), - p->listener.get()}); - } - - // Okx - for (const auto &okx : cfg.okx()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_OKX)); - listener_data.subscription.add(okx::OkxSubscription{ - &me_->dispatcher, okx, me_->sec_id_store.get(), p->listener.get()}); - } - - LOG(md, Info, "MarketDataGateway::_start_cc ending t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); - return reg; -} - -void MarketDataGateway::unsubscribe(MarketDataListenerID id) { - LOG(md, Info, "Unsubscribing listener {}", id); - me_->listeners.erase(id); -} -/// glbx ////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////////// -void MarketDataGateway::start_glbx(const bts::api::md::MarketDataConfig &cfg){ - for (const auto &glbx : cfg.glbx()) { - if(!glbx.use_databento()){ - _start_onix_glbx(glbx); - return; - } - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_GLBX)); - // Capture `p->listener` and `me_->sec_id_store` by value - auto listener = p->listener; - auto sec_id_store = me_->sec_id_store; - std::string tid_name = "glbx"; - bts::threading::make_thread(tid_name, [listener, sec_id_store, glbx](){ - api::md::glbx::GlbxMarketDatahandler::run( - listener, - sec_id_store, - glbx); - }).detach(); - } -} -static void run_onixs() { - LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name", - os::get_thread_id(), os::get_thread_name()); - bts::md::MBOWriter *mbo = nullptr; - bts::md::TradeSummarySignalDispatcher *tssd = nullptr; - std::vector> platforms; - // get all glbx symbols - auto cme_symbols = SymbolLoader::load( - bts::domain::SecurityCriteria::create({api::domain::VENUE_GLBX})); - if (!cme_symbols.empty()) { - auto *main = new bts::md::MDP3Main(); - main->start({cme_symbols.begin(), cme_symbols.end()}, mbo, tssd); - platforms.emplace_back(main); - } - // deribit - // auto *d_platform = new bts::md::EmMDListener() - // Timers - auto &disp = utils::Dispatcher::current(); - utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() { - for (auto &p : platforms) - p->send_heartbeats(); - }); - heartbeat_timer.start(); - - disp.run(); -} - -void MarketDataGateway::_start_onix_glbx( - bts::api::md::GlbxMarketDataConfig cfg) { - LOG(md, Info, - "MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. " - "cfg:\n{}", - os::get_thread_id(), os::get_thread_name(), cfg.DebugString()); - bts::threading::make_thread("Onix CME", [&]() { - bts::md::run_onixs(); - }).detach(); - -} - - -// deribit //////////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////////////////////// - -void MarketDataGateway::_start_emumba_deribit( - bts::api::md::EmumbaDeribitMarketDataConfig cfg) { - LOG(md, Info, - "MarketDataGateway::_start_emumba_deribit starting t_id:{} t_name:{}. " - "cfg:\n{}", - os::get_thread_id(), os::get_thread_name(), cfg.DebugString()); - auto start_deribit_thr = [=]() { - const auto config_path = Path{config::get().config_path()} / - cfg.json_conf_file_name(); - const auto conf = - bts::cmds::DeribitDemoConfiguration::read_file(config_path.string()); - bts_assert_msg(conf.has_value(), "Failed to parse deribit config"); - // log - for (const auto &s : conf.value().symbol_config_list) { - LOG(md, Info, "sub.symbol: {}", s.symbol_name); - LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future, - s.interested_in_option); - for (const auto &i : s.interested_instruments) { - LOG(md, Info, "sub.instrument: {}", i); - } - } - EmumbaDeribitData ddata_obj; - EmumbaDeribitData *ddata = &ddata_obj; - // end log - bts_assert(conf.has_value()); - LOG(md, Info, "Deribit conf: {}", conf.value()); - ddata->real_handler = - std::make_shared( - conf.value().symbol_config_list, conf.value().trading_config, - conf.value().credential_map, conf.value().logging_configuration, - conf.value().deribit_url, &ddata->listener); - // since the trading interface is housed inside the EmMDListener, we need to - // create and send emumba's trading_session_interface - for (const auto &p : conf.value().credential_map) { - const auto &sess = p.second; - auto ts_ptr = std::make_shared< - emumba::deribit_connector::trading_session_interface>( - ddata->real_handler->create_new_trading_session(sess)); - ddata->trading_session_vec.push_back({sess, ts_ptr}); - } - bts_assert(!ddata->trading_session_vec.empty()); - ddata->listener.set_handler( - ddata->real_handler.get(), ddata->trading_session_vec); - ddata->real_handler->run(); - }; - threading::make_thread("Emumba", [&]() { start_deribit_thr(); }).detach(); - LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); -} - -// replays //////////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////////////////////// -void MarketDataGateway::_start_replays( - const bts::api::md::MarketDataConfig &cfg) { - LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); - for (const auto &r : - MarketDataConfigUtils::get_effective_replay_config(cfg)) { - me_->replays.emplace_back(std::make_unique(r)); - } - for (const auto &replayer : me_->replays) { - std::string tid_name = "MDReplayer_" + Venue_Name(replayer->venue); - bts::threading::make_thread(tid_name, [&]() { - replayer->md_replayer.run(); - }).detach(); - } - LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); -} - -void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) { - // taifex - for (const auto &taifex : cfg.taifex()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_TAIFEX)); - // Capture `p->listener` and `me_->sec_id_store` by value - auto listener = p->listener; - auto sec_id_store = me_->sec_id_store; - std::string tid_name = "taifex"; - bts::threading::make_thread(tid_name, [listener, sec_id_store, taifex]() { - bts::md::taifex::TaifexDataGateway::run( - taifex, listener, sec_id_store); - }).detach(); - - } -} -void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg) { - for (const api::md::CboeMarketDataConfig &cboe : cfg.cboe()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_CBOE)); - std::string tid_name = "CBOE"; - - // Capture `p->listener` and `me_->sec_id_store` by value - auto listener = p->listener; - auto sec_id_store = me_->sec_id_store; - - bts::threading::make_thread(tid_name, [listener, sec_id_store, cboe]() { - api::md::cboe::CboeMarketDatahandler::run( - listener, - sec_id_store, - cboe); - }).detach(); - - } -} - } // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/MarketDataGateway.h b/src/app_sparkd/sparkd/md/MarketDataGateway.h index d158e8114..11e2bc3a0 100644 --- a/src/app_sparkd/sparkd/md/MarketDataGateway.h +++ b/src/app_sparkd/sparkd/md/MarketDataGateway.h @@ -1,45 +1,33 @@ #pragma once -#include "spark/api/md/GlbxMarketDataConfig.pb.h" -#include "sparkd/md/MarketDataListenerID.h" - #include namespace bts::api::md { -class EmumbaDeribitMarketDataConfig; class MarketDataConfig; -} // namespace bts::api::md +} + +namespace bts::persistence { +class IDGenerator; +} namespace bts::md { -class IMarketDataListener; -class MarketDataRegistration; - +/** + * @brief The `MarketDataGateway` is responsible for starting all configured market data sources. + * It connects each source to a special `IMarketDataListener` that writes the data to shared memory. + */ class MarketDataGateway { public: - MarketDataGateway(); + explicit MarketDataGateway( + api::md::MarketDataConfig config, + std::shared_ptr sec_id_store); ~MarketDataGateway(); - /** - * @brief Connects a market data listener to a market data source - * @return MarketDataRegistration - */ - void run(); - void unsubscribe(MarketDataListenerID id); - static void test_py(); + void start(); private: struct PImpl; std::unique_ptr me_; - MarketDataRegistration _start_cc(const api::md::MarketDataConfig &) noexcept; - void start_py_(); - static void _start_emumba_deribit( - bts::api::md::EmumbaDeribitMarketDataConfig cfg); - static void _start_onix_glbx(bts::api::md::GlbxMarketDataConfig cfg); - void _start_replays(const bts::api::md::MarketDataConfig &cfg); - void start_glbx(const bts::api::md::MarketDataConfig &cfg); - void start_cboe(const bts::api::md::MarketDataConfig &cfg); - void start_taifex(const bts::api::md::MarketDataConfig &cfg); }; } // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/MarketDataSubscription.cpp b/src/app_sparkd/sparkd/md/MarketDataSubscription.cpp index 6ec34a995..597487490 100644 --- a/src/app_sparkd/sparkd/md/MarketDataSubscription.cpp +++ b/src/app_sparkd/sparkd/md/MarketDataSubscription.cpp @@ -1,60 +1,60 @@ -#include "MarketDataSubscription.h" -#include "sparkd/md/binance/BinanceSubscription.h" -#include "sparkd/md/binance_us/BinanceUsSubscription.h" -#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h" -#include "sparkd/md/coinbase/CoinbaseSubscription.h" -#include "sparkd/md/deribit/DeribitSubscription.h" -#include "sparkd/md/ftx/FTXSubscription.h" -#include "sparkd/md/okx/OkxSubscription.h" +// #include "MarketDataSubscription.h" +// #include "sparkd/md/binance/BinanceSubscription.h" +// #include "sparkd/md/binance_us/BinanceUsSubscription.h" +// #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h" +// #include "sparkd/md/coinbase/CoinbaseSubscription.h" +// #include "sparkd/md/deribit/DeribitSubscription.h" +// #include "sparkd/md/ftx/FTXSubscription.h" +// #include "sparkd/md/okx/OkxSubscription.h" -#include +// #include -namespace bts::md { +// namespace bts::md { -using std::vector; +// using std::vector; -struct MarketDataSubscription::PImpl { - vector ftx_subs; - vector binance_subs; - vector binance_us_subs; - vector coinbase_subs; - vector - binance_usds_futures_subs; - vector deribit_subs; - vector okx_subs; - // TODO: add more exchanges -}; +// struct MarketDataSubscription::PImpl { +// vector ftx_subs; +// vector binance_subs; +// vector binance_us_subs; +// vector coinbase_subs; +// vector +// binance_usds_futures_subs; +// vector deribit_subs; +// vector okx_subs; +// // TODO: add more exchanges +// }; -MarketDataSubscription::MarketDataSubscription() - : me_(std::make_unique()) {} +// MarketDataSubscription::MarketDataSubscription() +// : me_(std::make_unique()) {} -MarketDataSubscription::~MarketDataSubscription() = default; +// MarketDataSubscription::~MarketDataSubscription() = default; -void MarketDataSubscription::add(binance::BinanceSubscription sub) { - me_->binance_subs.push_back(std::move(sub)); -} +// void MarketDataSubscription::add(binance::BinanceSubscription sub) { +// me_->binance_subs.push_back(std::move(sub)); +// } -void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) { - me_->binance_us_subs.push_back(std::move(sub)); -} -void MarketDataSubscription::add(coinbase::CoinbaseSubscription sub) { - me_->coinbase_subs.push_back(std::move(sub)); -} -void MarketDataSubscription::add( - binance_usds_futures::BinanceUsdsFuturesSubscription sub) { - me_->binance_usds_futures_subs.push_back(std::move(sub)); -} +// void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) { +// me_->binance_us_subs.push_back(std::move(sub)); +// } +// void MarketDataSubscription::add(coinbase::CoinbaseSubscription sub) { +// me_->coinbase_subs.push_back(std::move(sub)); +// } +// void MarketDataSubscription::add( +// binance_usds_futures::BinanceUsdsFuturesSubscription sub) { +// me_->binance_usds_futures_subs.push_back(std::move(sub)); +// } -void MarketDataSubscription::add(deribit::DeribitSubscription sub) { - me_->deribit_subs.push_back(std::move(sub)); -} +// void MarketDataSubscription::add(deribit::DeribitSubscription sub) { +// me_->deribit_subs.push_back(std::move(sub)); +// } -void MarketDataSubscription::add(ftx::FTXSubscription sub) { - me_->ftx_subs.push_back(std::move(sub)); -} +// void MarketDataSubscription::add(ftx::FTXSubscription sub) { +// me_->ftx_subs.push_back(std::move(sub)); +// } -void MarketDataSubscription::add(okx::OkxSubscription sub) { - me_->okx_subs.push_back(std::move(sub)); -} +// void MarketDataSubscription::add(okx::OkxSubscription sub) { +// me_->okx_subs.push_back(std::move(sub)); +// } -} // namespace bts::md +// } // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/MarketDataSubscription.h b/src/app_sparkd/sparkd/md/MarketDataSubscription.h index 9a3de2aeb..9ddcffbdf 100644 --- a/src/app_sparkd/sparkd/md/MarketDataSubscription.h +++ b/src/app_sparkd/sparkd/md/MarketDataSubscription.h @@ -1,59 +1,59 @@ -#pragma once +// #pragma once -#include +// #include -namespace bts::md::binance { -class BinanceSubscription; -} +// namespace bts::md::binance { +// class BinanceSubscription; +// } -namespace bts::md::binance_us { -class BinanceUsSubscription; -} -namespace bts::md::coinbase { -class CoinbaseSubscription; -} -namespace bts::md::binance_usds_futures { -class BinanceUsdsFuturesSubscription; -} +// namespace bts::md::binance_us { +// class BinanceUsSubscription; +// } +// namespace bts::md::coinbase { +// class CoinbaseSubscription; +// } +// namespace bts::md::binance_usds_futures { +// class BinanceUsdsFuturesSubscription; +// } -namespace bts::md::deribit { -class DeribitSubscription; -} +// namespace bts::md::deribit { +// class DeribitSubscription; +// } -namespace bts::md::ftx { -class FTXSubscription; -} +// namespace bts::md::ftx { +// class FTXSubscription; +// } -namespace bts::md::okx { -class OkxSubscription; -} +// namespace bts::md::okx { +// class OkxSubscription; +// } -namespace bts::md { +// namespace bts::md { -class MarketDataSubscription { - public: - MarketDataSubscription(); +// class MarketDataSubscription { +// public: +// MarketDataSubscription(); - MarketDataSubscription(const MarketDataSubscription &) noexcept = delete; - MarketDataSubscription(MarketDataSubscription &&) noexcept = default; - MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept = - delete; - MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept = - default; +// MarketDataSubscription(const MarketDataSubscription &) noexcept = delete; +// MarketDataSubscription(MarketDataSubscription &&) noexcept = default; +// MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept = +// delete; +// MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept = +// default; - ~MarketDataSubscription(); +// ~MarketDataSubscription(); - void add(binance::BinanceSubscription); - void add(binance_us::BinanceUsSubscription); - void add(coinbase::CoinbaseSubscription); - void add(binance_usds_futures::BinanceUsdsFuturesSubscription); - void add(deribit::DeribitSubscription); - void add(ftx::FTXSubscription); - void add(okx::OkxSubscription); +// void add(binance::BinanceSubscription); +// void add(binance_us::BinanceUsSubscription); +// void add(coinbase::CoinbaseSubscription); +// void add(binance_usds_futures::BinanceUsdsFuturesSubscription); +// void add(deribit::DeribitSubscription); +// void add(ftx::FTXSubscription); +// void add(okx::OkxSubscription); - private: - struct PImpl; - std::unique_ptr me_; -}; +// private: +// struct PImpl; +// std::unique_ptr me_; +// }; -} // namespace bts::md +// } // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/OnixsMarketDataProvider.cpp b/src/app_sparkd/sparkd/md/OnixsMarketDataProvider.cpp new file mode 100644 index 000000000..60ce7c4a7 --- /dev/null +++ b/src/app_sparkd/sparkd/md/OnixsMarketDataProvider.cpp @@ -0,0 +1,44 @@ +#include "OnixsMarketDataProvider.h" +#include "spark/api/domain/Venue.pb.h" +#include "spark/persistence/IDGenerator.h" +#include "sparkd/md/glbx/OnixsGlbxSubscription.h" + +#include +#include + +namespace bts::md { + +using bts::api::domain::Venue; +using bts::api::md::MarketDataConfig; +using bts::md::OnixsMarketDataProvider; +using bts::persistence::IDGenerator; +using std::pair; +using std::shared_ptr; +using std::string; +using std::vector; + +struct OnixsMarketDataProvider::PImpl { + MarketDataConfig config; + std::shared_ptr sec_id_store; + vector> sources; +}; + +OnixsMarketDataProvider::OnixsMarketDataProvider(MarketDataConfig config) + : me_(std::make_unique()) { + me_->config = std::move(config); +} + +OnixsMarketDataProvider::~OnixsMarketDataProvider() = default; + +vector>> +OnixsMarketDataProvider::get_sources() { + vector>> rv; + for (const auto &glbx : me_->config.glbx()) { + me_->sources.push_back( + std::make_shared(me_->sec_id_store, glbx)); + rv.push_back({Venue::VENUE_GLBX, me_->sources.back()}); + } + return rv; +} + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/OnixsMarketDataProvider.h b/src/app_sparkd/sparkd/md/OnixsMarketDataProvider.h new file mode 100644 index 000000000..519d59a31 --- /dev/null +++ b/src/app_sparkd/sparkd/md/OnixsMarketDataProvider.h @@ -0,0 +1,26 @@ +#pragma once + +#include "spark/api/md/MarketDataConfig.pb.h" +#include "sparkd/md/IMarketDataProvider.h" +#include "sparkd/md/IMarketDataSource.h" + +#include + +namespace bts::md { + +class OnixsMarketDataProvider : public IMarketDataProvider { + public: + OnixsMarketDataProvider(); + explicit OnixsMarketDataProvider(api::md::MarketDataConfig config); + ~OnixsMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/PyMarketDataProvider.cpp b/src/app_sparkd/sparkd/md/PyMarketDataProvider.cpp new file mode 100644 index 000000000..a18dedbaf --- /dev/null +++ b/src/app_sparkd/sparkd/md/PyMarketDataProvider.cpp @@ -0,0 +1,63 @@ +#include "PyMarketDataProvider.h" +#include "spark/api/md/MarketDataConfig.pb.h" +#include "spark/os/os.h" +#include "spark/persistence/IDGenerator.h" +#include "spark/utils/bts_assert.h" +#include "sparkd/md/IMarketDataSource.h" +#include "sparkd/md/MarketDataSubscription.h" +#include "sparkd/md/PyMarketDataSubscription.h" + +namespace bts::md { + +using bts::api::md::MarketDataConfig; +using bts::os::sparkd_wd; +using bts::persistence::IDGenerator; +using std::pair; +using std::shared_ptr; +using std::string; +using std::vector; + +using Path = boost::filesystem::path; + +struct PyMarketDataProvider::PImpl { + MarketDataConfig config; + shared_ptr sec_id_store; + vector> py_md_subs; + + PImpl(MarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} +}; + +PyMarketDataProvider::PyMarketDataProvider( + MarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { +} + +PyMarketDataProvider::~PyMarketDataProvider() = default; + +vector>> +PyMarketDataProvider::get_sources() { + vector>> rv; + const Path wd{sparkd_wd()}; + vector module_paths{ + wd.string(), + (wd / "python_md_providers").string(), + }; + for (const auto &py_plugin : me_->config.python_md_providers()) { + try { + auto source = std::make_shared( + py_plugin, me_->sec_id_store.get()); + me_->py_md_subs.push_back(source); + rv.push_back( + std::make_pair>( + py_plugin.venue(), source)); + } catch (const std::exception &ex) { + bts_assert_msgf( + false, "PyMarketDataSubscription threw: {}", + std::string{ex.what()}); + } + } + return rv; +} + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/PyMarketDataProvider.h b/src/app_sparkd/sparkd/md/PyMarketDataProvider.h new file mode 100644 index 000000000..2a8dbae26 --- /dev/null +++ b/src/app_sparkd/sparkd/md/PyMarketDataProvider.h @@ -0,0 +1,27 @@ +#pragma once + +#include "spark/api/md/MarketDataConfig.pb.h" +#include "spark/persistence/IDGenerator.h" +#include "sparkd/md/IMarketDataProvider.h" + +#include + +namespace bts::md { + +class PyMarketDataProvider : public IMarketDataProvider { + public: + explicit PyMarketDataProvider( + api::md::MarketDataConfig config, + std::shared_ptr sec_id_store); + ~PyMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp index 52d52f11a..c9646cc9e 100644 --- a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp +++ b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp @@ -6,21 +6,19 @@ #include "python_md/py_security_list.h" #include "python_md/py_trade.h" #include "spark/api/domain/CallPut.pb.h" +#include "spark/api/domain/SecurityType.h" #include "spark/api/domain/Venue.h" +#include "spark/api/domain/Venue.pb.h" #include "spark/api/md/PyMarketDataProviderConfig.pb.h" #include "spark/core/TimeStamp.h" #include "spark/domain/AggressorSide.h" -#include "spark/domain/DomainSecurityUtils.h" #include "spark/domain/FutureSecurity.h" #include "spark/domain/Security.h" #include "spark/domain/SecurityID.h" #include "spark/domain/SpotSecurity.h" #include "spark/domain/TradeRecord.h" -#include "spark/domain_model/Security.pb.h" #include "spark/logging/logging.h" -#include "spark/md/GeneralMarketDataListener.h" -#include "spark/md/GeneralSecurityIndex.h" -#include "spark/md/MDProducer.h" +#include "spark/md/IMarketDataListener.h" #include "spark/os/os.h" #include "spark/persistence/IDGenerator.h" #include "spark/utils/bts_assert.h" @@ -29,7 +27,7 @@ #include #include #include -#include +#include #include namespace bts::md { @@ -44,23 +42,12 @@ using bts::md::py_interop::PySecurityList; using bts::md::py_interop::PyTrade; using bts::md::py_interop::PyTradeList; using bts::persistence::IDGenerator; -using std::pair; using std::shared_ptr; using std::string; -using std::unique_ptr; -using std::unordered_map; using std::vector; using Path = boost::filesystem::path; -struct VenueSink { - VenueSink(api::Venue venue, GeneralSecurityIndex *index) - : producer(), listener(*index, producer, venue) {} - - MDProducer producer; - GeneralMarketDataListener listener; -}; - struct TradeTuple { py_interop::PyTrade py_trade; domain::TradeRecord trade; @@ -74,41 +61,40 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink { {this->config.module_path().begin(), this->config.module_path().end()}, this), - sec_id_store_(sec_id_store), - index_(std::make_unique()) {} + sec_id_store_(sec_id_store) {} + + void start(shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener_ = std::move(listener); + LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}", + os::get_thread_id(), os::get_thread_name()); + py_md_exec_.start(); + } - void start() { py_md_exec_.start(); } void stop() { py_md_exec_.stop(); } private: PyMarketDataProviderConfig config; PyMdExecutor py_md_exec_; - unordered_map> sinks_; + shared_ptr listener_; IDGenerator *sec_id_store_; - unique_ptr index_; // The main store for securities. vector securities_; - // We also maintain Security pointers per venue (the GeneralMarketDataListener does not support - // writing arbitrary groups of securities, they must share a common venue when batched together). - unordered_map> - securities_by_venue_; + vector security_ptrs_; domain::Book book_; // Since the python md provider isn't tethered to a single venue, we // must know the venue for each individual trade. - vector trades_; - unordered_map< - SecurityID, - pair>> - trades_by_security_; + vector trades_; + vector trade_ptrs_; void on_securities(const PySecurityList &py_securities) override { LOG(md, Info, "Received securities from python plugin. count: {}", py_securities.size()); - // `securities_` and `securities_by_venue_` should already be empty, but no harm in clearing - // them again. We keep them as member variables to avoid (re)allocating memory in each callback. + // `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them + // again. We keep them as member variables to avoid (re)allocating memory in each callback. securities_.clear(); - securities_by_venue_.clear(); + security_ptrs_.clear(); for (const auto &py_sec : py_securities.items()) { LOG(md, Info, "PyMarketDataSubscription adding security {}", py_sec.name()); @@ -119,27 +105,23 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink { py_sec.name()); continue; } - securities_.emplace_back(std::move(*sec)); + securities_.push_back(std::move(*sec)); } // Populate vectors of security ptrs per each venue. for (auto &sec : securities_) { - securities_by_venue_[sec.venue()].push_back(&sec); + security_ptrs_.push_back(&sec); } // Pass securities to the sinks (per venue). - for (const auto &[venue, secs] : securities_by_venue_) { - get_listener_(venue).on_securities(secs); - } + listener_->on_securities(security_ptrs_); securities_.clear(); - securities_by_venue_.clear(); + security_ptrs_.clear(); } void on_books(const PyBookList &py_books) override { for (auto &py_book : py_books.items()) { - api::domain::Venue venue; - bts_assert(api::domain::venue_parse_smart(py_book.venue(), &venue)); bts_assert(book_.count_populated() == 0); to_book_(py_book, book_); - get_listener_(venue).on_book(book_); + listener_->on_book(book_); book_.clear(); } } @@ -147,50 +129,33 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink { void on_trades(const PyTradeList &py_trades) override { LOG(md, Info, "Received trades from python plugin. count: {}", py_trades.size()); - // `trades_` and `trades_by_security_` should already be empty, but no harm in clearing them - // again. We keep them as member variables to avoid (re)allocating memory in each callback. + // `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again. + // We keep them as member variables to avoid (re)allocating memory in each callback. trades_.clear(); - trades_by_security_.clear(); + trade_ptrs_.clear(); for (const auto &py_trade : py_trades.items()) { LOG(md, Info, "PyMarketDataSubscription received trade {}", py_trade.to_string()); - auto &trade = trades_.emplace_back(); - bts_assert( - api::domain::venue_parse_smart(py_trade.venue(), &trade.venue)); - to_bts_trade_(py_trade, trade.trade); + to_bts_trade_(py_trade, trades_.emplace_back()); } - // Populate vectors of trade ptrs per each security. - for (const auto &trade : trades_) { - auto &sec_trades = trades_by_security_[trade.trade.sec_id]; - sec_trades.first = trade.venue; - sec_trades.second.push_back(&trade.trade); - } - // Pass trades to the sinks (per venue) - for (const auto &[sec_id, p] : trades_by_security_) { - get_listener_(p.first).on_trades(p.second); + for (const auto &t : trades_) { + trade_ptrs_.push_back(&t); } + listener_->on_trades(trade_ptrs_); trades_.clear(); - trades_by_security_.clear(); + trade_ptrs_.clear(); } - GeneralMarketDataListener &get_listener_(api::domain::Venue venue) { - const auto api_venue = domain::DomainSecurityUtils::convert(venue); - return sinks_ - .try_emplace( - api_venue, std::make_shared(api_venue, index_.get())) - .first->second->listener; - } - - SecurityID get_sec_id_(const string &venue_name, const string &sec_name) { - api::domain::Venue venue; - // Although an overload of 'get_id' accepts a string, we parse `venue_name` to the enum to - // normalize different spellings of the same venue. For example, "deribt", "DERIBIT", and - // "VENUE_deribit" all represent the VENUE_DERIBIT enum (see `venue_parse_smart` for details). - bts_assert(api::domain::venue_parse_smart(venue_name, &venue)); - auto sec_id = sec_id_store_->get_id(venue, sec_name, true); + SecurityID get_sec_id_( + api::domain::Venue venue, + api::domain::SecurityType sec_type, + const string &sec_name) { + auto sec_id = sec_id_store_->get_id(venue, sec_type, sec_name, true); bts_assert_msgf( - sec_id, "Failed to get security id for py_book. venue: {}; name: {}", - venue, sec_name); + sec_id, + "Failed to get security id for py_book. venue: {}, type: {}, name: " + "{}", + venue, sec_type, sec_name); return SecurityID{*sec_id}; } @@ -205,7 +170,6 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink { LOG(md, Error, "Failed to parse venue from '{}'", py_sec.venue()); return std::nullopt; } - sec_comm.id = get_sec_id_(py_sec.venue(), py_sec.exchange_id()); sec_comm.exchange_sec_id = py_sec.exchange_id(); sec_comm.currency = py_sec.currency(); sec_comm.asset = py_sec.asset(); @@ -236,11 +200,18 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink { "option)"); return std::nullopt; } + sec_comm.id = + get_sec_id_(sec_comm.venue, sec.type(), sec_comm.exchange_sec_id); return sec; } void to_bts_trade_(const PyTrade &source, domain::TradeRecord &dest) { - dest.sec_id = get_sec_id_(source.venue(), source.exchange_security_id()); + api::domain::Venue venue; + bts_assert(api::domain::venue_parse_smart(source.venue(), &venue)); + api::domain::SecurityType sec_type; + bts_assert(api::domain::security_type_parse_smart( + source.security_type(), &sec_type)); + dest.sec_id = get_sec_id_(venue, sec_type, source.exchange_security_id()); dest.time_stamp = core::TimeStamp::from_nanos(source.timestamp()); dest.agg_side = source.aggressor_side().starts_with('B') || source.aggressor_side().starts_with('b') @@ -266,20 +237,27 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink { void to_book_(const PyBook &source, domain::Book &dest) { to_book_side_(source.bids(), dest.bids()); to_book_side_(source.asks(), dest.asks()); + api::domain::Venue venue; + bts_assert(api::domain::venue_parse_smart(source.venue(), &venue)); + api::domain::SecurityType type; + bts_assert(api::domain::security_type_parse_smart( + source.security_type(), &type)); dest.set_security_id( - get_sec_id_(source.venue(), source.exchange_security_id())); + get_sec_id_(venue, type, source.exchange_security_id())); } }; PyMarketDataSubscription::PyMarketDataSubscription( PyMarketDataProviderConfig config, IDGenerator *sec_id_store) : me_(std::make_unique( - std::move(config), sec_id_store)) { - LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); - me_->start(); -} + std::move(config), sec_id_store)) {} PyMarketDataSubscription::~PyMarketDataSubscription() { me_->stop(); } +void PyMarketDataSubscription::start(shared_ptr listener) { + me_->start(std::move(listener)); +} + +void PyMarketDataSubscription::stop() { me_->stop(); } + } // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.h b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.h index 6284bf50a..c2534ba7b 100644 --- a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.h +++ b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.h @@ -1,5 +1,7 @@ #pragma once +#include "sparkd/md/IMarketDataSource.h" + #include namespace bts::api::md { @@ -12,13 +14,16 @@ class IDGenerator; namespace bts::md { -class PyMarketDataSubscription { +class PyMarketDataSubscription : public IMarketDataSource { public: explicit PyMarketDataSubscription( api::md::PyMarketDataProviderConfig config, persistence::IDGenerator *sec_id_store); ~PyMarketDataSubscription(); + void start(std::shared_ptr listener) override; + void stop() override; + private: struct PImpl; std::unique_ptr me_; diff --git a/src/app_sparkd/sparkd/md/ReplayMarketDataProvider.cpp b/src/app_sparkd/sparkd/md/ReplayMarketDataProvider.cpp new file mode 100644 index 000000000..1284fe71e --- /dev/null +++ b/src/app_sparkd/sparkd/md/ReplayMarketDataProvider.cpp @@ -0,0 +1,52 @@ +#include "ReplayMarketDataProvider.h" +#include "spark/api/md/MarketDataConfig.pb.h" +#include "spark/api/md/MarketDataConfigUtils.h" +#include "spark/config/config.h" +#include "spark/domain/DomainSecurityUtils.h" +#include "spark/persistence/IDGenerator.h" +#include "sparkd/md/IMarketDataSource.h" +#include "sparkd/md/recording/MDReplayer.h" + +namespace bts::md { + +using bts::api::md::MarketDataConfig; +using bts::domain::DomainSecurityUtils; +using bts::md::recording::MDReplayer; +using bts::persistence::IDGenerator; +using std::pair; +using std::shared_ptr; +using std::string; +using std::vector; + +struct ReplayMarketDataProvider::PImpl { + MarketDataConfig config; + shared_ptr sec_id_store; + vector>> sources; + + PImpl(MarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} +}; + +ReplayMarketDataProvider::ReplayMarketDataProvider() + : ReplayMarketDataProvider( + config::get(), IDGenerator::instance_ptr()) {} + +ReplayMarketDataProvider::ReplayMarketDataProvider( + MarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { +} + +ReplayMarketDataProvider::~ReplayMarketDataProvider() = default; + +vector>> +ReplayMarketDataProvider::get_sources() { + const auto replay_cfgs = + MarketDataConfigUtils::get_effective_replay_config(me_->config); + for (const auto &r : replay_cfgs) { + const auto v = DomainSecurityUtils::convert(r.venue()); + me_->sources.push_back({v, std::make_shared(r.venue())}); + } + return me_->sources; +} + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/ReplayMarketDataProvider.h b/src/app_sparkd/sparkd/md/ReplayMarketDataProvider.h new file mode 100644 index 000000000..c37e2025f --- /dev/null +++ b/src/app_sparkd/sparkd/md/ReplayMarketDataProvider.h @@ -0,0 +1,28 @@ +#pragma once + +#include "spark/api/md/MarketDataConfig.pb.h" +#include "spark/persistence/IDGenerator.h" +#include "sparkd/md/IMarketDataProvider.h" + +#include + +namespace bts::md { + +class ReplayMarketDataProvider : public IMarketDataProvider { + public: + ReplayMarketDataProvider(); + explicit ReplayMarketDataProvider( + api::md::MarketDataConfig config, + std::shared_ptr sec_id_store); + ~ReplayMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/ReplayMarketDataSubscription.cpp b/src/app_sparkd/sparkd/md/ReplayMarketDataSubscription.cpp new file mode 100644 index 000000000..77ded9aa3 --- /dev/null +++ b/src/app_sparkd/sparkd/md/ReplayMarketDataSubscription.cpp @@ -0,0 +1,47 @@ +#include "ReplayMarketDataSubscription.h" +#include "spark/api/md/MarketDataReplayConfig.pb.h" +#include "spark/logging/logging.h" +#include "spark/md/GeneralMarketDataListener.h" +#include "spark/md/IMarketDataListener.h" +#include "spark/md/MDProducer.h" +#include "spark/utils/thread_manager.h" +#include "sparkd/md/recording/MDReplayer.h" + +#include +#include + +namespace bts::md { + +using bts::api::md::MarketDataReplayConfig; +using bts::md::recording::MDReplayer; +using std::string; + +struct ReplayMarketDataSource::PImpl { + MarketDataReplayConfig config; + + PImpl(const MarketDataReplayConfig &config) : config(config) {} +}; + +ReplayMarketDataSource::ReplayMarketDataSource(MarketDataReplayConfig config) + : me_(std::make_unique(config)) { + me_->config = std::move(config); +} + +ReplayMarketDataSource::~ReplayMarketDataSource() = default; + +void ReplayMarketDataSource::start( + std::shared_ptr listener) { + LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}", + os::get_thread_id(), os::get_thread_name()); + auto md_replayer = std::make_shared(me_->config.venue()); + std::string tid_name = "MDReplayer_" + Venue_Name(me_->config.venue()); + bts::threading::make_thread(tid_name, [&, listener, md_replayer]() { + md_replayer->start(listener); + }).detach(); + LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}", + os::get_thread_id(), os::get_thread_name()); +} + +void ReplayMarketDataSource::stop() {} + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/ReplayMarketDataSubscription.h b/src/app_sparkd/sparkd/md/ReplayMarketDataSubscription.h new file mode 100644 index 000000000..4a4b52f15 --- /dev/null +++ b/src/app_sparkd/sparkd/md/ReplayMarketDataSubscription.h @@ -0,0 +1,25 @@ +#pragma once + +#include "spark/api/md/MarketDataReplayConfig.pb.h" +#include "spark/md/IMarketDataListener.h" +#include "sparkd/md/IMarketDataSource.h" + +#include + +namespace bts::md { + +class ReplayMarketDataSource : public IMarketDataSource { + public: + ReplayMarketDataSource(); + explicit ReplayMarketDataSource(api::md::MarketDataReplayConfig config); + ~ReplayMarketDataSource(); + + void start(std::shared_ptr listener) override; + void stop() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/binance/BinanceSubscription.cpp b/src/app_sparkd/sparkd/md/binance/BinanceSubscription.cpp index 8a90d8a63..2fb60425c 100644 --- a/src/app_sparkd/sparkd/md/binance/BinanceSubscription.cpp +++ b/src/app_sparkd/sparkd/md/binance/BinanceSubscription.cpp @@ -171,7 +171,7 @@ struct BinanceSubscription::PImpl { // The session automatically starts/stops in its ctr/dtr CCApiSession session; - IMarketDataListener *listener; + std::shared_ptr listener; BinanceSecurityRequestor sec_requestor; // These aliases are useful for quickly switching the handler types (e.g., @@ -194,11 +194,10 @@ struct BinanceSubscription::PImpl { explicit PImpl( ccapi::EventDispatcher *dispatcher, BinanceMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : sec_id_store(sec_id_store), cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), - session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener), + session(dispatcher, mk_session_cfg(this->cfg), {}), sec_requestor( &session, this->criteria, @@ -209,6 +208,11 @@ struct BinanceSubscription::PImpl { LOG(md, Info, "Initializing BinanceSubscription:\n{}", ProtoPrinter::to_string(this->cfg)); sec_requestor.callback().bind<&PImpl::on_securities>(*this); + } + + void start(std::shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); sec_requestor.request(); } @@ -216,7 +220,7 @@ struct BinanceSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - depth_handlers.try_emplace(sec_desc, sec_id, listener); + depth_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert(is_new); return it->second; } @@ -225,7 +229,7 @@ struct BinanceSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - ticker_handlers.try_emplace(sec_desc, sec_id, listener); + ticker_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Ticker handler already created for '{}'", sec_desc); return it->second; @@ -242,7 +246,7 @@ struct BinanceSubscription::PImpl { } auto [it, is_new] = stats_handlers.try_emplace( hnd_id, - std::make_unique(sec_desc_id_tuples, listener)); + std::make_unique(sec_desc_id_tuples, listener.get())); bts_assert_msgf( is_new, "StatsHandler already created with id '{}'", hnd_id); return *it->second; @@ -333,10 +337,8 @@ struct BinanceSubscription::PImpl { BinanceSubscription::BinanceSubscription( ccapi::EventDispatcher *dispatcher, BinanceMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} BinanceSubscription::BinanceSubscription(BinanceSubscription &&) noexcept = default; @@ -345,4 +347,8 @@ BinanceSubscription &BinanceSubscription::operator=( BinanceSubscription::~BinanceSubscription() = default; +void BinanceSubscription::start(std::shared_ptr listener) { + me_->start(listener); +} + } // namespace bts::md::binance diff --git a/src/app_sparkd/sparkd/md/binance/BinanceSubscription.h b/src/app_sparkd/sparkd/md/binance/BinanceSubscription.h index be94be9a4..873da260d 100644 --- a/src/app_sparkd/sparkd/md/binance/BinanceSubscription.h +++ b/src/app_sparkd/sparkd/md/binance/BinanceSubscription.h @@ -1,5 +1,6 @@ #pragma once +#include "sparkd/md/IMarketDataSource.h" #include namespace ccapi { @@ -25,13 +26,12 @@ namespace bts::md::binance { * * @a https://binance-docs.github.io/apidocs/spot/en/#exchange-information */ -class BinanceSubscription { +class BinanceSubscription : public IMarketDataSource { public: BinanceSubscription( ccapi::EventDispatcher *dispatcher, api::md::BinanceMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); BinanceSubscription(const BinanceSubscription &) noexcept = delete; BinanceSubscription(BinanceSubscription &&) noexcept; @@ -39,10 +39,9 @@ class BinanceSubscription { delete; BinanceSubscription &operator=(BinanceSubscription &&) noexcept; - ~BinanceSubscription(); + ~BinanceSubscription() override; - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.cpp b/src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.cpp index 9570c3bd9..ea1b07fbf 100644 --- a/src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.cpp +++ b/src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.cpp @@ -47,6 +47,7 @@ using bts::io::ProtoPrinter; using bts::io::QueryParams; using bts::persistence::IDGenerator; using std::pair; +using std::shared_ptr; using std::string; using std::tuple; using std::uint32_t; @@ -174,7 +175,7 @@ struct BinanceUsSubscription::PImpl { // The session automatically starts/stops in its ctr/dtr CCApiSession session; - IMarketDataListener *listener; + shared_ptr listener; BinanceUsSecurityRequestor sec_requestor; // These aliases are useful for quickly switching the handler types (e.g., @@ -191,17 +192,16 @@ struct BinanceUsSubscription::PImpl { // different map, 'stats_handlers' (unique ids are generated w/ a counter, // 'stats_handlers_id_counter'). unordered_map sec_stats_handler_map; - unordered_map> stats_handlers; + unordered_map> stats_handlers; uint32_t stats_handlers_id_counter = 0; explicit PImpl( ccapi::EventDispatcher *dispatcher, BinanceUsMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : sec_id_store(sec_id_store), cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), - session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener), + session(dispatcher, mk_session_cfg(this->cfg), {}), sec_requestor( &session, this->criteria, @@ -212,6 +212,11 @@ struct BinanceUsSubscription::PImpl { LOG(md, Info, "Initializing BinanceUsSubscription:\n{}", ProtoPrinter::to_string(this->cfg)); sec_requestor.callback().bind<&PImpl::on_securities>(*this); + } + + void start(std::shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); sec_requestor.request(); } @@ -219,7 +224,7 @@ struct BinanceUsSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - depth_handlers.try_emplace(sec_desc, sec_id, listener); + depth_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert(is_new); return it->second; } @@ -228,7 +233,7 @@ struct BinanceUsSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - ticker_handlers.try_emplace(sec_desc, sec_id, listener); + ticker_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Ticker handler already created for '{}'", sec_desc); return it->second; @@ -245,7 +250,7 @@ struct BinanceUsSubscription::PImpl { } auto [it, is_new] = stats_handlers.try_emplace( hnd_id, - std::make_unique(sec_desc_id_tuples, listener)); + std::make_unique(sec_desc_id_tuples, listener.get())); bts_assert_msgf( is_new, "StatsHandler already created with id '{}'", hnd_id); return *it->second; @@ -335,10 +340,8 @@ struct BinanceUsSubscription::PImpl { BinanceUsSubscription::BinanceUsSubscription( ccapi::EventDispatcher *dispatcher, BinanceUsMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} BinanceUsSubscription::BinanceUsSubscription( BinanceUsSubscription &&) noexcept = default; @@ -347,4 +350,8 @@ BinanceUsSubscription &BinanceUsSubscription::operator=( BinanceUsSubscription::~BinanceUsSubscription() = default; +void BinanceUsSubscription::start(shared_ptr listener) { + me_->start(std::move(listener)); +} + } // namespace bts::md::binance_us diff --git a/src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.h b/src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.h index 12755cb82..ab40764ed 100644 --- a/src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.h +++ b/src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.h @@ -1,5 +1,6 @@ #pragma once +#include "sparkd/md/IMarketDataSource.h" #include namespace ccapi { @@ -25,13 +26,12 @@ namespace bts::md::binance_us { * * @a https://docs.binance.us/#get-exchange-information */ -class BinanceUsSubscription { +class BinanceUsSubscription : public IMarketDataSource { public: BinanceUsSubscription( ccapi::EventDispatcher *dispatcher, api::md::BinanceUsMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); BinanceUsSubscription(const BinanceUsSubscription &) noexcept = delete; BinanceUsSubscription(BinanceUsSubscription &&) noexcept; @@ -41,8 +41,7 @@ class BinanceUsSubscription { ~BinanceUsSubscription(); - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp b/src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp index 844f12c58..da03901f5 100644 --- a/src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp +++ b/src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp @@ -41,6 +41,7 @@ using bts::io::ProtoPrinter; using bts::io::QueryParams; using bts::persistence::IDGenerator; using std::pair; +using std::shared_ptr; using std::string; using std::uint32_t; using std::unique_ptr; @@ -167,7 +168,7 @@ struct BinanceUsdsFuturesSubscription::PImpl { SecurityCriteria criteria; // The session automatically starts/stops in its ctr/dtr CCApiSession session; - IMarketDataListener *listener; + std::shared_ptr listener; BinanceUsdsFuturesSecurityRequestor sec_requestor; // These aliases are useful for quickly switching the handler types (e.g., @@ -183,15 +184,19 @@ struct BinanceUsdsFuturesSubscription::PImpl { explicit PImpl( ccapi::EventDispatcher *dispatcher, BinanceUsdsFuturesMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : sec_id_store(sec_id_store), cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), - session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener), + session(dispatcher, mk_session_cfg(this->cfg), {}), sec_requestor(&session, this->cfg, sec_id_store) { LOG(md, Info, "Initializing BinanceUsdsFuturesSubscription:\n{}", ProtoPrinter::to_string(this->cfg)); sec_requestor.callback().bind<&PImpl::on_securities>(*this); + } + + void start(shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); sec_requestor.request(); } @@ -218,7 +223,7 @@ struct BinanceUsdsFuturesSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = - depth_handlers.try_emplace(sec_desc, sec_id, listener); + depth_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Depth handler already created for '{}'", sec_desc); return it->second; @@ -228,7 +233,7 @@ struct BinanceUsdsFuturesSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = - ticker_handlers.try_emplace(sec_desc, sec_id, listener); + ticker_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Ticker handler already created for '{}'", sec_desc); return it->second; @@ -238,7 +243,7 @@ struct BinanceUsdsFuturesSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = - stats_handlers.try_emplace(sec_desc, sec_id, listener); + stats_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Stats handler already created for '{}'", sec_desc); return it->second; @@ -310,10 +315,8 @@ struct BinanceUsdsFuturesSubscription::PImpl { BinanceUsdsFuturesSubscription::BinanceUsdsFuturesSubscription( ccapi::EventDispatcher *dispatcher, BinanceUsdsFuturesMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} BinanceUsdsFuturesSubscription::BinanceUsdsFuturesSubscription( BinanceUsdsFuturesSubscription &&) noexcept = default; @@ -322,4 +325,9 @@ BinanceUsdsFuturesSubscription &BinanceUsdsFuturesSubscription::operator=( BinanceUsdsFuturesSubscription::~BinanceUsdsFuturesSubscription() = default; +void BinanceUsdsFuturesSubscription::start( + shared_ptr listener) { + me_->start(std::move(listener)); +} + } // namespace bts::md::binance_usds_futures diff --git a/src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h b/src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h index 519e56383..e2a477120 100644 --- a/src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h +++ b/src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h @@ -1,5 +1,6 @@ #pragma once +#include "sparkd/md/IMarketDataSource.h" #include namespace ccapi { @@ -25,13 +26,12 @@ namespace bts::md::binance_usds_futures { * * @a https://binance-docs.github.io/apidocs/futures/en/#exchange-information */ -class BinanceUsdsFuturesSubscription { +class BinanceUsdsFuturesSubscription : public IMarketDataSource { public: BinanceUsdsFuturesSubscription( ccapi::EventDispatcher *dispatcher, api::md::BinanceUsdsFuturesMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); BinanceUsdsFuturesSubscription( const BinanceUsdsFuturesSubscription &) noexcept = delete; @@ -43,8 +43,7 @@ class BinanceUsdsFuturesSubscription { ~BinanceUsdsFuturesSubscription(); - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/app_sparkd/sparkd/md/ccapi/CCApiMarketDataProvider.cpp b/src/app_sparkd/sparkd/md/ccapi/CCApiMarketDataProvider.cpp new file mode 100644 index 000000000..f3d3d7f63 --- /dev/null +++ b/src/app_sparkd/sparkd/md/ccapi/CCApiMarketDataProvider.cpp @@ -0,0 +1,140 @@ +#include "CCApiMarketDataProvider.h" +#include "spark/api/domain/Venue.pb.h" +#include "spark/api/md/MarketDataConfig.pb.h" +#include "spark/config/config.h" +#include "spark/io/ProtoPrinter.h" +#include "spark/logging/logging.h" +#include "spark/md/GeneralMarketDataListener.h" +#include "spark/md/GeneralSecurityIndex.h" +#include "spark/md/MDProducer.h" +#include "spark/persistence/IDGenerator.h" +#include "sparkd/md/MarketDataSubscription.h" +#include "sparkd/md/binance/BinanceSubscription.h" +#include "sparkd/md/binance_us/BinanceUsSubscription.h" +#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h" +#include "sparkd/md/coinbase/CoinbaseSubscription.h" +#include "sparkd/md/deribit/DeribitSubscription.h" +#include "sparkd/md/ftx/FTXSubscription.h" +#include "sparkd/md/okx/OkxSubscription.h" + +#include + +namespace bts::md { + +using bts::api::md::MarketDataConfig; +using bts::persistence::IDGenerator; +using std::pair; +using std::shared_ptr; +using std::string; +using std::unique_ptr; +using std::unordered_map; +using std::vector; + +static bool validate( + const MarketDataConfig & /*cfg*/, vector & /*errs*/) { + return true; +} + +struct CCApiMarketDataProvider::PImpl { + struct VenueSink { + GeneralSecurityIndex sec_index; + MDProducer md_producer; + GeneralMarketDataListener listener; + + VenueSink(api::Venue venue, shared_ptr sec_id_store) + : sec_index(), + listener(sec_index, md_producer, venue, sec_id_store.get()) {} + }; + + MarketDataConfig config; + ccapi::EventDispatcher dispatcher; + shared_ptr sec_id_store; + unordered_map> venue_sinks; + vector>> + sources; + + PImpl(MarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} + + VenueSink *get_venue_sink(api::Venue venue) { + auto it = venue_sinks.find(venue); + if (it != venue_sinks.end()) { + return it->second.get(); + } + return venue_sinks + .try_emplace(venue, std::make_unique(venue, sec_id_store)) + .first->second.get(); + } +}; + +CCApiMarketDataProvider::CCApiMarketDataProvider( + MarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { +} + +CCApiMarketDataProvider::~CCApiMarketDataProvider() = default; + +vector>> +CCApiMarketDataProvider::get_sources() { + auto &cfg = me_->config; + vector cfg_errs; + if (!validate(cfg, cfg_errs)) { + LOG(md, Error, + "CCApiMarketDataProvider::get_sources failed. Invalid config: {}", + fmt::to_string(fmt::join(cfg_errs, "\n"))); + } + + // Binance + for (const auto &binance : cfg.binance()) { + me_->sources.emplace_back( + api::domain::VENUE_BINANCE, + std::make_shared( + &me_->dispatcher, binance, me_->sec_id_store.get())); + } + // Binance US + for (const auto &binance_us : cfg.binance_us()) { + me_->sources.emplace_back( + api::domain::VENUE_BINANCE_US, + std::make_shared( + &me_->dispatcher, binance_us, me_->sec_id_store.get())); + } + // Binance USDS Futures + for (const auto &binance_ufs : cfg.binance_usds_futures()) { + me_->sources.emplace_back( + api::domain::VENUE_BINANCE_USDS_FUTURES, + std::make_shared< + binance_usds_futures::BinanceUsdsFuturesSubscription>( + &me_->dispatcher, binance_ufs, me_->sec_id_store.get())); + } + // Coinbase + for (const auto &coinbase : cfg.coinbase()) { + me_->sources.emplace_back( + api::domain::VENUE_COINBASE, + std::make_shared( + &me_->dispatcher, coinbase, me_->sec_id_store.get())); + } + // Deribit + for (const auto &deribit : cfg.deribit()) { + me_->sources.emplace_back( + api::domain::VENUE_DERIBIT, + std::make_shared( + &me_->dispatcher, deribit, me_->sec_id_store.get())); + } + // FTX + for (const auto &ftx : cfg.ftx()) { + me_->sources.emplace_back( + api::domain::VENUE_FTX, + std::make_shared( + &me_->dispatcher, ftx, me_->sec_id_store.get())); + } + // Okx + for (const auto &okx : cfg.okx()) { + me_->sources.emplace_back( + api::domain::VENUE_OKX, + std::make_shared( + &me_->dispatcher, okx, me_->sec_id_store.get())); + } + return me_->sources; +} + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/ccapi/CCApiMarketDataProvider.h b/src/app_sparkd/sparkd/md/ccapi/CCApiMarketDataProvider.h new file mode 100644 index 000000000..4fb81eb5c --- /dev/null +++ b/src/app_sparkd/sparkd/md/ccapi/CCApiMarketDataProvider.h @@ -0,0 +1,27 @@ +#pragma once + +#include "spark/api/md/MarketDataConfig.pb.h" +#include "spark/persistence/IDGenerator.h" +#include "sparkd/md/IMarketDataProvider.h" + +#include + +namespace bts::md { + +class CCApiMarketDataProvider : public IMarketDataProvider { + public: + explicit CCApiMarketDataProvider( + api::md::MarketDataConfig config, + std::shared_ptr sec_id_store); + ~CCApiMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/coinbase/CoinbaseSubscription.cpp b/src/app_sparkd/sparkd/md/coinbase/CoinbaseSubscription.cpp index b26c5545b..6ea148308 100644 --- a/src/app_sparkd/sparkd/md/coinbase/CoinbaseSubscription.cpp +++ b/src/app_sparkd/sparkd/md/coinbase/CoinbaseSubscription.cpp @@ -5,8 +5,10 @@ #include "spark/domain/SecurityCriteria.h" #include "spark/io/ProtoPrinter.h" #include "spark/io/QueryParams.h" +#include "spark/md/IMarketDataListener.h" #include "spark/md/common.h" #include "spark/persistence/IDGenerator.h" +#include "spark/utils/bts_assert.h" #include "sparkd/md/ccapi/CCApiMarketDepthAdaptor.h" #include "sparkd/md/ccapi/CCApiSession.h" #include "sparkd/md/ccapi/CCApiTradeAdaptor.h" @@ -36,6 +38,7 @@ using bts::io::ProtoPrinter; using bts::io::QueryParams; using bts::persistence::IDGenerator; using std::pair; +using std::shared_ptr; using std::string; using std::unique_ptr; using std::unordered_map; @@ -151,7 +154,7 @@ struct CoinbaseSubscription::PImpl { // The session automatically starts/stops in its ctr/dtr CCApiSession session; - IMarketDataListener *listener; + shared_ptr listener; CoinbaseSecurityRequestor sec_requestor; // These aliases are useful for quickly switching the handler types (e.g., @@ -172,11 +175,10 @@ struct CoinbaseSubscription::PImpl { explicit PImpl( ccapi::EventDispatcher *dispatcher, CoinbaseMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : sec_id_store(sec_id_store), cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), - session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener), + session(dispatcher, mk_session_cfg(this->cfg), {}), sec_requestor( &session, this->criteria, @@ -187,6 +189,11 @@ struct CoinbaseSubscription::PImpl { LOG(md, Info, "Initializing CoinbaseSubscription:\n{}", ProtoPrinter::to_string(this->cfg)); sec_requestor.callback().bind<&PImpl::on_securities>(*this); + } + + void start(shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); sec_requestor.request(); } @@ -195,7 +202,7 @@ struct CoinbaseSubscription::PImpl { auto sec_id = SecurityID{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - stats_handlers.try_emplace(sec_desc, sec_id, listener); + stats_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "StatsHandler already created with id '{}'", sec_id); return it->second; @@ -204,7 +211,7 @@ struct CoinbaseSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - depth_handlers.try_emplace(sec_desc, sec_id, listener); + depth_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert(is_new); return it->second; } @@ -213,7 +220,7 @@ struct CoinbaseSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - ticker_handlers.try_emplace(sec_desc, sec_id, listener); + ticker_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Ticker handler already created for '{}'", sec_desc); return it->second; @@ -285,10 +292,8 @@ struct CoinbaseSubscription::PImpl { CoinbaseSubscription::CoinbaseSubscription( ccapi::EventDispatcher *dispatcher, CoinbaseMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} CoinbaseSubscription::CoinbaseSubscription(CoinbaseSubscription &&) noexcept = default; @@ -297,4 +302,8 @@ CoinbaseSubscription &CoinbaseSubscription::operator=( CoinbaseSubscription::~CoinbaseSubscription() = default; +void CoinbaseSubscription::start(shared_ptr listener) { + me_->start(std::move(listener)); +} + } // namespace bts::md::coinbase diff --git a/src/app_sparkd/sparkd/md/coinbase/CoinbaseSubscription.h b/src/app_sparkd/sparkd/md/coinbase/CoinbaseSubscription.h index 999a55cab..2a8a512d3 100644 --- a/src/app_sparkd/sparkd/md/coinbase/CoinbaseSubscription.h +++ b/src/app_sparkd/sparkd/md/coinbase/CoinbaseSubscription.h @@ -1,5 +1,6 @@ #pragma once +#include "sparkd/md/IMarketDataSource.h" #include namespace ccapi { @@ -20,13 +21,12 @@ class IDGenerator; namespace bts::md::coinbase { -class CoinbaseSubscription { +class CoinbaseSubscription : public IMarketDataSource { public: CoinbaseSubscription( ccapi::EventDispatcher *dispatcher, api::md::CoinbaseMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); CoinbaseSubscription(const CoinbaseSubscription &) noexcept = delete; CoinbaseSubscription(CoinbaseSubscription &&) noexcept; @@ -36,8 +36,7 @@ class CoinbaseSubscription { ~CoinbaseSubscription(); - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/app_sparkd/sparkd/md/databento/DatabentoMarketDataProvider.cpp b/src/app_sparkd/sparkd/md/databento/DatabentoMarketDataProvider.cpp new file mode 100644 index 000000000..25f1c34bd --- /dev/null +++ b/src/app_sparkd/sparkd/md/databento/DatabentoMarketDataProvider.cpp @@ -0,0 +1,51 @@ +#include "DatabentoMarketDataProvider.h" +#include "spark/api/domain/Venue.pb.h" +#include "spark/api/md/MarketDataConfig.pb.h" +#include "spark/persistence/IDGenerator.h" +#include "sparkd/md/IMarketDataSource.h" +#include "sparkd/md/databento/DatabentoDataHandler.h" +#include "sparkd/md/glbx/DatabentoGlbxSubscription.h" + +#include +#include + +namespace bts::md { + +using bts::api::md::MarketDataConfig; +using bts::persistence::IDGenerator; +using std::pair; +using std::shared_ptr; +using std::string; +using std::vector; + +struct DatabentoMarketDataProvider::PImpl { + MarketDataConfig config; + shared_ptr sec_id_store; + vector> glbx_sources; + + PImpl(MarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} +}; + +DatabentoMarketDataProvider::DatabentoMarketDataProvider( + MarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { + me_->config = std::move(config); +} + +DatabentoMarketDataProvider::~DatabentoMarketDataProvider() = default; + +vector>> +DatabentoMarketDataProvider::get_sources() { + vector>> rv; + for (const auto &glbx : me_->config.glbx()) { + if (glbx.use_databento()) { + me_->glbx_sources.push_back( + std::make_shared(glbx, me_->sec_id_store)); + rv.push_back({api::domain::VENUE_GLBX, me_->glbx_sources.back()}); + } + } + return rv; +} + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/databento/DatabentoMarketDataProvider.h b/src/app_sparkd/sparkd/md/databento/DatabentoMarketDataProvider.h new file mode 100644 index 000000000..793e90711 --- /dev/null +++ b/src/app_sparkd/sparkd/md/databento/DatabentoMarketDataProvider.h @@ -0,0 +1,33 @@ +#pragma once + +#include "sparkd/md/IMarketDataProvider.h" + +#include + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::api::md { +class MarketDataConfig; +} + +namespace bts::md { + +class DatabentoMarketDataProvider : public IMarketDataProvider { + public: + explicit DatabentoMarketDataProvider( + api::md::MarketDataConfig config, + std::shared_ptr sec_id_store); + ~DatabentoMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/deribit/DeribitSubscription.cpp b/src/app_sparkd/sparkd/md/deribit/DeribitSubscription.cpp index fbb8ab106..dfd5b40a6 100644 --- a/src/app_sparkd/sparkd/md/deribit/DeribitSubscription.cpp +++ b/src/app_sparkd/sparkd/md/deribit/DeribitSubscription.cpp @@ -52,6 +52,7 @@ using bts::io::websocket::JsonRpcId; using bts::io::websocket::JsonRpcRequest; using bts::persistence::IDGenerator; using nlohmann::json; +using std::shared_ptr; using std::string; using std::uint32_t; using std::unique_ptr; @@ -134,14 +135,13 @@ struct DeribitSubscription::PImpl { const bool subscribe_trades = true; size_t sub_id_counter = 0; CCApiSession session; - IMarketDataListener *listener; + shared_ptr listener; DeribitChannelDemuxer channel_demuxer; explicit PImpl( ccapi::EventDispatcher *dispatcher, DeribitMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), excluded_secs( @@ -159,11 +159,15 @@ struct DeribitSubscription::PImpl { criteria, &instrument_store, {this->cfg.currencies().begin(), this->cfg.currencies().end()}), - session(dispatcher, mk_session_cfg(this->cfg), {}), - listener(listener) { + session(dispatcher, mk_session_cfg(this->cfg), {}) { // DON'T reference the ctr arg 'cfg' (it has been moved to 'this->cfg') _INFO("ctr. config:\n{}", ProtoPrinter::to_string(this->cfg)); sec_requestor.callback().bind<&PImpl::on_securities>(*this); + } + + void start(std::shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); sec_requestor.request_all(); subscribe_instrument_state(); } @@ -172,7 +176,7 @@ struct DeribitSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = security_data_map.try_emplace( - sec_desc, std::make_unique(sid, listener)); + sec_desc, std::make_unique(sid, listener.get())); _ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc); _INFO("add_security_data. sec: '{}'", sec_desc); return *it->second; @@ -303,10 +307,8 @@ struct DeribitSubscription::PImpl { DeribitSubscription::DeribitSubscription( ccapi::EventDispatcher *dispatcher, DeribitMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} DeribitSubscription::DeribitSubscription(DeribitSubscription &&) noexcept = default; @@ -315,4 +317,8 @@ DeribitSubscription &DeribitSubscription::operator=( DeribitSubscription::~DeribitSubscription() = default; +void DeribitSubscription::start(shared_ptr listener) { + me_->start(std::move(listener)); +} + } // namespace bts::md::deribit diff --git a/src/app_sparkd/sparkd/md/deribit/DeribitSubscription.h b/src/app_sparkd/sparkd/md/deribit/DeribitSubscription.h index c8f41ec6e..0deea6ab0 100644 --- a/src/app_sparkd/sparkd/md/deribit/DeribitSubscription.h +++ b/src/app_sparkd/sparkd/md/deribit/DeribitSubscription.h @@ -1,5 +1,6 @@ #pragma once +#include "sparkd/md/IMarketDataSource.h" #include namespace ccapi { @@ -24,13 +25,12 @@ namespace bts::md::deribit { * @brief Subscribes to market data at Deribit * @a https://docs.deribit.com/?#market-data */ -class DeribitSubscription { +class DeribitSubscription : public IMarketDataSource { public: DeribitSubscription( ccapi::EventDispatcher *dispatcher, api::md::DeribitMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); DeribitSubscription(const DeribitSubscription &) noexcept = delete; DeribitSubscription(DeribitSubscription &&) noexcept; @@ -40,9 +40,8 @@ class DeribitSubscription { ~DeribitSubscription(); - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); - + void start(std::shared_ptr listener) override; + private: struct PImpl; std::unique_ptr me_; diff --git a/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitSubscription.cpp b/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitSubscription.cpp new file mode 100644 index 000000000..334fa09cf --- /dev/null +++ b/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitSubscription.cpp @@ -0,0 +1,77 @@ +#include "EmumbaDeribitSubscription.h" +#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h" +#include "spark/config/protos/system_config.pb.h" +#include "spark/utils/thread_manager.h" +#include "sparkd/md/emumba/EmumbaDeribitData.h" + +namespace bts::md { +using bts::config::SystemConfig; + +using Path = boost::filesystem::path; + +struct EmumbaDeribitSubscription::PImpl { + api::md::EmumbaDeribitMarketDataConfig config; +}; + +EmumbaDeribitSubscription::EmumbaDeribitSubscription( + api::md::EmumbaDeribitMarketDataConfig config) + : me_(std::make_unique()) { + me_->config = std::move(config); +} + +EmumbaDeribitSubscription::~EmumbaDeribitSubscription() = default; + +void EmumbaDeribitSubscription::start( + std::shared_ptr /*listener*/) { + auto &cfg = me_->config; + LOG(md, Info, "EmumbaMarketDataProvider::start t_id:{} t_name:{}. cfg:\n{}", + os::get_thread_id(), os::get_thread_name(), cfg.DebugString()); + auto start_deribit_thr = [=]() { + const auto config_path = Path{config::get().config_path()} / + cfg.json_conf_file_name(); + const auto conf = + bts::cmds::DeribitDemoConfiguration::read_file(config_path.string()); + bts_assert_msg(conf.has_value(), "Failed to parse deribit config"); + // log + for (const auto &s : conf.value().symbol_config_list) { + LOG(md, Info, "sub.symbol: {}", s.symbol_name); + LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future, + s.interested_in_option); + for (const auto &i : s.interested_instruments) { + LOG(md, Info, "sub.instrument: {}", i); + } + } + EmumbaDeribitData ddata_obj; + EmumbaDeribitData *ddata = &ddata_obj; + // end log + bts_assert(conf.has_value()); + LOG(md, Info, "Deribit conf: {}", conf.value()); + ddata->real_handler = + std::make_shared( + conf.value().symbol_config_list, conf.value().trading_config, + conf.value().credential_map, conf.value().logging_configuration, + conf.value().deribit_url, &ddata->listener); + // since the trading interface is housed inside the EmMDListener, we need to + // create and send emumba's trading_session_interface + for (const auto &p : conf.value().credential_map) { + const auto &sess = p.second; + auto ts_ptr = std::make_shared< + emumba::deribit_connector::trading_session_interface>( + ddata->real_handler->create_new_trading_session(sess)); + ddata->trading_session_vec.push_back({sess, ts_ptr}); + } + bts_assert(!ddata->trading_session_vec.empty()); + ddata->listener.set_handler( + ddata->real_handler.get(), ddata->trading_session_vec); + ddata->real_handler->run(); + }; + threading::make_thread("Emumba", [&]() { start_deribit_thr(); }).detach(); + LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}", + os::get_thread_id(), os::get_thread_name()); +} + +void EmumbaDeribitSubscription::stop() { + // +} + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitSubscription.h b/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitSubscription.h new file mode 100644 index 000000000..f977beb8b --- /dev/null +++ b/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitSubscription.h @@ -0,0 +1,24 @@ +#pragma once + +#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h" +#include "sparkd/md/IMarketDataSource.h" + +#include + +namespace bts::md { + +class EmumbaDeribitSubscription : public IMarketDataSource { + public: + explicit EmumbaDeribitSubscription( + api::md::EmumbaDeribitMarketDataConfig config); + ~EmumbaDeribitSubscription() override; + + void start(std::shared_ptr listener) override; + void stop() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/emumba/EmumbaMarketDataProvider.cpp b/src/app_sparkd/sparkd/md/emumba/EmumbaMarketDataProvider.cpp new file mode 100644 index 000000000..8645340b5 --- /dev/null +++ b/src/app_sparkd/sparkd/md/emumba/EmumbaMarketDataProvider.cpp @@ -0,0 +1,40 @@ +#include "EmumbaMarketDataProvider.h" +#include "spark/api/md/MarketDataConfig.pb.h" +#include "spark/api/md/MarketDataConfigUtils.h" +#include "sparkd/md/IMarketDataSource.h" +#include "sparkd/md/emumba/EmumbaDeribitSubscription.h" + +#include +#include + +namespace bts::md { + +using bts::api::md::MarketDataConfig; +using std::pair; +using std::shared_ptr; +using std::string; +using std::vector; + +struct EmumbaMarketDataProvider::PImpl { + MarketDataConfig config; + vector>> deribit_subs; + + PImpl(MarketDataConfig config) : config(std::move(config)) {} +}; + +EmumbaMarketDataProvider::EmumbaMarketDataProvider(MarketDataConfig config) + : me_(std::make_unique(std::move(config))) {} + +EmumbaMarketDataProvider::~EmumbaMarketDataProvider() = default; + +vector>> +EmumbaMarketDataProvider::get_sources() { + for (const auto &cfg : me_->config.emumba_deribit()) { + me_->deribit_subs.push_back( + {api::domain::VENUE_DERIBIT, + std::make_shared(cfg)}); + } + return me_->deribit_subs; +} + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/emumba/EmumbaMarketDataProvider.h b/src/app_sparkd/sparkd/md/emumba/EmumbaMarketDataProvider.h new file mode 100644 index 000000000..ddd60370b --- /dev/null +++ b/src/app_sparkd/sparkd/md/emumba/EmumbaMarketDataProvider.h @@ -0,0 +1,25 @@ +#pragma once + +#include "spark/api/md/MarketDataConfig.pb.h" +#include "sparkd/md/IMarketDataProvider.h" + +#include + +namespace bts::md { + +class EmumbaMarketDataProvider : public IMarketDataProvider { + public: + EmumbaMarketDataProvider(); + explicit EmumbaMarketDataProvider(api::md::MarketDataConfig config); + ~EmumbaMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/ftx/FTXSubscription.cpp b/src/app_sparkd/sparkd/md/ftx/FTXSubscription.cpp index 7cc3adad3..1e6e1178b 100644 --- a/src/app_sparkd/sparkd/md/ftx/FTXSubscription.cpp +++ b/src/app_sparkd/sparkd/md/ftx/FTXSubscription.cpp @@ -38,6 +38,7 @@ using bts::domain::SecurityID; using bts::io::ProtoPrinter; using bts::persistence::IDGenerator; using std::pair; +using std::shared_ptr; using std::string; using std::unique_ptr; using std::unordered_map; @@ -105,7 +106,7 @@ struct FTXSubscription::PImpl { FTXSecurityRequestor sec_requestor; // The session automatically starts/stops in its ctr/dtr CCApiSession session; - IMarketDataListener *listener; + shared_ptr listener; // These aliases are useful for quickly switching the handler types (e.g., // a special handler that logs messages for testing/diagnostic purposes) @@ -120,16 +121,20 @@ struct FTXSubscription::PImpl { explicit PImpl( ccapi::EventDispatcher *dispatcher, FTXMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener) + persistence::IDGenerator *sec_id_store) : sec_id_store(sec_id_store), cfg(std::move(cfg)), criteria(domain::SecurityCriteria::from_proto( this->cfg.security_criteria())), - sec_requestor(dispatcher, criteria, sec_id_store), session(dispatcher), - listener(listener) { + sec_requestor(dispatcher, criteria, sec_id_store), + session(dispatcher) { LOG(md, Info, "Initializing FTXSubscription:\n{}", ProtoPrinter::to_string(this->cfg)); sec_requestor.callback().bind<&PImpl::on_securities>(*this); + } + + void start(shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = std::move(listener); sec_requestor.request(); } @@ -137,7 +142,7 @@ struct FTXSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - depth_handlers.try_emplace(sec_desc, sec_id, listener); + depth_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert(is_new); return it->second; } @@ -146,7 +151,7 @@ struct FTXSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - ticker_handlers.try_emplace(sec_desc, sec_id, listener); + ticker_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Ticker handler already created for '{}'", sec_desc); return it->second; @@ -156,7 +161,7 @@ struct FTXSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = - stats_handlers.try_emplace(sec_desc, sec_id, listener); + stats_handlers.try_emplace(sec_desc, sec_id, listener.get()); bts_assert_msgf( is_new, "Statistics handler already created for '{}'", sec_desc); return it->second; @@ -228,10 +233,8 @@ struct FTXSubscription::PImpl { FTXSubscription::FTXSubscription( ccapi::EventDispatcher *dispatcher, FTXMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} FTXSubscription::FTXSubscription(FTXSubscription &&) noexcept = default; FTXSubscription &FTXSubscription::operator=(FTXSubscription &&) noexcept = @@ -239,4 +242,8 @@ FTXSubscription &FTXSubscription::operator=(FTXSubscription &&) noexcept = FTXSubscription::~FTXSubscription() = default; +void FTXSubscription::start(shared_ptr listener) { + me_->start(std::move(listener)); +} + } // namespace bts::md::ftx diff --git a/src/app_sparkd/sparkd/md/ftx/FTXSubscription.h b/src/app_sparkd/sparkd/md/ftx/FTXSubscription.h index a9d3d58cd..9c547fcf3 100644 --- a/src/app_sparkd/sparkd/md/ftx/FTXSubscription.h +++ b/src/app_sparkd/sparkd/md/ftx/FTXSubscription.h @@ -1,5 +1,6 @@ #pragma once +#include "sparkd/md/IMarketDataSource.h" #include namespace ccapi { @@ -25,20 +26,21 @@ namespace bts::md::ftx { * * @details Uses ccapi infrastructure to request/subscribe data */ -class FTXSubscription { +class FTXSubscription : public IMarketDataSource { public: FTXSubscription( ccapi::EventDispatcher *dispatcher, api::md::FTXMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); FTXSubscription(const FTXSubscription &) noexcept = delete; FTXSubscription(FTXSubscription &&) noexcept; FTXSubscription &operator=(const FTXSubscription &) noexcept = delete; FTXSubscription &operator=(FTXSubscription &&) noexcept; - ~FTXSubscription(); + ~FTXSubscription() override; + + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/app_sparkd/sparkd/md/glbx/DatabentoGlbxSubscription.cpp b/src/app_sparkd/sparkd/md/glbx/DatabentoGlbxSubscription.cpp new file mode 100644 index 000000000..7bf905ad9 --- /dev/null +++ b/src/app_sparkd/sparkd/md/glbx/DatabentoGlbxSubscription.cpp @@ -0,0 +1,41 @@ +#include "DatabentoGlbxSubscription.h" +#include "spark/api/md/GlbxMarketDataConfig.pb.h" +#include "spark/utils/thread_manager.h" +#include "sparkd/md/databento/DatabentoDataHandler.h" +#include "sparkd/md/glbx/GlbxMarketDataHandler.h" + +namespace bts::md { + +using bts::api::md::GlbxMarketDataConfig; +using bts::persistence::IDGenerator; +using std::shared_ptr; + +struct DatabentoGlbxSubscription::PImpl { + GlbxMarketDataConfig config; + shared_ptr sec_id_store; + + PImpl(GlbxMarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} +}; + +DatabentoGlbxSubscription::DatabentoGlbxSubscription( + GlbxMarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { +} + +DatabentoGlbxSubscription::~DatabentoGlbxSubscription() = default; + +void DatabentoGlbxSubscription::start( + std::shared_ptr listener) { + bts::threading::make_thread( + "glbx", + [config = me_->config, sec_id_store = me_->sec_id_store, + listener = std::move(listener)]() { + api::md::glbx::GlbxMarketDatahandler::run( + std::move(listener), std::move(sec_id_store), config); + }) + .detach(); + // TODO: implement 'stop' (signal the thread above to terminate, then join it). +} + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/glbx/DatabentoGlbxSubscription.h b/src/app_sparkd/sparkd/md/glbx/DatabentoGlbxSubscription.h new file mode 100644 index 000000000..a2971b079 --- /dev/null +++ b/src/app_sparkd/sparkd/md/glbx/DatabentoGlbxSubscription.h @@ -0,0 +1,47 @@ +#pragma once + +#include "sparkd/md/IMarketDataSource.h" + +#include + +namespace bts::api::md { +class GlbxMarketDataConfig; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::md { + +/** + * @brief Subscribes to market data at Glbx via Databento + * + * @a https://www.Databento.biz/ + */ +class DatabentoGlbxSubscription : public IMarketDataSource { + public: + DatabentoGlbxSubscription( + api::md::GlbxMarketDataConfig config, + std::shared_ptr sec_id_store); + DatabentoGlbxSubscription(const DatabentoGlbxSubscription &) noexcept = + delete; + DatabentoGlbxSubscription(DatabentoGlbxSubscription &&) noexcept; + DatabentoGlbxSubscription &operator=( + const DatabentoGlbxSubscription &) noexcept = delete; + DatabentoGlbxSubscription &operator=(DatabentoGlbxSubscription &&) noexcept; + ~DatabentoGlbxSubscription() override; + + /** + * Starts market data subscription on a background thread. + * + * @note Does not block. + */ + void start(std::shared_ptr listener) override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp b/src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp new file mode 100644 index 000000000..577bedc40 --- /dev/null +++ b/src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp @@ -0,0 +1,118 @@ +#include "OnixsGlbxSubscription.h" +#include "spark/api/md/GlbxMarketDataConfig.pb.h" +#include "spark/core/LOCK_GUARD.h" +#include "spark/logging/logging.h" +#include "spark/md/IMarketDataListener.h" +#include "spark/md/SymbolLoader.h" +#include "spark/os/os.h" +#include "spark/persistence/IDGenerator.h" +#include "spark/utils/Dispatcher.h" +#include "sparkd/md/platforms/IMDPlatform.h" +#include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h" + +#include +#include +#include +#include + +namespace bts::md { + +using bts::api::md::GlbxMarketDataConfig; +using bts::domain::SecurityCriteria; +using bts::md::IMDPlatform; +using bts::persistence::IDGenerator; +using std::shared_ptr; +using std::string; +using std::vector; + +struct OnixsGlbxSubscription::PImpl { + GlbxMarketDataConfig config; + shared_ptr sec_id_store; + + PImpl(shared_ptr sec_id_store, GlbxMarketDataConfig config) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} + + std::thread dispatcher_thread; + utils::Dispatcher *dispatcher{nullptr}; + std::optional heartbeat_timer; + bool is_running = false; + std::mutex mx; + + ~PImpl() { stop(); } + + /** + * Starts market data subscription on this thread. + * + * @note Designed to be called from a thread callback inside `OnixsGlbxSubscription::start`. + * @note Blocks until subscription stops. + */ + void run() { + { + BTS_LOCK_GUARD(mx); + if (!is_running) { + return; + } + const auto symbols = SymbolLoader::load( + SecurityCriteria::create({api::domain::VENUE_GLBX})); + if (symbols.empty()) { + LOG(md, Info, "OnixsGlbxSubscription aborting. No symbols loaded"); + return; + } + LOG(md, Info, + "OnixsGlbxSubscription starting. t_id: {}; t_name: {}; config: " + "{}; symbols: {}", + os::get_thread_id(), os::get_thread_name(), config.DebugString(), + FMT_JOIN(symbols, ", ")); + vector> platforms; + auto main = shared_ptr(); + main->start({symbols.begin(), symbols.end()}, nullptr, nullptr); + platforms.emplace_back(std::move(main)); + dispatcher = &utils::Dispatcher::current(); + heartbeat_timer.emplace(*dispatcher, 250, [&]() { + for (auto &p : platforms) { + p->send_heartbeats(); + } + }); + heartbeat_timer->start(); + is_running = true; + } + // This will block until dispatcher->terminate() is called. + // There is a slim, but non-zero chance that this is executed after + // 'stop' is called, since we 'run' outside of the lock. + dispatcher->run(); + } + + void stop() { + BTS_LOCK_GUARD(mx); + if (!is_running) { + return; + } + is_running = false; + if (dispatcher != nullptr) { + LOG(md, Diag, "OnixsGlbxSubscription stopping."); + dispatcher->io_service()->stop(); + } + if (dispatcher_thread.joinable()) { + LOG(md, Diag, "OnixsGlbxSubscription joining dispatcher thread."); + dispatcher_thread.join(); + } + } +}; + +OnixsGlbxSubscription::OnixsGlbxSubscription( + shared_ptr sec_id_store, GlbxMarketDataConfig config) + : me_(std::make_unique(std::move(sec_id_store), std::move(config))) { +} + +OnixsGlbxSubscription::~OnixsGlbxSubscription() = default; + +void OnixsGlbxSubscription::start( + shared_ptr /*listener*/) { + std::atomic_thread_fence(std::memory_order_release); + bts_assert(!me_->dispatcher_thread.joinable()); + me_->dispatcher_thread = std::thread{[&]() { me_->run(); }}; +} + +void OnixsGlbxSubscription::stop() { me_->stop(); } + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.h b/src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.h new file mode 100644 index 000000000..ba1c1db80 --- /dev/null +++ b/src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.h @@ -0,0 +1,48 @@ +#pragma once + +#include "sparkd/md/IMarketDataSource.h" + +#include + +namespace bts::api::md { +class GlbxMarketDataConfig; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::md { + +/** + * @brief Subscribes to market data at Glbx via Onixs + * + * @a https://www.onixs.biz/ + */ +class OnixsGlbxSubscription : public IMarketDataSource { + public: + OnixsGlbxSubscription( + std::shared_ptr sec_id_store, + api::md::GlbxMarketDataConfig config); + OnixsGlbxSubscription(const OnixsGlbxSubscription &) noexcept = delete; + OnixsGlbxSubscription(OnixsGlbxSubscription &&) noexcept; + OnixsGlbxSubscription &operator=(const OnixsGlbxSubscription &) noexcept = + delete; + OnixsGlbxSubscription &operator=(OnixsGlbxSubscription &&) noexcept; + ~OnixsGlbxSubscription() override; + + /** + * Starts market data subscription on a background thread. + * + * @note Does not block. + */ + void start(std::shared_ptr listener) override; + + void stop() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/okx/OkxSubscription.cpp b/src/app_sparkd/sparkd/md/okx/OkxSubscription.cpp index 5f9540f3b..98e58227a 100644 --- a/src/app_sparkd/sparkd/md/okx/OkxSubscription.cpp +++ b/src/app_sparkd/sparkd/md/okx/OkxSubscription.cpp @@ -46,6 +46,7 @@ using bts::domain::SecurityID; using bts::io::ProtoPrinter; using bts::persistence::IDGenerator; using std::optional; +using std::shared_ptr; using std::size_t; using std::string; using std::unique_ptr; @@ -136,14 +137,13 @@ struct OkxSubscription::PImpl { const bool subscribe_trades = true; size_t sub_id_counter = 0; CCApiSession session; - IMarketDataListener *listener; + shared_ptr listener; OkxChannelDemuxer channel_demuxer; explicit PImpl( ccapi::EventDispatcher *dispatcher, OkxMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) + IDGenerator *sec_id_store) : cfg(std::move(cfg)), criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())), excluded_secs( @@ -157,10 +157,16 @@ struct OkxSubscription::PImpl { !CONTAINS(excluded_secs, s.desc()); }), session( - dispatcher, mk_session_cfg(this->cfg), mk_session_opts(this->cfg)), - listener(listener) { + dispatcher, + mk_session_cfg(this->cfg), + mk_session_opts(this->cfg)) { // DON'T reference the ctr arg 'cfg' (it has been moved to 'this->cfg') _INFO("ctr. config:\n{}", ProtoPrinter::to_string(this->cfg)); + } + + void start(std::shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener = listener; sec_sub.emplace( &session, &instrument_store, this->criteria, [this](const auto &secs) { on_securities(secs); }); @@ -170,7 +176,7 @@ struct OkxSubscription::PImpl { SecurityType sec_type, const string &sec_desc) { const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = security_data_map.try_emplace( - sec_desc, std::make_unique(sid, listener)); + sec_desc, std::make_unique(sid, listener.get())); _ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc); _INFO("add_security_data. sec: '{}'", sec_desc); return *it->second; @@ -253,10 +259,8 @@ struct OkxSubscription::PImpl { OkxSubscription::OkxSubscription( ccapi::EventDispatcher *dispatcher, OkxMarketDataConfig cfg, - IDGenerator *sec_id_store, - IMarketDataListener *listener) - : me_(std::make_unique( - dispatcher, std::move(cfg), sec_id_store, listener)) {} + IDGenerator *sec_id_store) + : me_(std::make_unique(dispatcher, std::move(cfg), sec_id_store)) {} OkxSubscription::OkxSubscription(OkxSubscription &&) noexcept = default; OkxSubscription &OkxSubscription::operator=(OkxSubscription &&) noexcept = @@ -264,4 +268,9 @@ OkxSubscription &OkxSubscription::operator=(OkxSubscription &&) noexcept = OkxSubscription::~OkxSubscription() = default; +void OkxSubscription::start( + std::shared_ptr listener) { + me_->start(listener); +} + } // namespace bts::md::okx diff --git a/src/app_sparkd/sparkd/md/okx/OkxSubscription.h b/src/app_sparkd/sparkd/md/okx/OkxSubscription.h index df5128c3c..ce5e6fc32 100644 --- a/src/app_sparkd/sparkd/md/okx/OkxSubscription.h +++ b/src/app_sparkd/sparkd/md/okx/OkxSubscription.h @@ -1,5 +1,6 @@ #pragma once +#include "sparkd/md/IMarketDataSource.h" #include namespace ccapi { @@ -25,25 +26,22 @@ namespace bts::md::okx { * * @a https://www.okx.com/docs-v5/en/#websocket-api-public-channel */ -class OkxSubscription { +class OkxSubscription : public IMarketDataSource { public: OkxSubscription( ccapi::EventDispatcher *dispatcher, api::md::OkxMarketDataConfig cfg, - persistence::IDGenerator *sec_id_store, - IMarketDataListener *listener); + persistence::IDGenerator *sec_id_store); OkxSubscription(const OkxSubscription &) noexcept = delete; OkxSubscription(OkxSubscription &&) noexcept; - OkxSubscription &operator=(const OkxSubscription &) noexcept = - delete; + OkxSubscription &operator=(const OkxSubscription &) noexcept = delete; OkxSubscription &operator=(OkxSubscription &&) noexcept; ~OkxSubscription(); - void add_listener(IMarketDataListener *listener); - void remove_listener(IMarketDataListener *listener); - + void start(std::shared_ptr listener) override; + private: struct PImpl; std::unique_ptr me_; diff --git a/src/app_sparkd/sparkd/md/recording/MDReplayer.cpp b/src/app_sparkd/sparkd/md/recording/MDReplayer.cpp index f52a1f14c..35047bae4 100644 --- a/src/app_sparkd/sparkd/md/recording/MDReplayer.cpp +++ b/src/app_sparkd/sparkd/md/recording/MDReplayer.cpp @@ -12,9 +12,11 @@ #include "spark/domain_model/Security.pb.h" #include "spark/logging/logging.h" #include "spark/md/GeneralMarketDataListener.h" +#include "spark/md/IMarketDataListener.h" #include "spark/md/MDProducer.h" #include "spark/md/protos/co2.pb.h" #include "spark/os/os.h" +#include "spark/persistence/IDGenerator.h" #include "spark/utils/PBUtils.h" #include "spark/utils/bts_assert.h" #include "sparkd/md/recording/MDRecorder.h" @@ -52,27 +54,26 @@ void copy_books(const bts::co2::BookMessage &from, bts::domain::Book &to) { struct MDReplayer ::PImpl { bts::api::Venue venue; std::set sec_keys = {}; - uint64_t start_time = bts::core::TimeStamp::now_ns(); - GeneralMarketDataListener listener; + uint64_t start_time = 0; + std::shared_ptr listener{nullptr}; double replay_speed; - PImpl( - bts::api::Venue venue, - bts::md::GeneralSecurityIndex &index, - bts::md::MDProducer &md_producer, double replay_speed) - : venue(venue), listener(index, md_producer, venue), replay_speed(replay_speed) {} + + PImpl(bts::api::Venue venue, double replay_speed) + : venue(venue), replay_speed(replay_speed) {} }; -MDReplayer::MDReplayer( - bts::api::Venue venue, - bts::md::GeneralSecurityIndex &index, - bts::md::MDProducer &md_producer, - double replay_speed) - : me(new PImpl(venue, index, md_producer, replay_speed)) { + +MDReplayer::MDReplayer(bts::api::Venue venue, double replay_speed) + : me(new PImpl(venue, replay_speed)) { LOG(md, Info, "MDReplayer::MDReplayer venue: {}", Venue_Name(venue)); } MDReplayer::~MDReplayer() {} -void MDReplayer::run() { +void MDReplayer::start(std::shared_ptr listener) { + LOG(md, Info, "MDReplayer::start venue: {}", Venue_Name(me->venue)); + bts_assert(listener != nullptr); + me->listener = std::move(listener); + me->start_time = bts::core::TimeStamp::now_ns(); std::string directory = MDRecorder::recording_path(); auto include_file = [&](const std::string &file_name) { return MDRecorder::is_md_recording_file(file_name, me->venue); @@ -132,7 +133,7 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) { me->sec_keys.insert(m.security().id()); std::vector sec_vec = {}; sec_vec.push_back(&new_sec); - me->listener.on_securities(sec_vec); + me->listener->on_securities(sec_vec); break; } case (co2::RecordedMDMessage::MDRecordCase::kTotalBook): { @@ -146,20 +147,20 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) { bts::domain::Book book = {}; copy_books(tot_book.direct_book(), book); book.set_security_id(bts::domain::SecurityID(m.sec_key())); - me->listener.on_book(book); + me->listener->on_book(book); } break; } case (co2::RecordedMDMessage::MDRecordCase::kMisc): { bts::domain::SecurityMarketDataStatistics stats(m.sec_key(), m.misc()); - me->listener.on_security_statistics(stats); + me->listener->on_security_statistics(stats); break; } case (co2::RecordedMDMessage::MDRecordCase::kTrade): { bts::domain::TradeRecord tr(m.trade()); std::vector trades; trades.push_back(&tr); - me->listener.on_trades(trades); + me->listener->on_trades(trades); break; } default: { diff --git a/src/app_sparkd/sparkd/md/recording/MDReplayer.h b/src/app_sparkd/sparkd/md/recording/MDReplayer.h index 6a2090b7b..9dfb37df4 100644 --- a/src/app_sparkd/sparkd/md/recording/MDReplayer.h +++ b/src/app_sparkd/sparkd/md/recording/MDReplayer.h @@ -1,7 +1,8 @@ #pragma once #include "spark/domain_model/Security.pb.h" -#include "spark/md/GeneralSecurityIndex.h" +#include "spark/md/IMarketDataListener.h" +#include "sparkd/md/IMarketDataSource.h" namespace bts::md { class MDProducer; @@ -14,15 +15,12 @@ class RecordedMDMessage; namespace bts::md::recording { -class MDReplayer { +class MDReplayer : public IMarketDataSource { public: - MDReplayer( - bts::api::Venue venue, - bts::md::GeneralSecurityIndex &index, - bts::md::MDProducer &md_producer, - double replay_speed=1); - ~MDReplayer(); - void run(); + explicit MDReplayer(bts::api::Venue venue, double replay_speed = 1); + ~MDReplayer() override; + + void start(std::shared_ptr listener) override; private: struct PImpl; diff --git a/src/app_sparkd/sparkd/md/taifex/TaifexMarketDataProvider.cpp b/src/app_sparkd/sparkd/md/taifex/TaifexMarketDataProvider.cpp new file mode 100644 index 000000000..71988eb40 --- /dev/null +++ b/src/app_sparkd/sparkd/md/taifex/TaifexMarketDataProvider.cpp @@ -0,0 +1,50 @@ +#include "TaifexMarketDataProvider.h" +#include "spark/api/domain/Venue.pb.h" +#include "spark/api/md/MarketDataConfig.pb.h" +#include "spark/persistence/IDGenerator.h" +#include "sparkd/md/taifex/TaifexSubscription.h" + +#include +#include + +namespace bts::md { + +using bts::api::domain::Venue; +using bts::api::md::MarketDataConfig; +using bts::persistence::IDGenerator; +using std::pair; +using std::shared_ptr; +using std::string; +using std::vector; + +struct TaifexMarketDataProvider::PImpl { + MarketDataConfig config; + std::shared_ptr sec_id_store; + vector> sources; + + PImpl( + MarketDataConfig config, + std::shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} +}; + +TaifexMarketDataProvider::TaifexMarketDataProvider( + MarketDataConfig config, + std::shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { +} + +TaifexMarketDataProvider::~TaifexMarketDataProvider() = default; + +vector>> +TaifexMarketDataProvider::get_sources() { + vector>> rv; + for (const auto &taifex : me_->config.taifex()) { + me_->sources.push_back( + std::make_shared(taifex, me_->sec_id_store)); + rv.push_back({Venue::VENUE_TAIFEX, me_->sources.back()}); + } + return rv; +} + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/taifex/TaifexMarketDataProvider.h b/src/app_sparkd/sparkd/md/taifex/TaifexMarketDataProvider.h new file mode 100644 index 000000000..a78352305 --- /dev/null +++ b/src/app_sparkd/sparkd/md/taifex/TaifexMarketDataProvider.h @@ -0,0 +1,33 @@ +#pragma once + +#include "sparkd/md/IMarketDataProvider.h" + +#include + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::api::md { +class MarketDataConfig; +} + +namespace bts::md { + +class TaifexMarketDataProvider : public IMarketDataProvider { + public: + explicit TaifexMarketDataProvider( + api::md::MarketDataConfig config, + std::shared_ptr sec_id_store); + ~TaifexMarketDataProvider(); + + std::vector< + std::pair>> + get_sources() override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/taifex/TaifexSubscription.cpp b/src/app_sparkd/sparkd/md/taifex/TaifexSubscription.cpp new file mode 100644 index 000000000..3f2f7f9cb --- /dev/null +++ b/src/app_sparkd/sparkd/md/taifex/TaifexSubscription.cpp @@ -0,0 +1,41 @@ +#include "TaifexSubscription.h" +#include "spark/api/md/TaifexMarketDataConfig.pb.h" +#include "spark/utils/thread_manager.h" +#include "sparkd/md/databento/DatabentoDataHandler.h" +#include "sparkd/md/glbx/GlbxMarketDataHandler.h" +#include "sparkd/md/taifex/TaifexDataGateway.h" + +namespace bts::md { + +using bts::api::md::TaifexMarketDataConfig; +using bts::persistence::IDGenerator; +using std::shared_ptr; + +struct TaifexSubscription::PImpl { + TaifexMarketDataConfig config; + shared_ptr sec_id_store; + + PImpl(TaifexMarketDataConfig config, shared_ptr sec_id_store) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} +}; + +TaifexSubscription::TaifexSubscription( + TaifexMarketDataConfig config, shared_ptr sec_id_store) + : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { +} + +TaifexSubscription::~TaifexSubscription() = default; + +void TaifexSubscription::start(std::shared_ptr listener) { + bts::threading::make_thread( + "taifex", + [config = me_->config, sec_id_store = me_->sec_id_store, + listener = std::move(listener)]() { + md::taifex::TaifexDataGateway::run( + config, std::move(listener), std::move(sec_id_store)); + }) + .detach(); + // TODO: implement 'stop' (signal the thread above to terminate, then join it). +} + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/taifex/TaifexSubscription.h b/src/app_sparkd/sparkd/md/taifex/TaifexSubscription.h new file mode 100644 index 000000000..d2cbda2d9 --- /dev/null +++ b/src/app_sparkd/sparkd/md/taifex/TaifexSubscription.h @@ -0,0 +1,43 @@ +#pragma once + +#include "sparkd/md/IMarketDataSource.h" + +#include + +namespace bts::api::md { +class TaifexMarketDataConfig; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::md { + +/** + * @brief Subscribes to market data at Taifex + */ +class TaifexSubscription : public IMarketDataSource { + public: + TaifexSubscription( + api::md::TaifexMarketDataConfig config, + std::shared_ptr sec_id_store); + TaifexSubscription(const TaifexSubscription &) noexcept = delete; + TaifexSubscription(TaifexSubscription &&) noexcept; + TaifexSubscription &operator=(const TaifexSubscription &) noexcept = delete; + TaifexSubscription &operator=(TaifexSubscription &&) noexcept; + ~TaifexSubscription() override; + + /** + * Starts market data subscription on a background thread. + * + * @note Does not block. + */ + void start(std::shared_ptr listener) override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/lib_python_md/python_md/py_book.h b/src/lib_python_md/python_md/py_book.h index 592568ce2..e1813b83f 100644 --- a/src/lib_python_md/python_md/py_book.h +++ b/src/lib_python_md/python_md/py_book.h @@ -20,19 +20,32 @@ class PyBook { const std::string &venue() const; void set_venue(std::string value); + /** + * @brief Gets the name of the security type (e.g., "SPOT", "FUTURE", "OPTION"). + * + * @return const std::string& The name of the security type. + */ + const std::string &security_type() const; + /** + * @brief Sets the security type. + * + * @param value The new security type (should be "SPOT", "FUTURE", or "OPTION"). + */ + void set_security_type(std::string value); + /** * @brief Gets the exchange security id. This is typically the security's name * (e.g., "BTS-PERPETUAL", or "ESH5"). * - * @note Must be unique for each security at a particular venue (i.e., a single - * venue cannot have 2 securities with the same name). + * @note Should be unique for each security at a particular venue (i.e., a single + * venue cannot have 2 securities with the same name). MUST be unique for each security + * of the same type (e.g., SPOT, FUTURE, OPTION) at a particular venue. * * @note There is no "security_id" member because that is an internal id that * we generate elsewhere in c++ code. The python plugins just need to set the * venue and exchange_security_id. */ const std::string &exchange_security_id() const; - /** * @brief Sets the exchange security id. * @@ -57,6 +70,7 @@ class PyBook { private: std::string venue_; + std::string security_type_; std::string exchange_security_id_; PyBookSide bids_; PyBookSide asks_; @@ -69,6 +83,13 @@ class PyBook { inline const std::string &PyBook::venue() const { return venue_; } inline void PyBook::set_venue(std::string value) { venue_ = std::move(value); } +inline const std::string &PyBook::security_type() const { + return security_type_; +} +inline void PyBook::set_security_type(std::string value) { + security_type_ = std::move(value); +} + inline const std::string &PyBook::exchange_security_id() const { return exchange_security_id_; } diff --git a/src/lib_python_md/python_md/py_module.cpp b/src/lib_python_md/python_md/py_module.cpp index 1156aa3b0..25224ecad 100644 --- a/src/lib_python_md/python_md/py_module.cpp +++ b/src/lib_python_md/python_md/py_module.cpp @@ -37,9 +37,6 @@ namespace py = pybind11; // Binds getter -> , and setter -> set_. #define PROP(type, name) \ def(_STR(name), &type::name).def(_STR(set_##name), &type::set_##name) -// namespace blah { -// typedef bts::md::py_interop::PyBookSide & (bts::md::py_interop::PyBook::*bids()); -// } // Define python module with c++ classes. PYBIND11_MODULE(python_md_module, m) { @@ -93,6 +90,7 @@ PYBIND11_MODULE(python_md_module, m) { py::class_(m, "PyBook") .def(py::init<>()) .PROP(PyBook, venue) + .PROP(PyBook, security_type) .PROP(PyBook, exchange_security_id) .EXPLICIT_DEF(PyBook, bids, BookSideGetter) .DEF(PyBook, set_bids) @@ -110,6 +108,7 @@ PYBIND11_MODULE(python_md_module, m) { py::class_(m, "PyTrade") .def(py::init<>()) .PROP(PyTrade, venue) + .PROP(PyTrade, security_type) .PROP(PyTrade, exchange_security_id) .PROP(PyTrade, price) .PROP(PyTrade, quantity) @@ -142,5 +141,6 @@ PYBIND11_MODULE(python_md_module, m) { } #undef PROP +#undef EXPLICIT_DEF #undef DEF #undef _STR diff --git a/src/lib_python_md/python_md/py_trade.h b/src/lib_python_md/python_md/py_trade.h index 258509507..bfa65f45d 100644 --- a/src/lib_python_md/python_md/py_trade.h +++ b/src/lib_python_md/python_md/py_trade.h @@ -12,6 +12,9 @@ class PyTrade { const std::string &venue() const; void set_venue(std::string value); + const std::string &security_type() const; + void set_security_type(std::string value); + const std::string &exchange_security_id() const; void set_exchange_security_id(std::string value); @@ -37,6 +40,7 @@ class PyTrade { private: std::string venue_; + std::string security_type_; std::string exchange_security_id_; double price_; double quantity_; @@ -51,6 +55,9 @@ class PyTrade { inline const std::string &PyTrade::venue() const { return venue_; } inline void PyTrade::set_venue(std::string value) { venue_ = std::move(value); } +inline const std::string &PyTrade::security_type() const { return security_type_; } +inline void PyTrade::set_security_type(std::string value) { security_type_ = std::move(value); } + inline const std::string &PyTrade::exchange_security_id() const { return exchange_security_id_; } diff --git a/src/lib_python_md/python_md_providers/dummy_md_provider.py b/src/lib_python_md/python_md_providers/dummy_md_provider.py index 691943529..58a252f43 100644 --- a/src/lib_python_md/python_md_providers/dummy_md_provider.py +++ b/src/lib_python_md/python_md_providers/dummy_md_provider.py @@ -54,6 +54,7 @@ def _get_book_side(levels: list[(float, float)], is_bid_side: bool): def _get_trade( venue: str, + security_type: str, exchange_security_id: str, price: float, quantity: float, @@ -61,6 +62,7 @@ def _get_trade( ): trade = pymd.PyTrade() trade.set_venue(venue) + trade.set_security_type(security_type) trade.set_exchange_security_id(exchange_security_id) trade.set_price(price) trade.set_quantity(quantity) @@ -94,6 +96,7 @@ class DummyMdProvider(pymd.MdProvider): asks = _get_book_side([(100.25, 3)], is_bid_side=False) book = pymd.PyBook() book.set_venue("Glbx") + book.set_security_type("FUTURE") book.set_exchange_security_id("ESH5") book.set_bids(bids) book.set_asks(asks) @@ -105,6 +108,7 @@ class DummyMdProvider(pymd.MdProvider): def add_fake_trades(): trade = _get_trade( venue="Glbx", + security_type="FUTURE", exchange_security_id="ESH5", price=100, quantity=2, @@ -119,4 +123,4 @@ class DummyMdProvider(pymd.MdProvider): add_fake_trades() def stop(self): - print("Stopping python MdProviderImpl") + print("Stopping python DummyMdProvider") diff --git a/src/lib_spark/spark/core/LOCK_GUARD.h b/src/lib_spark/spark/core/LOCK_GUARD.h new file mode 100644 index 000000000..c90b94b94 --- /dev/null +++ b/src/lib_spark/spark/core/LOCK_GUARD.h @@ -0,0 +1,7 @@ +#pragma once + +#define BTS_LOCK_GUARD(lockable) \ + std::lock_guard __lock_guard { lockable } + +#define BTS_LOCK_GUARD_AS(lockable, name) \ + std::lock_guard __lock_guard_##name { lockable } diff --git a/src/lib_spark/spark/md/GeneralMarketDataListener.cpp b/src/lib_spark/spark/md/GeneralMarketDataListener.cpp index 8b8bf3ee7..00f54ef42 100644 --- a/src/lib_spark/spark/md/GeneralMarketDataListener.cpp +++ b/src/lib_spark/spark/md/GeneralMarketDataListener.cpp @@ -277,8 +277,7 @@ struct GeneralMarketDataListener::PImpl { const bool use_given_sec_key = false; const std::string cc_name_prefix = "cc_security"; size_t n_bk_updates = 0; - bts::persistence::IDGenerator &id_gen = - bts::persistence::IDGenerator::instance(); + bts::persistence::IDGenerator *id_gen; GeneralSecurityIndex &index; MDProducer &md_producer; const std::shared_ptr sim_ex_gwy = @@ -296,10 +295,10 @@ struct GeneralMarketDataListener::PImpl { api::domain::Venue venue, GeneralSecurityIndex &index, MDProducer &md_producer, - std::shared_ptr sim_ex_gwy) - : venue(venue), index(index), - md_producer(md_producer), - sim_ex_gwy(std::move(sim_ex_gwy)) { + std::shared_ptr sim_ex_gwy, + bts::persistence::IDGenerator *sec_id_store) + : venue(venue), id_gen(sec_id_store), index(index), + md_producer(md_producer), sim_ex_gwy(std::move(sim_ex_gwy)) { if (this->sim_ex_gwy != nullptr) { this->sim_ex_gwy->set_producers(&this->index); } @@ -361,13 +360,17 @@ void copy_books( // interface functions //////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// GeneralMarketDataListener::GeneralMarketDataListener( - GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue) + GeneralSecurityIndex &index, + MDProducer &md_producer, + api::Venue venue, + persistence::IDGenerator *sec_id_store) : me(std::make_unique( domain::DomainSecurityUtils::convert(venue), index, md_producer, bts::orderman::SimExecutionGatewayFactory::get()->get_getayeway( - venue))) { + venue), + sec_id_store)) { LOG(md, Info, "GeneralMarketDataListener::GeneralMarketDataListener venue: {}", bts::api::Venue_Name(venue)); @@ -504,7 +507,7 @@ CCSecSearchResult GeneralMarketDataListener::_find_sec( if (add_if_absent) { uint64_t bts_id = cc_id.value(); if (!me->use_given_sec_key) { - bts_id = me->id_gen.get_or_add(me->venue, sec.type(), sec.desc()); + bts_id = me->id_gen->get_or_add(me->venue, sec.type(), sec.desc()); } me->cc_sec_id_map.insert({cc_id, {bts_id}}); LOG(md, Info, @@ -651,7 +654,7 @@ uint64_t GeneralMarketDataListener::_generate_security_key( auto logical_key = get_logical_key(sec); // if (me->use_given_sec_key) // return sec.security_common.id.value(); - return me->id_gen.get_or_add(me->venue, sec.type(), sec.desc()); + return me->id_gen->get_or_add(me->venue, sec.type(), sec.desc()); } void GeneralMarketDataListener::_possibly_match_with_sim_orders( diff --git a/src/lib_spark/spark/md/GeneralMarketDataListener.h b/src/lib_spark/spark/md/GeneralMarketDataListener.h index e9b808fec..332f9088e 100644 --- a/src/lib_spark/spark/md/GeneralMarketDataListener.h +++ b/src/lib_spark/spark/md/GeneralMarketDataListener.h @@ -5,6 +5,7 @@ #include "spark/md/GeneralSecurityIndex.h" #include "spark/md/IMarketDataListener.h" #include "spark/md/Shm/FixedSecurity.h" +#include "spark/persistence/IDGenerator.h" namespace bts::md { class MDProducer; @@ -16,7 +17,10 @@ namespace bts::md { class GeneralMarketDataListener : public IMarketDataListener { public: GeneralMarketDataListener( - GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue); + GeneralSecurityIndex &index, + MDProducer &md_producer, + api::Venue venue, + persistence::IDGenerator *sec_id_store); virtual ~GeneralMarketDataListener(); void on_securities(const std::vector &complete_security_list) override;