refactor: Split MarketDataGateway up into providers/sources

MarketDataGateway still exists, but is greatly simplified. It creates
several `IMarketDataProvider`s, each of which generates a list of
`IMarketDataSource`s. The market data sources provide market data to an
`IMarketDataListener`.

* EmumbaMarketDataProvider
* CCApiMarketDataProvider
* PyMarketDataProvider
* OnixsMarketDataProvider
* ReplayMarketDataProvider

As a side effect, individual py market data sources are now explicitly
limited to a single venue (it probably should have been this way all
along, since the shared memory writers do not expect updates to involve
multiple venues).
This commit is contained in:
John Colvin committed 2024-10-21 13:40:00 +00:00
1 parent 11b056d27a
commit ee06eca830
56 files changed
+1663 -729

No files matched your search

+9 -1
View File
@@ -1,4 +1,5 @@
#include "Run.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/api/public/ServiceContext.h"
#include "spark/config/config.h"
#include "spark/config/protos/ports.pb.h"
@@ -15,6 +16,7 @@
#include "spark/orderman/OrderServiceServer.h"
#include "spark/orderman/events/TradeSummarySignalDispatcher.h"
#include "spark/orderman/order_factory/DynamicOrderRegistry.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/persistence/PersistenceServiceServer.h"
#include "spark/plugin/SparkServices.h"
#include "spark/plugin/plugin.h"
@@ -24,8 +26,10 @@
#include "md/cmds/Run.h"
#include "sparkd/algo/AlgoDefinitionServiceServer.h"
#include "sparkd/analytics/AnalyticsServiceServer.h"
#include "sparkd/cmds/md/cmds/Cleanup.h"
#include "sparkd/http/WebServer.h"
#include "sparkd/md/MarketDataAuxiliaryServiceServer.h"
#include "sparkd/md/MarketDataGateway.h"
#include "sparkd/md/recording/MDRecorder.h"
#include "sparkd/position/MarginServiceServer.h"
#include "sparkd/position/PositionServiceServer.h"
@@ -63,7 +67,11 @@ static void execute() {
bts::md::MBOProcessor mbo_processor;
bts::md::TradeSummarySignalDispatcher tss_dispatcher;
md::cmds::Run::run();
md::cmds::Cleanup::execute();
bts::md::MarketDataGateway md_gateway{
config::get<api::md::MarketDataConfig>(),
persistence::IDGenerator::instance_ptr()};
md_gateway.start();
api::synthetic::SyntheticSecurityService synthetic_service(sc);
bts::md::MDClient mdclient(
@@ -1,5 +1,7 @@
#include "LiveLadderDemo.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/config/config.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/MarketDataGateway.h"
#include "sparkd/md/listeners/LiveLadderPrinter.h"
@@ -7,13 +9,16 @@
namespace bts::cmds {
LiveLadderDemo::LiveLadderDemo(const api::md::MarketDataConfig &) {
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
LiveLadderDemo::LiveLadderDemo(const MarketDataConfig &config) {
char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt";
putenv(ssl_cert_file);
md::MarketDataGateway gw;
md::MarketDataGateway gw{config, IDGenerator::instance_ptr()};
md::listeners::LiveLadderPrinter ladder_printer;
gw.run();
gw.start();
const std::size_t n_loops = 16;
for (std::size_t n = 0; n < n_loops; n++) {
+11 -3
View File
@@ -1,5 +1,6 @@
#include "Run.h"
#include "Cleanup.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/config/protos/market_data.pb.h"
#include "spark/domain/SecurityCriteria.h"
#include "spark/logging/logging.h"
@@ -7,6 +8,7 @@
#include "spark/md/UserSymbol.h"
#include "spark/md/relay/RelayRecorder.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/thread_manager.h"
#include "sparkd/md/MarketDataGateway.h"
@@ -14,6 +16,7 @@
#include "sparkd/md/platforms/ice_onixs/ICEMain.h"
#include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
#include <string>
#include <thread>
namespace bts::cmds::md::cmds {
@@ -91,9 +94,14 @@ static void run_onixs(
static void run_md_gwy() {
LOG(md, Info, "Run::run_md_gwy t_id:{} t_name:{}", os::get_thread_id(),
os::get_thread_name());
bts::md::MarketDataGateway gwy = {};
// this keeps going in a loop
gwy.run();
bts::md::MarketDataGateway gwy{
config::get<api::md::MarketDataConfig>(),
persistence::IDGenerator::instance_ptr()};
gwy.start();
while (true) {
using namespace std::chrono_literals;
std::this_thread::sleep_for(2s);
}
}
void Run::run() {
@@ -0,0 +1,7 @@
#include "IMarketDataProvider.h"
namespace bts::md {
IMarketDataProvider::~IMarketDataProvider() = default;
} // namespace bts::md
@@ -0,0 +1,25 @@
#pragma once
#include "spark/api/domain/Venue.pb.h"
#include "sparkd/md/IMarketDataSource.h"
#include <vector>
namespace bts::md {
/**
* @brief This is a single function interface, whose sole purpose is to provide a list of market
* data sources (per venue). A good example implementation is the `CCApiMarketDataProvider`, which
* reads `MarketDataConfig` and returns an `IMarketDataSource` for each venue configuration (e.g.,
* `bts::api::md::BinanceMarketDataConfig`, `bts::api::md::CoinbaseMarketDataConfig`, etc.).
*/
class IMarketDataProvider {
public:
virtual ~IMarketDataProvider();
virtual std::vector<
std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() = 0;
};
} // namespace bts::md
@@ -0,0 +1,9 @@
#include "IMarketDataSource.h"
namespace bts::md {
IMarketDataSource::~IMarketDataSource() = default;
void IMarketDataSource::stop() {}
} // namespace bts::md
@@ -0,0 +1,20 @@
#pragma once
#include <memory>
namespace bts::md {
class IMarketDataListener;
/**
* @brief This interface represents a source of market data. Call `IMarketDataSource::start` to
* connect an `IMarketDataListener` to the source and begin receiving market data.
*/
class IMarketDataSource {
public:
virtual ~IMarketDataSource();
virtual void start(std::shared_ptr<IMarketDataListener> listener) = 0;
virtual void stop();
};
} // namespace bts::md
+56 -377
View File
@@ -1,75 +1,41 @@
#include "MarketDataGateway.h"
#include "spark/api/md/CboeMarketDataConfig.pb.h"
#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/api/md/MarketDataConfigUtils.h"
#include "spark/api/md/MarketDataReplayConfig.pb.h"
#include "spark/collections/Contains.h"
#include "spark/config/protos/system_config.pb.h"
#include "spark/core/FMT.h"
#include "spark/domain/DomainSecurityUtils.h"
#include "spark/domain/SecurityID.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/MDProducer.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/bts_fatal.h"
#include "spark/utils/thread_manager.h"
#include "sparkd/md/MarketDataListenerID.h"
#include "sparkd/md/MarketDataRegistration.h"
#include "sparkd/md/MarketDataRegistrationError.h"
#include "sparkd/md/MarketDataRegistrationFactory.h"
#include "sparkd/md/MarketDataSubscription.h"
#include "sparkd/md/IMarketDataProvider.h"
#include "sparkd/md/OnixsMarketDataProvider.h"
#include "sparkd/md/PyMarketDataProvider.h"
#include "sparkd/md/PyMarketDataSubscription.h"
#include "sparkd/md/binance/BinanceSubscription.h"
#include "sparkd/md/binance_us/BinanceUsSubscription.h"
#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
#include "sparkd/md/cboe/CboeMarketDataHandler.h"
#include "sparkd/md/coinbase/CoinbaseSubscription.h"
#include "sparkd/md/deribit/DeribitSubscription.h"
#include "sparkd/md/emumba/EmumbaDeribitData.h"
#include "sparkd/md/ftx/FTXSubscription.h"
#include "sparkd/md/glbx/GlbxMarketDataHandler.h"
#include "sparkd/md/okx/OkxSubscription.h"
#include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
#include "sparkd/md/recording/MDReplayer.h"
#include "sparkd/md/taifex/TaifexDataGateway.h"
#include "sparkd/md/ReplayMarketDataProvider.h"
#include "sparkd/md/ccapi/CCApiMarketDataProvider.h"
#include "sparkd/md/databento/DatabentoMarketDataProvider.h"
#include "sparkd/md/emumba/EmumbaMarketDataProvider.h"
#include "sparkd/md/taifex/TaifexMarketDataProvider.h"
#include <bits/stdint-uintn.h>
#include <boost/filesystem.hpp>
#include <boost/filesystem/path.hpp>
#include <ccapi_cpp/ccapi_event_dispatcher.h>
#include <emumba-deribit-connector/deribit_utils.h>
#include <memory>
#include <optional>
#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
#include <string>
#include <unordered_map>
#include <vector>
namespace py = pybind11;
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::config::SystemConfig;
//using bts::domain::Security;
using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::persistence::IDGenerator;
using os::sparkd_wd;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
using Path = boost::filesystem::path;
static bool validate(const MarketDataConfig &cfg, vector<string> &errs) {
auto orig_n_errs = errs.size();
if (!cfg.deribit().empty() && !cfg.emumba_deribit().empty()) {
@@ -79,353 +45,66 @@ static bool validate(const MarketDataConfig &cfg, vector<string> &errs) {
return errs.size() == orig_n_errs;
}
struct MarketDataListenerData {
MarketDataListenerData() = default;
MarketDataListenerData(const MarketDataListenerData &) noexcept = delete;
MarketDataListenerData(MarketDataListenerData &&) noexcept = default;
MarketDataListenerData &operator=(const MarketDataListenerData &) noexcept =
delete;
MarketDataListenerData &operator=(MarketDataListenerData &&) noexcept =
default;
~MarketDataListenerData() = default;
MarketDataListenerID id;
MarketDataSubscription subscription;
};
struct MDReplayData {
bts::api::md::MarketDataReplayConfig replay_conf;
bts::api::Venue venue;
bts::md::GeneralSecurityIndex sec_index;
bts::md::MDProducer md_producer;
bts::md::recording::MDReplayer md_replayer;
MDReplayData(const bts::api::md::MarketDataReplayConfig &replay_conf)
: replay_conf(replay_conf), venue(replay_conf.venue()), sec_index(),
md_replayer(replay_conf.venue(), sec_index, md_producer, replay_conf.replay_speed()) {}
};
struct VenueListenerData {
bts::md::GeneralSecurityIndex sec_index;
bts::md::MDProducer md_producer;
std::shared_ptr<bts::md::IMarketDataListener> listener;
VenueListenerData(bts::api::Venue venue)
: sec_index(),
listener(std::make_shared<bts::md::GeneralMarketDataListener>(
sec_index, md_producer, venue)) {}
};
struct MarketDataGateway::PImpl {
api::md::MarketDataConfig md_config =
config::get<api::md::MarketDataConfig>();
ccapi::EventDispatcher dispatcher;
MarketDataRegistrationFactory registration_factory;
std::shared_ptr<IDGenerator> sec_id_store = IDGenerator::instance_ptr();
unordered_map<MarketDataListenerID, MarketDataListenerData> listeners;
std::vector<std::unique_ptr<MDReplayData>> replays = {};
// we create these and pass them to the cc md sources
std::vector<std::shared_ptr<VenueListenerData>> venue_listeners = {};
std::vector<std::shared_ptr<PyMarketDataSubscription>> py_md_subs = {};
struct MdSink {
GeneralSecurityIndex sec_index;
MDProducer md_producer;
shared_ptr<IMarketDataListener> listener;
MdSink(api::Venue venue, IDGenerator *sec_id_store)
: sec_index(), listener(std::make_shared<GeneralMarketDataListener>(
sec_index, md_producer, venue, sec_id_store)) {}
};
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
vector<unique_ptr<IMarketDataProvider>> providers;
vector<unique_ptr<MdSink>> sinks;
vector<unique_ptr<IMarketDataSource>> sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {
providers.emplace_back(std::make_unique<CCApiMarketDataProvider>(
this->config, this->sec_id_store));
providers.emplace_back(
std::make_unique<EmumbaMarketDataProvider>(this->config));
providers.emplace_back(
std::make_unique<OnixsMarketDataProvider>(this->config));
providers.emplace_back(std::make_unique<DatabentoMarketDataProvider>(
this->config, this->sec_id_store));
providers.emplace_back(std::make_unique<TaifexMarketDataProvider>(
this->config, this->sec_id_store));
providers.emplace_back(std::make_unique<PyMarketDataProvider>(
this->config, this->sec_id_store));
providers.emplace_back(std::make_unique<ReplayMarketDataProvider>(
this->config, this->sec_id_store));
}
};
MarketDataGateway::MarketDataGateway()
: me_(std::make_unique<MarketDataGateway::PImpl>()) {}
MarketDataGateway::MarketDataGateway(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<MarketDataGateway::PImpl>(
std::move(config), std::move(sec_id_store))) {}
MarketDataGateway::~MarketDataGateway() = default;
// starts the market data sources and then keeps the function going
void MarketDataGateway::run() {
void MarketDataGateway::start() {
vector<string> cfg_errs;
if (!validate(me_->md_config, cfg_errs)) {
if (!validate(me_->config, cfg_errs)) {
BTS_FATAL_MSGF("Invalid MarketDataConfig:", FMT_JOIN(cfg_errs, "\n"));
return;
}
LOG(md, Info, "MarketDataGateway::run starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
_start_cc(me_->md_config);
if (!me_->md_config.emumba_deribit().empty()) {
bts_assert(me_->md_config.emumba_deribit().size() == 1);
_start_emumba_deribit(me_->md_config.emumba_deribit().at(0));
}
start_glbx(me_->md_config);
start_taifex(me_->md_config);
start_cboe(me_->md_config);
_start_replays(me_->md_config);
start_py_();
while (true) {
std::this_thread::sleep_for(std::chrono::seconds(1));
}
LOG(md, Info, "MarketDataGateway::run ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
void MarketDataGateway::start_py_() {
for (const auto &py_plugin : me_->md_config.python_md_providers()) {
try {
me_->py_md_subs.push_back(std::make_shared<PyMarketDataSubscription>(
py_plugin, me_->sec_id_store.get()));
} catch (const std::exception &ex) {
bts_assert_msgf(
false, "PyMarketDataSubscription threw: {}",
std::string{ex.what()});
for (auto &provider : me_->providers) {
for (auto &[venue, source] : provider->get_sources()) {
const auto v = domain::DomainSecurityUtils::convert(venue);
auto &sink = me_->sinks.emplace_back(
std::make_unique<PImpl::MdSink>(v, me_->sec_id_store.get()));
source->start(sink->listener);
}
}
}
// CCAPI //////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
MarketDataRegistration MarketDataGateway::_start_cc(
const MarketDataConfig &cfg) noexcept {
LOG(md, Info, "MarketDataGateway::_start_cc starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
auto reg = me_->registration_factory.create();
bts_assert(reg.listener_id.has_value());
bts_assert(!CONTAINS_KEY(me_->listeners, *reg.listener_id));
LOG(md, Info, "MarketDataGateway subscribing to data:\n{}",
ProtoPrinter::to_string(cfg));
vector<string> cfg_errs;
if (!validate(cfg, cfg_errs)) {
return MarketDataRegistration{
{MarketDataRegistrationError::Type::CONFIG_ERROR,
fmt::to_string(fmt::join(cfg_errs, "\n"))}};
}
MarketDataListenerData &listener_data = me_->listeners[*reg.listener_id];
listener_data.id = *reg.listener_id;
// FTX
for (const auto &ftx : cfg.ftx()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_FTX));
listener_data.subscription.add(ftx::FTXSubscription{
&me_->dispatcher, ftx, me_->sec_id_store.get(), p->listener.get()});
}
// Binance
for (const auto &binance : cfg.binance()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_BINANCE));
listener_data.subscription.add(binance::BinanceSubscription{
&me_->dispatcher, binance, me_->sec_id_store.get(),
p->listener.get()});
}
// Binance US
for (const auto &binance_us : cfg.binance_us()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_BINANCE_US));
listener_data.subscription.add(binance_us::BinanceUsSubscription{
&me_->dispatcher, binance_us, me_->sec_id_store.get(),
p->listener.get()});
}
// Coinbase
for (const auto &coinbase : cfg.coinbase()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_COINBASE));
listener_data.subscription.add(coinbase::CoinbaseSubscription{
&me_->dispatcher, coinbase, me_->sec_id_store.get(),
p->listener.get()});
}
// Binance USDS Futures
for (const auto &binance_ufs : cfg.binance_usds_futures()) {
auto p =
me_->venue_listeners.emplace_back(std::make_shared<VenueListenerData>(
bts::api::VENUE_BINANCE_USDS_FUTURES));
listener_data.subscription.add(
binance_usds_futures::BinanceUsdsFuturesSubscription{
&me_->dispatcher, binance_ufs, me_->sec_id_store.get(),
p->listener.get()});
}
// Deribit
for (const auto &deribit : cfg.deribit()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_DERIBIT));
listener_data.subscription.add(deribit::DeribitSubscription{
&me_->dispatcher, deribit, me_->sec_id_store.get(),
p->listener.get()});
}
// Okx
for (const auto &okx : cfg.okx()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_OKX));
listener_data.subscription.add(okx::OkxSubscription{
&me_->dispatcher, okx, me_->sec_id_store.get(), p->listener.get()});
}
LOG(md, Info, "MarketDataGateway::_start_cc ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
return reg;
}
void MarketDataGateway::unsubscribe(MarketDataListenerID id) {
LOG(md, Info, "Unsubscribing listener {}", id);
me_->listeners.erase(id);
}
/// glbx //////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////
void MarketDataGateway::start_glbx(const bts::api::md::MarketDataConfig &cfg){
for (const auto &glbx : cfg.glbx()) {
if(glbx.use_databento()){
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_GLBX));
std::string tid_name = "glbx";
bts::threading::make_thread(tid_name, [&]() {
api::md::glbx::GlbxMarketDatahandler::run(
p->listener,
me_->sec_id_store,
glbx);
}).detach();
}
else{
_start_onix_glbx(glbx);
}
}
}
static void run_onixs() {
LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name",
os::get_thread_id(), os::get_thread_name());
bts::md::MBOWriter *mbo = nullptr;
bts::md::TradeSummarySignalDispatcher *tssd = nullptr;
std::vector<std::shared_ptr<bts::md::IMDPlatform>> platforms;
// get all glbx symbols
auto cme_symbols = SymbolLoader::load(
bts::domain::SecurityCriteria::create({api::domain::VENUE_GLBX}));
if (!cme_symbols.empty()) {
auto *main = new bts::md::MDP3Main();
main->start({cme_symbols.begin(), cme_symbols.end()}, mbo, tssd);
platforms.emplace_back(main);
}
// deribit
// auto *d_platform = new bts::md::EmMDListener()
// Timers
auto &disp = utils::Dispatcher::current();
utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() {
for (auto &p : platforms)
p->send_heartbeats();
});
heartbeat_timer.start();
disp.run();
}
void MarketDataGateway::_start_onix_glbx(
bts::api::md::GlbxMarketDataConfig cfg) {
LOG(md, Info,
"MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. "
"cfg:\n{}",
os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
bts::md::run_onixs();
}
// deribit ////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
void MarketDataGateway::_start_emumba_deribit(
bts::api::md::EmumbaDeribitMarketDataConfig cfg) {
LOG(md, Info,
"MarketDataGateway::_start_emumba_deribit starting t_id:{} t_name:{}. "
"cfg:\n{}",
os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
auto start_deribit_thr = [=]() {
const auto config_path = Path{config::get<SystemConfig>().config_path()} /
cfg.json_conf_file_name();
const auto conf =
bts::cmds::DeribitDemoConfiguration::read_file(config_path.string());
bts_assert_msg(conf.has_value(), "Failed to parse deribit config");
// log
for (const auto &s : conf.value().symbol_config_list) {
LOG(md, Info, "sub.symbol: {}", s.symbol_name);
LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future,
s.interested_in_option);
for (const auto &i : s.interested_instruments) {
LOG(md, Info, "sub.instrument: {}", i);
}
}
EmumbaDeribitData ddata_obj;
EmumbaDeribitData *ddata = &ddata_obj;
// end log
bts_assert(conf.has_value());
LOG(md, Info, "Deribit conf: {}", conf.value());
ddata->real_handler =
std::make_shared<emumba::deribit_connector::deribit_handler>(
conf.value().symbol_config_list, conf.value().trading_config,
conf.value().credential_map, conf.value().logging_configuration,
conf.value().deribit_url, &ddata->listener);
// since the trading interface is housed inside the EmMDListener, we need to
// create and send emumba's trading_session_interface
for (const auto &p : conf.value().credential_map) {
const auto &sess = p.second;
auto ts_ptr = std::make_shared<
emumba::deribit_connector::trading_session_interface>(
ddata->real_handler->create_new_trading_session(sess));
ddata->trading_session_vec.push_back({sess, ts_ptr});
}
bts_assert(!ddata->trading_session_vec.empty());
ddata->listener.set_handler(
ddata->real_handler.get(), ddata->trading_session_vec);
ddata->real_handler->run();
};
threading::make_thread("Emumba", [&]() { start_deribit_thr(); }).detach();
LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
// replays ////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
void MarketDataGateway::_start_replays(
const bts::api::md::MarketDataConfig &cfg) {
LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
for (const auto &r :
MarketDataConfigUtils::get_effective_replay_config(cfg)) {
me_->replays.emplace_back(std::make_unique<MDReplayData>(r));
}
for (const auto &replayer : me_->replays) {
std::string tid_name = "MDReplayer_" + Venue_Name(replayer->venue);
bts::threading::make_thread(tid_name, [&]() {
replayer->md_replayer.run();
}).detach();
}
LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) {
// taifex
int counter = 0;
for (const auto &taifex : cfg.taifex()) {
counter++;
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_TAIFEX));
std::string tid_name = "taifex" +std::to_string(counter);
bts::threading::make_thread(tid_name, [&]() {
bts::md::taifex::TaifexDataGateway::run(
taifex, p->listener, me_->sec_id_store);
}).detach();
}
}
void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg){
// cboe
for (const api::md::CboeMarketDataConfig &cboe : cfg.cboe()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_CBOE));
std::string tid_name = "CBOE";
bts::threading::make_thread(tid_name, [&]() {
api::md::cboe::CboeMarketDatahandler::run(
p->listener,
me_->sec_id_store,
cboe);
}).detach();
UNUSED(cboe);
}
}
} // namespace bts::md
+13 -25
View File
@@ -1,45 +1,33 @@
#pragma once
#include "spark/api/md/GlbxMarketDataConfig.pb.h"
#include "sparkd/md/MarketDataListenerID.h"
#include <memory>
namespace bts::api::md {
class EmumbaDeribitMarketDataConfig;
class MarketDataConfig;
} // namespace bts::api::md
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
class IMarketDataListener;
class MarketDataRegistration;
/**
* @brief The `MarketDataGateway` is responsible for starting all configured market data sources.
* It connects each source to a special `IMarketDataListener` that writes the data to shared memory.
*/
class MarketDataGateway {
public:
MarketDataGateway();
explicit MarketDataGateway(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~MarketDataGateway();
/**
* @brief Connects a market data listener to a market data source
* @return MarketDataRegistration
*/
void run();
void unsubscribe(MarketDataListenerID id);
static void test_py();
void start();
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
MarketDataRegistration _start_cc(const api::md::MarketDataConfig &) noexcept;
void start_py_();
static void _start_emumba_deribit(
bts::api::md::EmumbaDeribitMarketDataConfig cfg);
static void _start_onix_glbx(bts::api::md::GlbxMarketDataConfig cfg);
void _start_replays(const bts::api::md::MarketDataConfig &cfg);
void start_glbx(const bts::api::md::MarketDataConfig &cfg);
void start_cboe(const bts::api::md::MarketDataConfig &cfg);
void start_taifex(const bts::api::md::MarketDataConfig &cfg);
};
} // namespace bts::md
@@ -1,60 +1,60 @@
#include "MarketDataSubscription.h"
#include "sparkd/md/binance/BinanceSubscription.h"
#include "sparkd/md/binance_us/BinanceUsSubscription.h"
#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
#include "sparkd/md/coinbase/CoinbaseSubscription.h"
#include "sparkd/md/deribit/DeribitSubscription.h"
#include "sparkd/md/ftx/FTXSubscription.h"
#include "sparkd/md/okx/OkxSubscription.h"
// #include "MarketDataSubscription.h"
// #include "sparkd/md/binance/BinanceSubscription.h"
// #include "sparkd/md/binance_us/BinanceUsSubscription.h"
// #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
// #include "sparkd/md/coinbase/CoinbaseSubscription.h"
// #include "sparkd/md/deribit/DeribitSubscription.h"
// #include "sparkd/md/ftx/FTXSubscription.h"
// #include "sparkd/md/okx/OkxSubscription.h"
#include <vector>
// #include <vector>
namespace bts::md {
// namespace bts::md {
using std::vector;
// using std::vector;
struct MarketDataSubscription::PImpl {
vector<ftx::FTXSubscription> ftx_subs;
vector<binance::BinanceSubscription> binance_subs;
vector<binance_us::BinanceUsSubscription> binance_us_subs;
vector<coinbase::CoinbaseSubscription> coinbase_subs;
vector<binance_usds_futures::BinanceUsdsFuturesSubscription>
binance_usds_futures_subs;
vector<deribit::DeribitSubscription> deribit_subs;
vector<okx::OkxSubscription> okx_subs;
// TODO: add more exchanges
};
// struct MarketDataSubscription::PImpl {
// vector<ftx::FTXSubscription> ftx_subs;
// vector<binance::BinanceSubscription> binance_subs;
// vector<binance_us::BinanceUsSubscription> binance_us_subs;
// vector<coinbase::CoinbaseSubscription> coinbase_subs;
// vector<binance_usds_futures::BinanceUsdsFuturesSubscription>
// binance_usds_futures_subs;
// vector<deribit::DeribitSubscription> deribit_subs;
// vector<okx::OkxSubscription> okx_subs;
// // TODO: add more exchanges
// };
MarketDataSubscription::MarketDataSubscription()
: me_(std::make_unique<PImpl>()) {}
// MarketDataSubscription::MarketDataSubscription()
// : me_(std::make_unique<PImpl>()) {}
MarketDataSubscription::~MarketDataSubscription() = default;
// MarketDataSubscription::~MarketDataSubscription() = default;
void MarketDataSubscription::add(binance::BinanceSubscription sub) {
me_->binance_subs.push_back(std::move(sub));
}
// void MarketDataSubscription::add(binance::BinanceSubscription sub) {
// me_->binance_subs.push_back(std::move(sub));
// }
void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) {
me_->binance_us_subs.push_back(std::move(sub));
}
void MarketDataSubscription::add(coinbase::CoinbaseSubscription sub) {
me_->coinbase_subs.push_back(std::move(sub));
}
void MarketDataSubscription::add(
binance_usds_futures::BinanceUsdsFuturesSubscription sub) {
me_->binance_usds_futures_subs.push_back(std::move(sub));
}
// void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) {
// me_->binance_us_subs.push_back(std::move(sub));
// }
// void MarketDataSubscription::add(coinbase::CoinbaseSubscription sub) {
// me_->coinbase_subs.push_back(std::move(sub));
// }
// void MarketDataSubscription::add(
// binance_usds_futures::BinanceUsdsFuturesSubscription sub) {
// me_->binance_usds_futures_subs.push_back(std::move(sub));
// }
void MarketDataSubscription::add(deribit::DeribitSubscription sub) {
me_->deribit_subs.push_back(std::move(sub));
}
// void MarketDataSubscription::add(deribit::DeribitSubscription sub) {
// me_->deribit_subs.push_back(std::move(sub));
// }
void MarketDataSubscription::add(ftx::FTXSubscription sub) {
me_->ftx_subs.push_back(std::move(sub));
}
// void MarketDataSubscription::add(ftx::FTXSubscription sub) {
// me_->ftx_subs.push_back(std::move(sub));
// }
void MarketDataSubscription::add(okx::OkxSubscription sub) {
me_->okx_subs.push_back(std::move(sub));
}
// void MarketDataSubscription::add(okx::OkxSubscription sub) {
// me_->okx_subs.push_back(std::move(sub));
// }
} // namespace bts::md
// } // namespace bts::md
@@ -1,59 +1,59 @@
#pragma once
// #pragma once
#include <memory>
// #include <memory>
namespace bts::md::binance {
class BinanceSubscription;
}
// namespace bts::md::binance {
// class BinanceSubscription;
// }
namespace bts::md::binance_us {
class BinanceUsSubscription;
}
namespace bts::md::coinbase {
class CoinbaseSubscription;
}
namespace bts::md::binance_usds_futures {
class BinanceUsdsFuturesSubscription;
}
// namespace bts::md::binance_us {
// class BinanceUsSubscription;
// }
// namespace bts::md::coinbase {
// class CoinbaseSubscription;
// }
// namespace bts::md::binance_usds_futures {
// class BinanceUsdsFuturesSubscription;
// }
namespace bts::md::deribit {
class DeribitSubscription;
}
// namespace bts::md::deribit {
// class DeribitSubscription;
// }
namespace bts::md::ftx {
class FTXSubscription;
}
// namespace bts::md::ftx {
// class FTXSubscription;
// }
namespace bts::md::okx {
class OkxSubscription;
}
// namespace bts::md::okx {
// class OkxSubscription;
// }
namespace bts::md {
// namespace bts::md {
class MarketDataSubscription {
public:
MarketDataSubscription();
// class MarketDataSubscription {
// public:
// MarketDataSubscription();
MarketDataSubscription(const MarketDataSubscription &) noexcept = delete;
MarketDataSubscription(MarketDataSubscription &&) noexcept = default;
MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept =
delete;
MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept =
default;
// MarketDataSubscription(const MarketDataSubscription &) noexcept = delete;
// MarketDataSubscription(MarketDataSubscription &&) noexcept = default;
// MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept =
// delete;
// MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept =
// default;
~MarketDataSubscription();
// ~MarketDataSubscription();
void add(binance::BinanceSubscription);
void add(binance_us::BinanceUsSubscription);
void add(coinbase::CoinbaseSubscription);
void add(binance_usds_futures::BinanceUsdsFuturesSubscription);
void add(deribit::DeribitSubscription);
void add(ftx::FTXSubscription);
void add(okx::OkxSubscription);
// void add(binance::BinanceSubscription);
// void add(binance_us::BinanceUsSubscription);
// void add(coinbase::CoinbaseSubscription);
// void add(binance_usds_futures::BinanceUsdsFuturesSubscription);
// void add(deribit::DeribitSubscription);
// void add(ftx::FTXSubscription);
// void add(okx::OkxSubscription);
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
// private:
// struct PImpl;
// std::unique_ptr<PImpl> me_;
// };
} // namespace bts::md
// } // namespace bts::md
@@ -0,0 +1,44 @@
#include "OnixsMarketDataProvider.h"
#include "spark/api/domain/Venue.pb.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/glbx/OnixsGlbxSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::domain::Venue;
using bts::api::md::MarketDataConfig;
using bts::md::OnixsMarketDataProvider;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct OnixsMarketDataProvider::PImpl {
MarketDataConfig config;
std::shared_ptr<IDGenerator> sec_id_store;
vector<shared_ptr<OnixsGlbxSubscription>> sources;
};
OnixsMarketDataProvider::OnixsMarketDataProvider(MarketDataConfig config)
: me_(std::make_unique<PImpl>()) {
me_->config = std::move(config);
}
OnixsMarketDataProvider::~OnixsMarketDataProvider() = default;
vector<pair<Venue, shared_ptr<IMarketDataSource>>>
OnixsMarketDataProvider::get_sources() {
vector<pair<Venue, shared_ptr<IMarketDataSource>>> rv;
for (const auto &glbx : me_->config.glbx()) {
me_->sources.push_back(
std::make_shared<OnixsGlbxSubscription>(me_->sec_id_store, glbx));
rv.push_back({Venue::VENUE_GLBX, me_->sources.back()});
}
return rv;
}
} // namespace bts::md
@@ -0,0 +1,26 @@
#pragma once
#include "spark/api/md/MarketDataConfig.pb.h"
#include "sparkd/md/IMarketDataProvider.h"
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::md {
class OnixsMarketDataProvider : public IMarketDataProvider {
public:
OnixsMarketDataProvider();
explicit OnixsMarketDataProvider(api::md::MarketDataConfig config);
~OnixsMarketDataProvider();
std::vector<
std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,63 @@
#include "PyMarketDataProvider.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_assert.h"
#include "sparkd/md/IMarketDataSource.h"
#include "sparkd/md/MarketDataSubscription.h"
#include "sparkd/md/PyMarketDataSubscription.h"
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::os::sparkd_wd;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
using Path = boost::filesystem::path;
struct PyMarketDataProvider::PImpl {
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
vector<shared_ptr<PyMarketDataSubscription>> py_md_subs;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
PyMarketDataProvider::PyMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
PyMarketDataProvider::~PyMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
PyMarketDataProvider::get_sources() {
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> rv;
const Path wd{sparkd_wd()};
vector<string> module_paths{
wd.string(),
(wd / "python_md_providers").string(),
};
for (const auto &py_plugin : me_->config.python_md_providers()) {
try {
auto source = std::make_shared<PyMarketDataSubscription>(
py_plugin, me_->sec_id_store.get());
me_->py_md_subs.push_back(source);
rv.push_back(
std::make_pair<api::domain::Venue, shared_ptr<IMarketDataSource>>(
py_plugin.venue(), source));
} catch (const std::exception &ex) {
bts_assert_msgf(
false, "PyMarketDataSubscription threw: {}",
std::string{ex.what()});
}
}
return rv;
}
} // namespace bts::md
@@ -0,0 +1,27 @@
#pragma once
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/IMarketDataProvider.h"
#include <memory>
namespace bts::md {
class PyMarketDataProvider : public IMarketDataProvider {
public:
explicit PyMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~PyMarketDataProvider();
std::vector<
std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -10,17 +10,13 @@
#include "spark/api/md/PyMarketDataProviderConfig.pb.h"
#include "spark/core/TimeStamp.h"
#include "spark/domain/AggressorSide.h"
#include "spark/domain/DomainSecurityUtils.h"
#include "spark/domain/FutureSecurity.h"
#include "spark/domain/Security.h"
#include "spark/domain/SecurityID.h"
#include "spark/domain/SpotSecurity.h"
#include "spark/domain/TradeRecord.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/MDProducer.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_assert.h"
@@ -29,7 +25,7 @@
#include <optional>
#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
#include <unordered_map>
#include <utility>
#include <vector>
namespace bts::md {
@@ -44,23 +40,12 @@ using bts::md::py_interop::PySecurityList;
using bts::md::py_interop::PyTrade;
using bts::md::py_interop::PyTradeList;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
using Path = boost::filesystem::path;
struct VenueSink {
VenueSink(api::Venue venue, GeneralSecurityIndex *index)
: producer(), listener(*index, producer, venue) {}
MDProducer producer;
GeneralMarketDataListener listener;
};
struct TradeTuple {
py_interop::PyTrade py_trade;
domain::TradeRecord trade;
@@ -74,41 +59,40 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
{this->config.module_path().begin(),
this->config.module_path().end()},
this),
sec_id_store_(sec_id_store),
index_(std::make_unique<GeneralSecurityIndex>()) {}
sec_id_store_(sec_id_store) {}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener_ = std::move(listener);
LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
py_md_exec_.start();
}
void start() { py_md_exec_.start(); }
void stop() { py_md_exec_.stop(); }
private:
PyMarketDataProviderConfig config;
PyMdExecutor py_md_exec_;
unordered_map<api::Venue, shared_ptr<VenueSink>> sinks_;
shared_ptr<IMarketDataListener> listener_;
IDGenerator *sec_id_store_;
unique_ptr<GeneralSecurityIndex> index_;
// The main store for securities.
vector<domain::Security> securities_;
// We also maintain Security pointers per venue (the GeneralMarketDataListener does not support
// writing arbitrary groups of securities, they must share a common venue when batched together).
unordered_map<api::domain::Venue, vector<const domain::Security *>>
securities_by_venue_;
vector<const domain::Security *> security_ptrs_;
domain::Book book_;
// Since the python md provider isn't tethered to a single venue, we
// must know the venue for each individual trade.
vector<TradeTuple> trades_;
unordered_map<
SecurityID,
pair<api::domain::Venue, vector<const domain::TradeRecord *>>>
trades_by_security_;
vector<domain::TradeRecord> trades_;
vector<const domain::TradeRecord *> trade_ptrs_;
void on_securities(const PySecurityList &py_securities) override {
LOG(md, Info, "Received securities from python plugin. count: {}",
py_securities.size());
// `securities_` and `securities_by_venue_` should already be empty, but no harm in clearing
// them again. We keep them as member variables to avoid (re)allocating memory in each callback.
// `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them
// again. We keep them as member variables to avoid (re)allocating memory in each callback.
securities_.clear();
securities_by_venue_.clear();
security_ptrs_.clear();
for (const auto &py_sec : py_securities.items()) {
LOG(md, Info, "PyMarketDataSubscription adding security {}",
py_sec.name());
@@ -119,27 +103,23 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
py_sec.name());
continue;
}
securities_.emplace_back(std::move(*sec));
securities_.push_back(std::move(*sec));
}
// Populate vectors of security ptrs per each venue.
for (auto &sec : securities_) {
securities_by_venue_[sec.venue()].push_back(&sec);
security_ptrs_.push_back(&sec);
}
// Pass securities to the sinks (per venue).
for (const auto &[venue, secs] : securities_by_venue_) {
get_listener_(venue).on_securities(secs);
}
listener_->on_securities(security_ptrs_);
securities_.clear();
securities_by_venue_.clear();
security_ptrs_.clear();
}
void on_books(const PyBookList &py_books) override {
for (auto &py_book : py_books.items()) {
api::domain::Venue venue;
bts_assert(api::domain::venue_parse_smart(py_book.venue(), &venue));
bts_assert(book_.count_populated() == 0);
to_book_(py_book, book_);
get_listener_(venue).on_book(book_);
listener_->on_book(book_);
book_.clear();
}
}
@@ -147,38 +127,21 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
void on_trades(const PyTradeList &py_trades) override {
LOG(md, Info, "Received trades from python plugin. count: {}",
py_trades.size());
// `trades_` and `trades_by_security_` should already be empty, but no harm in clearing them
// again. We keep them as member variables to avoid (re)allocating memory in each callback.
// `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again.
// We keep them as member variables to avoid (re)allocating memory in each callback.
trades_.clear();
trades_by_security_.clear();
trade_ptrs_.clear();
for (const auto &py_trade : py_trades.items()) {
LOG(md, Info, "PyMarketDataSubscription received trade {}",
py_trade.to_string());
auto &trade = trades_.emplace_back();
bts_assert(
api::domain::venue_parse_smart(py_trade.venue(), &trade.venue));
to_bts_trade_(py_trade, trade.trade);
to_bts_trade_(py_trade, trades_.emplace_back());
}
// Populate vectors of trade ptrs per each security.
for (const auto &trade : trades_) {
auto &sec_trades = trades_by_security_[trade.trade.sec_id];
sec_trades.first = trade.venue;
sec_trades.second.push_back(&trade.trade);
}
// Pass trades to the sinks (per venue)
for (const auto &[sec_id, p] : trades_by_security_) {
get_listener_(p.first).on_trades(p.second);
for (const auto &t : trades_) {
trade_ptrs_.push_back(&t);
}
listener_->on_trades(trade_ptrs_);
trades_.clear();
trades_by_security_.clear();
}
GeneralMarketDataListener &get_listener_(api::domain::Venue venue) {
const auto api_venue = domain::DomainSecurityUtils::convert(venue);
return sinks_
.try_emplace(
api_venue, std::make_shared<VenueSink>(api_venue, index_.get()))
.first->second->listener;
trade_ptrs_.clear();
}
SecurityID get_sec_id_(const string &venue_name, const string &sec_name) {
@@ -274,12 +237,14 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
PyMarketDataSubscription::PyMarketDataSubscription(
PyMarketDataProviderConfig config, IDGenerator *sec_id_store)
: me_(std::make_unique<PyMarketDataSubscription::PImpl>(
std::move(config), sec_id_store)) {
LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
me_->start();
}
std::move(config), sec_id_store)) {}
PyMarketDataSubscription::~PyMarketDataSubscription() { me_->stop(); }
void PyMarketDataSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
void PyMarketDataSubscription::stop() { me_->stop(); }
} // namespace bts::md
@@ -1,5 +1,7 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::api::md {
@@ -12,13 +14,16 @@ class IDGenerator;
namespace bts::md {
class PyMarketDataSubscription {
class PyMarketDataSubscription : public IMarketDataSource {
public:
explicit PyMarketDataSubscription(
api::md::PyMarketDataProviderConfig config,
persistence::IDGenerator *sec_id_store);
~PyMarketDataSubscription();
void start(std::shared_ptr<IMarketDataListener> listener) override;
void stop() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
@@ -0,0 +1,52 @@
#include "ReplayMarketDataProvider.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/api/md/MarketDataConfigUtils.h"
#include "spark/config/config.h"
#include "spark/domain/DomainSecurityUtils.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/IMarketDataSource.h"
#include "sparkd/md/recording/MDReplayer.h"
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::domain::DomainSecurityUtils;
using bts::md::recording::MDReplayer;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct ReplayMarketDataProvider::PImpl {
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
ReplayMarketDataProvider::ReplayMarketDataProvider()
: ReplayMarketDataProvider(
config::get<MarketDataConfig>(), IDGenerator::instance_ptr()) {}
ReplayMarketDataProvider::ReplayMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
ReplayMarketDataProvider::~ReplayMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
ReplayMarketDataProvider::get_sources() {
const auto replay_cfgs =
MarketDataConfigUtils::get_effective_replay_config(me_->config);
for (const auto &r : replay_cfgs) {
const auto v = DomainSecurityUtils::convert(r.venue());
me_->sources.push_back({v, std::make_shared<MDReplayer>(r.venue())});
}
return me_->sources;
}
} // namespace bts::md
@@ -0,0 +1,28 @@
#pragma once
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/IMarketDataProvider.h"
#include <memory>
namespace bts::md {
class ReplayMarketDataProvider : public IMarketDataProvider {
public:
ReplayMarketDataProvider();
explicit ReplayMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~ReplayMarketDataProvider();
std::vector<
std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,47 @@
#include "ReplayMarketDataSubscription.h"
#include "spark/api/md/MarketDataReplayConfig.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/MDProducer.h"
#include "spark/utils/thread_manager.h"
#include "sparkd/md/recording/MDReplayer.h"
#include <ccapi_cpp/ccapi_event_dispatcher.h>
#include <optional>
namespace bts::md {
using bts::api::md::MarketDataReplayConfig;
using bts::md::recording::MDReplayer;
using std::string;
struct ReplayMarketDataSource::PImpl {
MarketDataReplayConfig config;
PImpl(const MarketDataReplayConfig &config) : config(config) {}
};
ReplayMarketDataSource::ReplayMarketDataSource(MarketDataReplayConfig config)
: me_(std::make_unique<PImpl>(config)) {
me_->config = std::move(config);
}
ReplayMarketDataSource::~ReplayMarketDataSource() = default;
void ReplayMarketDataSource::start(
std::shared_ptr<IMarketDataListener> listener) {
LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
auto md_replayer = std::make_shared<MDReplayer>(me_->config.venue());
std::string tid_name = "MDReplayer_" + Venue_Name(me_->config.venue());
bts::threading::make_thread(tid_name, [&, listener, md_replayer]() {
md_replayer->start(listener);
}).detach();
LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
void ReplayMarketDataSource::stop() {}
} // namespace bts::md
@@ -0,0 +1,25 @@
#pragma once
#include "spark/api/md/MarketDataReplayConfig.pb.h"
#include "spark/md/IMarketDataListener.h"
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::md {
class ReplayMarketDataSource : public IMarketDataSource {
public:
ReplayMarketDataSource();
explicit ReplayMarketDataSource(api::md::MarketDataReplayConfig config);
~ReplayMarketDataSource();
void start(std::shared_ptr<IMarketDataListener> listener) override;
void stop() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -168,7 +168,7 @@ struct BinanceSubscription::PImpl {
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
std::shared_ptr<IMarketDataListener> listener;
BinanceSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
@@ -191,11 +191,10 @@ struct BinanceSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
BinanceMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
session(dispatcher, mk_session_cfg(this->cfg), {}),
sec_requestor(
&session,
this->criteria,
@@ -206,13 +205,18 @@ struct BinanceSubscription::PImpl {
LOG(md, Info, "Initializing BinanceSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(std::shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
DepthHandler &mk_depth_handler(const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert(is_new);
return it->second;
}
@@ -220,7 +224,7 @@ struct BinanceSubscription::PImpl {
TickerHandler &mk_ticker_handler(const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -236,7 +240,8 @@ struct BinanceSubscription::PImpl {
SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)});
}
auto [it, is_new] = stats_handlers.try_emplace(
hnd_id, std::make_unique<StatsHandler>(sec_desc_id_pairs, listener));
hnd_id,
std::make_unique<StatsHandler>(sec_desc_id_pairs, listener.get()));
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", hnd_id);
return *it->second;
@@ -325,10 +330,8 @@ struct BinanceSubscription::PImpl {
BinanceSubscription::BinanceSubscription(
ccapi::EventDispatcher *dispatcher,
BinanceMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
BinanceSubscription::BinanceSubscription(BinanceSubscription &&) noexcept =
default;
@@ -337,4 +340,8 @@ BinanceSubscription &BinanceSubscription::operator=(
BinanceSubscription::~BinanceSubscription() = default;
void BinanceSubscription::start(std::shared_ptr<IMarketDataListener> listener) {
me_->start(listener);
}
} // namespace bts::md::binance
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,13 +26,12 @@ namespace bts::md::binance {
*
* @a https://binance-docs.github.io/apidocs/spot/en/#exchange-information
*/
class BinanceSubscription {
class BinanceSubscription : public IMarketDataSource {
public:
BinanceSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::BinanceMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
BinanceSubscription(const BinanceSubscription &) noexcept = delete;
BinanceSubscription(BinanceSubscription &&) noexcept;
@@ -39,10 +39,9 @@ class BinanceSubscription {
delete;
BinanceSubscription &operator=(BinanceSubscription &&) noexcept;
~BinanceSubscription();
~BinanceSubscription() override;
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -46,6 +46,7 @@ using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::uint32_t;
using std::unique_ptr;
@@ -171,7 +172,7 @@ struct BinanceUsSubscription::PImpl {
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
BinanceUsSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
@@ -188,17 +189,16 @@ struct BinanceUsSubscription::PImpl {
// different map, 'stats_handlers' (unique ids are generated w/ a counter,
// 'stats_handlers_id_counter').
unordered_map<string, StatsHandler *> sec_stats_handler_map;
unordered_map<uint32_t, std::unique_ptr<StatsHandler>> stats_handlers;
unordered_map<uint32_t, unique_ptr<StatsHandler>> stats_handlers;
uint32_t stats_handlers_id_counter = 0;
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
BinanceUsMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
session(dispatcher, mk_session_cfg(this->cfg), {}),
sec_requestor(
&session,
this->criteria,
@@ -209,13 +209,18 @@ struct BinanceUsSubscription::PImpl {
LOG(md, Info, "Initializing BinanceUsSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(std::shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
DepthHandler &mk_depth_handler(const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert(is_new);
return it->second;
}
@@ -223,7 +228,7 @@ struct BinanceUsSubscription::PImpl {
TickerHandler &mk_ticker_handler(const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -239,7 +244,8 @@ struct BinanceUsSubscription::PImpl {
SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)});
}
auto [it, is_new] = stats_handlers.try_emplace(
hnd_id, std::make_unique<StatsHandler>(sec_desc_id_pairs, listener));
hnd_id,
std::make_unique<StatsHandler>(sec_desc_id_pairs, listener.get()));
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", hnd_id);
return *it->second;
@@ -328,10 +334,8 @@ struct BinanceUsSubscription::PImpl {
BinanceUsSubscription::BinanceUsSubscription(
ccapi::EventDispatcher *dispatcher,
BinanceUsMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
BinanceUsSubscription::BinanceUsSubscription(
BinanceUsSubscription &&) noexcept = default;
@@ -340,4 +344,8 @@ BinanceUsSubscription &BinanceUsSubscription::operator=(
BinanceUsSubscription::~BinanceUsSubscription() = default;
void BinanceUsSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::binance_us
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,13 +26,12 @@ namespace bts::md::binance_us {
*
* @a https://docs.binance.us/#get-exchange-information
*/
class BinanceUsSubscription {
class BinanceUsSubscription : public IMarketDataSource {
public:
BinanceUsSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::BinanceUsMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
BinanceUsSubscription(const BinanceUsSubscription &) noexcept = delete;
BinanceUsSubscription(BinanceUsSubscription &&) noexcept;
@@ -41,8 +41,7 @@ class BinanceUsSubscription {
~BinanceUsSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -39,6 +39,7 @@ using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::shared_ptr;
using std::string;
using std::uint32_t;
using std::unique_ptr;
@@ -164,7 +165,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
SecurityCriteria criteria;
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
std::shared_ptr<IMarketDataListener> listener;
BinanceUsdsFuturesSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
@@ -180,15 +181,19 @@ struct BinanceUsdsFuturesSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
BinanceUsdsFuturesMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
session(dispatcher, mk_session_cfg(this->cfg), {}),
sec_requestor(&session, this->cfg, sec_id_store) {
LOG(md, Info, "Initializing BinanceUsdsFuturesSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
@@ -214,7 +219,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
DepthHandler &add_depth_handler(const string &sec_desc) {
const auto sec_id = get_sec_id(sec_desc);
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Depth handler already created for '{}'", sec_desc);
return it->second;
@@ -223,7 +228,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
TickerHandler &add_ticker_handler(const string &sec_desc) {
const auto sec_id = get_sec_id(sec_desc);
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -232,7 +237,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
StatsHandler &add_stats_handler(const string &sec_desc) {
const auto sec_id = get_sec_id(sec_desc);
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
stats_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Stats handler already created for '{}'", sec_desc);
return it->second;
@@ -304,10 +309,8 @@ struct BinanceUsdsFuturesSubscription::PImpl {
BinanceUsdsFuturesSubscription::BinanceUsdsFuturesSubscription(
ccapi::EventDispatcher *dispatcher,
BinanceUsdsFuturesMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
BinanceUsdsFuturesSubscription::BinanceUsdsFuturesSubscription(
BinanceUsdsFuturesSubscription &&) noexcept = default;
@@ -316,4 +319,9 @@ BinanceUsdsFuturesSubscription &BinanceUsdsFuturesSubscription::operator=(
BinanceUsdsFuturesSubscription::~BinanceUsdsFuturesSubscription() = default;
void BinanceUsdsFuturesSubscription::start(
shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::binance_usds_futures
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,13 +26,12 @@ namespace bts::md::binance_usds_futures {
*
* @a https://binance-docs.github.io/apidocs/futures/en/#exchange-information
*/
class BinanceUsdsFuturesSubscription {
class BinanceUsdsFuturesSubscription : public IMarketDataSource {
public:
BinanceUsdsFuturesSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::BinanceUsdsFuturesMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
BinanceUsdsFuturesSubscription(
const BinanceUsdsFuturesSubscription &) noexcept = delete;
@@ -43,8 +43,7 @@ class BinanceUsdsFuturesSubscription {
~BinanceUsdsFuturesSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -0,0 +1,140 @@
#include "CCApiMarketDataProvider.h"
#include "spark/api/domain/Venue.pb.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/config/config.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/MDProducer.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/MarketDataSubscription.h"
#include "sparkd/md/binance/BinanceSubscription.h"
#include "sparkd/md/binance_us/BinanceUsSubscription.h"
#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
#include "sparkd/md/coinbase/CoinbaseSubscription.h"
#include "sparkd/md/deribit/DeribitSubscription.h"
#include "sparkd/md/ftx/FTXSubscription.h"
#include "sparkd/md/okx/OkxSubscription.h"
#include <ccapi_cpp/ccapi_event_dispatcher.h>
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
static bool validate(
const MarketDataConfig & /*cfg*/, vector<string> & /*errs*/) {
return true;
}
struct CCApiMarketDataProvider::PImpl {
struct VenueSink {
GeneralSecurityIndex sec_index;
MDProducer md_producer;
GeneralMarketDataListener listener;
VenueSink(api::Venue venue, shared_ptr<IDGenerator> sec_id_store)
: sec_index(),
listener(sec_index, md_producer, venue, sec_id_store.get()) {}
};
MarketDataConfig config;
ccapi::EventDispatcher dispatcher;
shared_ptr<IDGenerator> sec_id_store;
unordered_map<api::Venue, unique_ptr<VenueSink>> venue_sinks;
vector<std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
VenueSink *get_venue_sink(api::Venue venue) {
auto it = venue_sinks.find(venue);
if (it != venue_sinks.end()) {
return it->second.get();
}
return venue_sinks
.try_emplace(venue, std::make_unique<VenueSink>(venue, sec_id_store))
.first->second.get();
}
};
CCApiMarketDataProvider::CCApiMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
CCApiMarketDataProvider::~CCApiMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
CCApiMarketDataProvider::get_sources() {
auto &cfg = me_->config;
vector<string> cfg_errs;
if (!validate(cfg, cfg_errs)) {
LOG(md, Error,
"CCApiMarketDataProvider::get_sources failed. Invalid config: {}",
fmt::to_string(fmt::join(cfg_errs, "\n")));
}
// Binance
for (const auto &binance : cfg.binance()) {
me_->sources.emplace_back(
api::domain::VENUE_BINANCE,
std::make_shared<binance::BinanceSubscription>(
&me_->dispatcher, binance, me_->sec_id_store.get()));
}
// Binance US
for (const auto &binance_us : cfg.binance_us()) {
me_->sources.emplace_back(
api::domain::VENUE_BINANCE_US,
std::make_shared<binance_us::BinanceUsSubscription>(
&me_->dispatcher, binance_us, me_->sec_id_store.get()));
}
// Binance USDS Futures
for (const auto &binance_ufs : cfg.binance_usds_futures()) {
me_->sources.emplace_back(
api::domain::VENUE_BINANCE_USDS_FUTURES,
std::make_shared<
binance_usds_futures::BinanceUsdsFuturesSubscription>(
&me_->dispatcher, binance_ufs, me_->sec_id_store.get()));
}
// Coinbase
for (const auto &coinbase : cfg.coinbase()) {
me_->sources.emplace_back(
api::domain::VENUE_COINBASE,
std::make_shared<coinbase::CoinbaseSubscription>(
&me_->dispatcher, coinbase, me_->sec_id_store.get()));
}
// Deribit
for (const auto &deribit : cfg.deribit()) {
me_->sources.emplace_back(
api::domain::VENUE_DERIBIT,
std::make_shared<deribit::DeribitSubscription>(
&me_->dispatcher, deribit, me_->sec_id_store.get()));
}
// FTX
for (const auto &ftx : cfg.ftx()) {
me_->sources.emplace_back(
api::domain::VENUE_FTX,
std::make_shared<ftx::FTXSubscription>(
&me_->dispatcher, ftx, me_->sec_id_store.get()));
}
// Okx
for (const auto &okx : cfg.okx()) {
me_->sources.emplace_back(
api::domain::VENUE_OKX,
std::make_shared<okx::OkxSubscription>(
&me_->dispatcher, okx, me_->sec_id_store.get()));
}
return me_->sources;
}
} // namespace bts::md
@@ -0,0 +1,27 @@
#pragma once
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/IMarketDataProvider.h"
#include <memory>
namespace bts::md {
class CCApiMarketDataProvider : public IMarketDataProvider {
public:
explicit CCApiMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~CCApiMarketDataProvider();
std::vector<
std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -5,8 +5,10 @@
#include "spark/domain/SecurityCriteria.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/io/QueryParams.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/common.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_assert.h"
#include "sparkd/md/ccapi/CCApiMarketDepthAdaptor.h"
#include "sparkd/md/ccapi/CCApiSession.h"
#include "sparkd/md/ccapi/CCApiTradeAdaptor.h"
@@ -32,6 +34,7 @@ using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
@@ -145,7 +148,7 @@ struct CoinbaseSubscription::PImpl {
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
CoinbaseSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
@@ -166,11 +169,10 @@ struct CoinbaseSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
CoinbaseMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
session(dispatcher, mk_session_cfg(this->cfg), {}),
sec_requestor(
&session,
this->criteria,
@@ -181,12 +183,18 @@ struct CoinbaseSubscription::PImpl {
LOG(md, Info, "Initializing CoinbaseSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
StatsHandler &mk_stats_handler(const string &sec_desc) {
auto sec_id = SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
stats_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", sec_id);
return it->second;
@@ -194,7 +202,7 @@ struct CoinbaseSubscription::PImpl {
DepthHandler &mk_depth_handler(const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert(is_new);
return it->second;
}
@@ -202,7 +210,7 @@ struct CoinbaseSubscription::PImpl {
TickerHandler &mk_ticker_handler(const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -273,10 +281,8 @@ struct CoinbaseSubscription::PImpl {
CoinbaseSubscription::CoinbaseSubscription(
ccapi::EventDispatcher *dispatcher,
CoinbaseMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
CoinbaseSubscription::CoinbaseSubscription(CoinbaseSubscription &&) noexcept =
default;
@@ -285,4 +291,8 @@ CoinbaseSubscription &CoinbaseSubscription::operator=(
CoinbaseSubscription::~CoinbaseSubscription() = default;
void CoinbaseSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::coinbase
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -20,13 +21,12 @@ class IDGenerator;
namespace bts::md::coinbase {
class CoinbaseSubscription {
class CoinbaseSubscription : public IMarketDataSource {
public:
CoinbaseSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::CoinbaseMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
CoinbaseSubscription(const CoinbaseSubscription &) noexcept = delete;
CoinbaseSubscription(CoinbaseSubscription &&) noexcept;
@@ -36,8 +36,7 @@ class CoinbaseSubscription {
~CoinbaseSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -0,0 +1,51 @@
#include "DatabentoMarketDataProvider.h"
#include "spark/api/domain/Venue.pb.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/IMarketDataSource.h"
#include "sparkd/md/databento/DatabentoDataHandler.h"
#include "sparkd/md/glbx/DatabentoGlbxSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct DatabentoMarketDataProvider::PImpl {
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
vector<shared_ptr<IMarketDataSource>> glbx_sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
DatabentoMarketDataProvider::DatabentoMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
me_->config = std::move(config);
}
DatabentoMarketDataProvider::~DatabentoMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
DatabentoMarketDataProvider::get_sources() {
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> rv;
for (const auto &glbx : me_->config.glbx()) {
if (glbx.use_databento()) {
me_->glbx_sources.push_back(
std::make_shared<DatabentoGlbxSubscription>(glbx, me_->sec_id_store));
rv.push_back({api::domain::VENUE_GLBX, me_->glbx_sources.back()});
}
}
return rv;
}
} // namespace bts::md
@@ -0,0 +1,33 @@
#pragma once
#include "sparkd/md/IMarketDataProvider.h"
#include <memory>
namespace bts::persistence {
class IDGenerator;
}
namespace bts::api::md {
class MarketDataConfig;
}
namespace bts::md {
class DatabentoMarketDataProvider : public IMarketDataProvider {
public:
explicit DatabentoMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~DatabentoMarketDataProvider();
std::vector<
std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -51,6 +51,7 @@ using bts::io::websocket::JsonRpcId;
using bts::io::websocket::JsonRpcRequest;
using bts::persistence::IDGenerator;
using nlohmann::json;
using std::shared_ptr;
using std::string;
using std::uint32_t;
using std::unique_ptr;
@@ -132,14 +133,13 @@ struct DeribitSubscription::PImpl {
const bool subscribe_trades = true;
size_t sub_id_counter = 0;
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
DeribitChannelDemuxer channel_demuxer;
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
DeribitMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
excluded_secs(
@@ -157,11 +157,15 @@ struct DeribitSubscription::PImpl {
criteria,
&instrument_store,
{this->cfg.currencies().begin(), this->cfg.currencies().end()}),
session(dispatcher, mk_session_cfg(this->cfg), {}),
listener(listener) {
session(dispatcher, mk_session_cfg(this->cfg), {}) {
// DON'T reference the ctr arg 'cfg' (it has been moved to 'this->cfg')
_INFO("ctr. config:\n{}", ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(std::shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request_all();
subscribe_instrument_state();
}
@@ -169,7 +173,7 @@ struct DeribitSubscription::PImpl {
SecurityData &add_security_data(const string sec_desc) {
const SecurityID sid{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
auto [it, is_new] = security_data_map.try_emplace(
sec_desc, std::make_unique<SecurityData>(sid, listener));
sec_desc, std::make_unique<SecurityData>(sid, listener.get()));
_ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc);
_INFO("add_security_data. sec: '{}'", sec_desc);
return *it->second;
@@ -294,10 +298,8 @@ struct DeribitSubscription::PImpl {
DeribitSubscription::DeribitSubscription(
ccapi::EventDispatcher *dispatcher,
DeribitMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
DeribitSubscription::DeribitSubscription(DeribitSubscription &&) noexcept =
default;
@@ -306,4 +308,8 @@ DeribitSubscription &DeribitSubscription::operator=(
DeribitSubscription::~DeribitSubscription() = default;
void DeribitSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::deribit
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -24,13 +25,12 @@ namespace bts::md::deribit {
* @brief Subscribes to market data at Deribit
* @a https://docs.deribit.com/?#market-data
*/
class DeribitSubscription {
class DeribitSubscription : public IMarketDataSource {
public:
DeribitSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::DeribitMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
DeribitSubscription(const DeribitSubscription &) noexcept = delete;
DeribitSubscription(DeribitSubscription &&) noexcept;
@@ -40,9 +40,8 @@ class DeribitSubscription {
~DeribitSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
@@ -0,0 +1,77 @@
#include "EmumbaDeribitSubscription.h"
#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h"
#include "spark/config/protos/system_config.pb.h"
#include "spark/utils/thread_manager.h"
#include "sparkd/md/emumba/EmumbaDeribitData.h"
namespace bts::md {
using bts::config::SystemConfig;
using Path = boost::filesystem::path;
struct EmumbaDeribitSubscription::PImpl {
api::md::EmumbaDeribitMarketDataConfig config;
};
EmumbaDeribitSubscription::EmumbaDeribitSubscription(
api::md::EmumbaDeribitMarketDataConfig config)
: me_(std::make_unique<PImpl>()) {
me_->config = std::move(config);
}
EmumbaDeribitSubscription::~EmumbaDeribitSubscription() = default;
void EmumbaDeribitSubscription::start(
std::shared_ptr<IMarketDataListener> /*listener*/) {
auto &cfg = me_->config;
LOG(md, Info, "EmumbaMarketDataProvider::start t_id:{} t_name:{}. cfg:\n{}",
os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
auto start_deribit_thr = [=]() {
const auto config_path = Path{config::get<SystemConfig>().config_path()} /
cfg.json_conf_file_name();
const auto conf =
bts::cmds::DeribitDemoConfiguration::read_file(config_path.string());
bts_assert_msg(conf.has_value(), "Failed to parse deribit config");
// log
for (const auto &s : conf.value().symbol_config_list) {
LOG(md, Info, "sub.symbol: {}", s.symbol_name);
LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future,
s.interested_in_option);
for (const auto &i : s.interested_instruments) {
LOG(md, Info, "sub.instrument: {}", i);
}
}
EmumbaDeribitData ddata_obj;
EmumbaDeribitData *ddata = &ddata_obj;
// end log
bts_assert(conf.has_value());
LOG(md, Info, "Deribit conf: {}", conf.value());
ddata->real_handler =
std::make_shared<emumba::deribit_connector::deribit_handler>(
conf.value().symbol_config_list, conf.value().trading_config,
conf.value().credential_map, conf.value().logging_configuration,
conf.value().deribit_url, &ddata->listener);
// since the trading interface is housed inside the EmMDListener, we need to
// create and send emumba's trading_session_interface
for (const auto &p : conf.value().credential_map) {
const auto &sess = p.second;
auto ts_ptr = std::make_shared<
emumba::deribit_connector::trading_session_interface>(
ddata->real_handler->create_new_trading_session(sess));
ddata->trading_session_vec.push_back({sess, ts_ptr});
}
bts_assert(!ddata->trading_session_vec.empty());
ddata->listener.set_handler(
ddata->real_handler.get(), ddata->trading_session_vec);
ddata->real_handler->run();
};
threading::make_thread("Emumba", [&]() { start_deribit_thr(); }).detach();
LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
void EmumbaDeribitSubscription::stop() {
//
}
} // namespace bts::md
@@ -0,0 +1,24 @@
#pragma once
#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h"
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::md {
class EmumbaDeribitSubscription : public IMarketDataSource {
public:
explicit EmumbaDeribitSubscription(
api::md::EmumbaDeribitMarketDataConfig config);
~EmumbaDeribitSubscription() override;
void start(std::shared_ptr<IMarketDataListener> listener) override;
void stop() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,40 @@
#include "EmumbaMarketDataProvider.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/api/md/MarketDataConfigUtils.h"
#include "sparkd/md/IMarketDataSource.h"
#include "sparkd/md/emumba/EmumbaDeribitSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::md::MarketDataConfig;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct EmumbaMarketDataProvider::PImpl {
MarketDataConfig config;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> deribit_subs;
PImpl(MarketDataConfig config) : config(std::move(config)) {}
};
EmumbaMarketDataProvider::EmumbaMarketDataProvider(MarketDataConfig config)
: me_(std::make_unique<PImpl>(std::move(config))) {}
EmumbaMarketDataProvider::~EmumbaMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
EmumbaMarketDataProvider::get_sources() {
for (const auto &cfg : me_->config.emumba_deribit()) {
me_->deribit_subs.push_back(
{api::domain::VENUE_DERIBIT,
std::make_shared<EmumbaDeribitSubscription>(cfg)});
}
return me_->deribit_subs;
}
} // namespace bts::md
@@ -0,0 +1,25 @@
#pragma once
#include "spark/api/md/MarketDataConfig.pb.h"
#include "sparkd/md/IMarketDataProvider.h"
#include <memory>
namespace bts::md {
class EmumbaMarketDataProvider : public IMarketDataProvider {
public:
EmumbaMarketDataProvider();
explicit EmumbaMarketDataProvider(api::md::MarketDataConfig config);
~EmumbaMarketDataProvider();
std::vector<
std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -36,6 +36,7 @@ using bts::domain::SecurityCriteria;
using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::persistence::IDGenerator;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
@@ -102,7 +103,7 @@ struct FTXSubscription::PImpl {
FTXSecurityRequestor sec_requestor;
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
// These aliases are useful for quickly switching the handler types (e.g.,
// a special handler that logs messages for testing/diagnostic purposes)
@@ -117,23 +118,27 @@ struct FTXSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
FTXMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener)
persistence::IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(domain::SecurityCriteria::from_proto(
this->cfg.security_criteria())),
sec_requestor(dispatcher, criteria, sec_id_store), session(dispatcher),
listener(listener) {
sec_requestor(dispatcher, criteria, sec_id_store),
session(dispatcher) {
LOG(md, Info, "Initializing FTXSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
DepthHandler &create_depth_handler(string sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert(is_new);
return it->second;
}
@@ -141,7 +146,7 @@ struct FTXSubscription::PImpl {
TickerHandler &create_ticker_handler(string sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -150,7 +155,7 @@ struct FTXSubscription::PImpl {
StatsHandler &create_stats_handler(string sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
stats_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Statistics handler already created for '{}'", sec_desc);
return it->second;
@@ -221,10 +226,8 @@ struct FTXSubscription::PImpl {
FTXSubscription::FTXSubscription(
ccapi::EventDispatcher *dispatcher,
FTXMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
FTXSubscription::FTXSubscription(FTXSubscription &&) noexcept = default;
FTXSubscription &FTXSubscription::operator=(FTXSubscription &&) noexcept =
@@ -232,4 +235,8 @@ FTXSubscription &FTXSubscription::operator=(FTXSubscription &&) noexcept =
FTXSubscription::~FTXSubscription() = default;
void FTXSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::ftx
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,20 +26,21 @@ namespace bts::md::ftx {
*
* @details Uses ccapi infrastructure to request/subscribe data
*/
class FTXSubscription {
class FTXSubscription : public IMarketDataSource {
public:
FTXSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::FTXMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
FTXSubscription(const FTXSubscription &) noexcept = delete;
FTXSubscription(FTXSubscription &&) noexcept;
FTXSubscription &operator=(const FTXSubscription &) noexcept = delete;
FTXSubscription &operator=(FTXSubscription &&) noexcept;
~FTXSubscription();
~FTXSubscription() override;
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -0,0 +1,41 @@
#include "DatabentoGlbxSubscription.h"
#include "spark/api/md/GlbxMarketDataConfig.pb.h"
#include "spark/utils/thread_manager.h"
#include "sparkd/md/databento/DatabentoDataHandler.h"
#include "sparkd/md/glbx/GlbxMarketDataHandler.h"
namespace bts::md {
using bts::api::md::GlbxMarketDataConfig;
using bts::persistence::IDGenerator;
using std::shared_ptr;
struct DatabentoGlbxSubscription::PImpl {
GlbxMarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
PImpl(GlbxMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
DatabentoGlbxSubscription::DatabentoGlbxSubscription(
GlbxMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
DatabentoGlbxSubscription::~DatabentoGlbxSubscription() = default;
void DatabentoGlbxSubscription::start(
std::shared_ptr<IMarketDataListener> listener) {
bts::threading::make_thread(
"glbx",
[config = me_->config, sec_id_store = me_->sec_id_store,
listener = std::move(listener)]() {
api::md::glbx::GlbxMarketDatahandler::run(
std::move(listener), std::move(sec_id_store), config);
})
.detach();
// TODO: implement 'stop' (signal the thread above to terminate, then join it).
}
} // namespace bts::md
@@ -0,0 +1,47 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::api::md {
class GlbxMarketDataConfig;
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
/**
* @brief Subscribes to market data at Glbx via Databento
*
* @a https://www.Databento.biz/
*/
class DatabentoGlbxSubscription : public IMarketDataSource {
public:
DatabentoGlbxSubscription(
api::md::GlbxMarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
DatabentoGlbxSubscription(const DatabentoGlbxSubscription &) noexcept =
delete;
DatabentoGlbxSubscription(DatabentoGlbxSubscription &&) noexcept;
DatabentoGlbxSubscription &operator=(
const DatabentoGlbxSubscription &) noexcept = delete;
DatabentoGlbxSubscription &operator=(DatabentoGlbxSubscription &&) noexcept;
~DatabentoGlbxSubscription() override;
/**
* Starts market data subscription on a background thread.
*
* @note Does not block.
*/
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,118 @@
#include "OnixsGlbxSubscription.h"
#include "spark/api/md/GlbxMarketDataConfig.pb.h"
#include "spark/core/LOCK_GUARD.h"
#include "spark/logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/SymbolLoader.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/Dispatcher.h"
#include "sparkd/md/platforms/IMDPlatform.h"
#include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
#include <atomic>
#include <memory>
#include <thread>
#include <vector>
namespace bts::md {
using bts::api::md::GlbxMarketDataConfig;
using bts::domain::SecurityCriteria;
using bts::md::IMDPlatform;
using bts::persistence::IDGenerator;
using std::shared_ptr;
using std::string;
using std::vector;
struct OnixsGlbxSubscription::PImpl {
GlbxMarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
PImpl(shared_ptr<IDGenerator> sec_id_store, GlbxMarketDataConfig config)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
std::thread dispatcher_thread;
utils::Dispatcher *dispatcher{nullptr};
std::optional<utils::DispatcherTimer> heartbeat_timer;
bool is_running = false;
std::mutex mx;
~PImpl() { stop(); }
/**
* Starts market data subscription on this thread.
*
* @note Designed to be called from a thread callback inside `OnixsGlbxSubscription::start`.
* @note Blocks until subscription stops.
*/
void run() {
{
BTS_LOCK_GUARD(mx);
if (!is_running) {
return;
}
const auto symbols = SymbolLoader::load(
SecurityCriteria::create({api::domain::VENUE_GLBX}));
if (symbols.empty()) {
LOG(md, Info, "OnixsGlbxSubscription aborting. No symbols loaded");
return;
}
LOG(md, Info,
"OnixsGlbxSubscription starting. t_id: {}; t_name: {}; config: "
"{}; symbols: {}",
os::get_thread_id(), os::get_thread_name(), config.DebugString(),
FMT_JOIN(symbols, ", "));
vector<shared_ptr<IMDPlatform>> platforms;
auto main = shared_ptr<MDP3Main>();
main->start({symbols.begin(), symbols.end()}, nullptr, nullptr);
platforms.emplace_back(std::move(main));
dispatcher = &utils::Dispatcher::current();
heartbeat_timer.emplace(*dispatcher, 250, [&]() {
for (auto &p : platforms) {
p->send_heartbeats();
}
});
heartbeat_timer->start();
is_running = true;
}
// This will block until dispatcher->terminate() is called.
// There is a slim, but non-zero chance that this is executed after
// 'stop' is called, since we 'run' outside of the lock.
dispatcher->run();
}
void stop() {
BTS_LOCK_GUARD(mx);
if (!is_running) {
return;
}
is_running = false;
if (dispatcher != nullptr) {
LOG(md, Diag, "OnixsGlbxSubscription stopping.");
dispatcher->io_service()->stop();
}
if (dispatcher_thread.joinable()) {
LOG(md, Diag, "OnixsGlbxSubscription joining dispatcher thread.");
dispatcher_thread.join();
}
}
};
OnixsGlbxSubscription::OnixsGlbxSubscription(
shared_ptr<IDGenerator> sec_id_store, GlbxMarketDataConfig config)
: me_(std::make_unique<PImpl>(std::move(sec_id_store), std::move(config))) {
}
OnixsGlbxSubscription::~OnixsGlbxSubscription() = default;
void OnixsGlbxSubscription::start(
shared_ptr<IMarketDataListener> /*listener*/) {
std::atomic_thread_fence(std::memory_order_release);
bts_assert(!me_->dispatcher_thread.joinable());
me_->dispatcher_thread = std::thread{[&]() { me_->run(); }};
}
void OnixsGlbxSubscription::stop() { me_->stop(); }
} // namespace bts::md
@@ -0,0 +1,48 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::api::md {
class GlbxMarketDataConfig;
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
/**
* @brief Subscribes to market data at Glbx via Onixs
*
* @a https://www.onixs.biz/
*/
class OnixsGlbxSubscription : public IMarketDataSource {
public:
OnixsGlbxSubscription(
std::shared_ptr<persistence::IDGenerator> sec_id_store,
api::md::GlbxMarketDataConfig config);
OnixsGlbxSubscription(const OnixsGlbxSubscription &) noexcept = delete;
OnixsGlbxSubscription(OnixsGlbxSubscription &&) noexcept;
OnixsGlbxSubscription &operator=(const OnixsGlbxSubscription &) noexcept =
delete;
OnixsGlbxSubscription &operator=(OnixsGlbxSubscription &&) noexcept;
~OnixsGlbxSubscription() override;
/**
* Starts market data subscription on a background thread.
*
* @note Does not block.
*/
void start(std::shared_ptr<IMarketDataListener> listener) override;
void stop() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -45,6 +45,7 @@ using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::persistence::IDGenerator;
using std::optional;
using std::shared_ptr;
using std::size_t;
using std::string;
using std::unique_ptr;
@@ -134,14 +135,13 @@ struct OkxSubscription::PImpl {
const bool subscribe_trades = true;
size_t sub_id_counter = 0;
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
OkxChannelDemuxer channel_demuxer;
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
OkxMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
excluded_secs(
@@ -155,10 +155,16 @@ struct OkxSubscription::PImpl {
!CONTAINS(excluded_secs, s.desc());
}),
session(
dispatcher, mk_session_cfg(this->cfg), mk_session_opts(this->cfg)),
listener(listener) {
dispatcher,
mk_session_cfg(this->cfg),
mk_session_opts(this->cfg)) {
// DON'T reference the ctr arg 'cfg' (it has been moved to 'this->cfg')
_INFO("ctr. config:\n{}", ProtoPrinter::to_string(this->cfg));
}
void start(std::shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = listener;
sec_sub.emplace(
&session, &instrument_store, this->criteria,
[this](const auto &secs) { on_securities(secs); });
@@ -167,7 +173,7 @@ struct OkxSubscription::PImpl {
SecurityData &add_security_data(const string sec_desc) {
const SecurityID sid{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
auto [it, is_new] = security_data_map.try_emplace(
sec_desc, std::make_unique<SecurityData>(sid, listener));
sec_desc, std::make_unique<SecurityData>(sid, listener.get()));
_ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc);
_INFO("add_security_data. sec: '{}'", sec_desc);
return *it->second;
@@ -248,10 +254,8 @@ struct OkxSubscription::PImpl {
OkxSubscription::OkxSubscription(
ccapi::EventDispatcher *dispatcher,
OkxMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
OkxSubscription::OkxSubscription(OkxSubscription &&) noexcept = default;
OkxSubscription &OkxSubscription::operator=(OkxSubscription &&) noexcept =
@@ -259,4 +263,9 @@ OkxSubscription &OkxSubscription::operator=(OkxSubscription &&) noexcept =
OkxSubscription::~OkxSubscription() = default;
void OkxSubscription::start(
std::shared_ptr<bts::md::IMarketDataListener> listener) {
me_->start(listener);
}
} // namespace bts::md::okx
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,25 +26,22 @@ namespace bts::md::okx {
*
* @a https://www.okx.com/docs-v5/en/#websocket-api-public-channel
*/
class OkxSubscription {
class OkxSubscription : public IMarketDataSource {
public:
OkxSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::OkxMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
OkxSubscription(const OkxSubscription &) noexcept = delete;
OkxSubscription(OkxSubscription &&) noexcept;
OkxSubscription &operator=(const OkxSubscription &) noexcept =
delete;
OkxSubscription &operator=(const OkxSubscription &) noexcept = delete;
OkxSubscription &operator=(OkxSubscription &&) noexcept;
~OkxSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
@@ -12,9 +12,11 @@
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/MDProducer.h"
#include "spark/md/protos/co2.pb.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/PBUtils.h"
#include "spark/utils/bts_assert.h"
#include "sparkd/md/recording/MDRecorder.h"
@@ -52,27 +54,26 @@ void copy_books(const bts::co2::BookMessage &from, bts::domain::Book &to) {
struct MDReplayer ::PImpl {
bts::api::Venue venue;
std::set<uint32_t> sec_keys = {};
uint64_t start_time = bts::core::TimeStamp::now_ns();
GeneralMarketDataListener listener;
uint64_t start_time = 0;
std::shared_ptr<IMarketDataListener> listener{nullptr};
double replay_speed;
PImpl(
bts::api::Venue venue,
bts::md::GeneralSecurityIndex &index,
bts::md::MDProducer &md_producer, double replay_speed)
: venue(venue), listener(index, md_producer, venue), replay_speed(replay_speed) {}
PImpl(bts::api::Venue venue, double replay_speed)
: venue(venue), replay_speed(replay_speed) {}
};
MDReplayer::MDReplayer(
bts::api::Venue venue,
bts::md::GeneralSecurityIndex &index,
bts::md::MDProducer &md_producer,
double replay_speed)
: me(new PImpl(venue, index, md_producer, replay_speed)) {
MDReplayer::MDReplayer(bts::api::Venue venue, double replay_speed)
: me(new PImpl(venue, replay_speed)) {
LOG(md, Info, "MDReplayer::MDReplayer venue: {}", Venue_Name(venue));
}
MDReplayer::~MDReplayer() {}
void MDReplayer::run() {
void MDReplayer::start(std::shared_ptr<IMarketDataListener> listener) {
LOG(md, Info, "MDReplayer::start venue: {}", Venue_Name(me->venue));
bts_assert(listener != nullptr);
me->listener = std::move(listener);
me->start_time = bts::core::TimeStamp::now_ns();
std::string directory = MDRecorder::recording_path();
auto include_file = [&](const std::string &file_name) {
return MDRecorder::is_md_recording_file(file_name, me->venue);
@@ -132,7 +133,7 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) {
me->sec_keys.insert(m.security().id());
std::vector<domain::Security const *> sec_vec = {};
sec_vec.push_back(&new_sec);
me->listener.on_securities(sec_vec);
me->listener->on_securities(sec_vec);
break;
}
case (co2::RecordedMDMessage::MDRecordCase::kTotalBook): {
@@ -146,20 +147,20 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) {
bts::domain::Book book = {};
copy_books(tot_book.direct_book(), book);
book.set_security_id(bts::domain::SecurityID(m.sec_key()));
me->listener.on_book(book);
me->listener->on_book(book);
}
break;
}
case (co2::RecordedMDMessage::MDRecordCase::kMisc): {
bts::domain::SecurityMarketDataStatistics stats(m.sec_key(), m.misc());
me->listener.on_security_statistics(stats);
me->listener->on_security_statistics(stats);
break;
}
case (co2::RecordedMDMessage::MDRecordCase::kTrade): {
bts::domain::TradeRecord tr(m.trade());
std::vector<domain::TradeRecord const *> trades;
trades.push_back(&tr);
me->listener.on_trades(trades);
me->listener->on_trades(trades);
break;
}
default: {
@@ -1,7 +1,8 @@
#pragma once
#include "spark/domain_model/Security.pb.h"
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/IMarketDataListener.h"
#include "sparkd/md/IMarketDataSource.h"
namespace bts::md {
class MDProducer;
@@ -14,15 +15,12 @@ class RecordedMDMessage;
namespace bts::md::recording {
class MDReplayer {
class MDReplayer : public IMarketDataSource {
public:
MDReplayer(
bts::api::Venue venue,
bts::md::GeneralSecurityIndex &index,
bts::md::MDProducer &md_producer,
double replay_speed=1);
~MDReplayer();
void run();
explicit MDReplayer(bts::api::Venue venue, double replay_speed = 1);
~MDReplayer() override;
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -0,0 +1,50 @@
#include "TaifexMarketDataProvider.h"
#include "spark/api/domain/Venue.pb.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/taifex/TaifexSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::domain::Venue;
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct TaifexMarketDataProvider::PImpl {
MarketDataConfig config;
std::shared_ptr<IDGenerator> sec_id_store;
vector<shared_ptr<TaifexSubscription>> sources;
PImpl(
MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
TaifexMarketDataProvider::TaifexMarketDataProvider(
MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
TaifexMarketDataProvider::~TaifexMarketDataProvider() = default;
vector<pair<Venue, shared_ptr<IMarketDataSource>>>
TaifexMarketDataProvider::get_sources() {
vector<pair<Venue, shared_ptr<IMarketDataSource>>> rv;
for (const auto &taifex : me_->config.taifex()) {
me_->sources.push_back(
std::make_shared<TaifexSubscription>(taifex, me_->sec_id_store));
rv.push_back({Venue::VENUE_TAIFEX, me_->sources.back()});
}
return rv;
}
} // namespace bts::md
@@ -0,0 +1,33 @@
#pragma once
#include "sparkd/md/IMarketDataProvider.h"
#include <memory>
namespace bts::persistence {
class IDGenerator;
}
namespace bts::api::md {
class MarketDataConfig;
}
namespace bts::md {
class TaifexMarketDataProvider : public IMarketDataProvider {
public:
explicit TaifexMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~TaifexMarketDataProvider();
std::vector<
std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,41 @@
#include "TaifexSubscription.h"
#include "spark/api/md/TaifexMarketDataConfig.pb.h"
#include "spark/utils/thread_manager.h"
#include "sparkd/md/databento/DatabentoDataHandler.h"
#include "sparkd/md/glbx/GlbxMarketDataHandler.h"
#include "sparkd/md/taifex/TaifexDataGateway.h"
namespace bts::md {
using bts::api::md::TaifexMarketDataConfig;
using bts::persistence::IDGenerator;
using std::shared_ptr;
struct TaifexSubscription::PImpl {
TaifexMarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
PImpl(TaifexMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
TaifexSubscription::TaifexSubscription(
TaifexMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
TaifexSubscription::~TaifexSubscription() = default;
void TaifexSubscription::start(std::shared_ptr<IMarketDataListener> listener) {
bts::threading::make_thread(
"taifex",
[config = me_->config, sec_id_store = me_->sec_id_store,
listener = std::move(listener)]() {
md::taifex::TaifexDataGateway::run(
config, std::move(listener), std::move(sec_id_store));
})
.detach();
// TODO: implement 'stop' (signal the thread above to terminate, then join it).
}
} // namespace bts::md
@@ -0,0 +1,43 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::api::md {
class TaifexMarketDataConfig;
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
/**
* @brief Subscribes to market data at Taifex
*/
class TaifexSubscription : public IMarketDataSource {
public:
TaifexSubscription(
api::md::TaifexMarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
TaifexSubscription(const TaifexSubscription &) noexcept = delete;
TaifexSubscription(TaifexSubscription &&) noexcept;
TaifexSubscription &operator=(const TaifexSubscription &) noexcept = delete;
TaifexSubscription &operator=(TaifexSubscription &&) noexcept;
~TaifexSubscription() override;
/**
* Starts market data subscription on a background thread.
*
* @note Does not block.
*/
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
+7
View File
@@ -0,0 +1,7 @@
#pragma once
#define BTS_LOCK_GUARD(lockable) \
std::lock_guard<decltype(lockable)> __lock_guard { lockable }
#define BTS_LOCK_GUARD_AS(lockable, name) \
std::lock_guard<decltype(lockable)> __lock_guard_##name { lockable }
@@ -177,8 +177,7 @@ struct GeneralMarketDataListener::PImpl {
const bool use_given_sec_key = true;
const std::string cc_name_prefix = "cc_security";
size_t n_bk_updates = 0;
bts::persistence::IDGenerator &id_gen =
bts::persistence::IDGenerator::instance();
bts::persistence::IDGenerator *id_gen;
GeneralSecurityIndex &index;
MDProducer &md_producer;
const std::shared_ptr<bts::orderman::SimExecutionGateway> sim_ex_gwy =
@@ -195,8 +194,10 @@ struct GeneralMarketDataListener::PImpl {
explicit PImpl(
GeneralSecurityIndex &index,
MDProducer &md_producer,
std::shared_ptr<bts::orderman::SimExecutionGateway> sim_ex_gwy)
: index(index), md_producer(md_producer), sim_ex_gwy(sim_ex_gwy) {
std::shared_ptr<bts::orderman::SimExecutionGateway> sim_ex_gwy,
persistence::IDGenerator *sec_id_store)
: id_gen(sec_id_store), index(index), md_producer(md_producer),
sim_ex_gwy(sim_ex_gwy) {
if (sim_ex_gwy != nullptr) {
sim_ex_gwy->set_producers(&index);
}
@@ -258,12 +259,16 @@ void copy_books(
// interface functions ////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
GeneralMarketDataListener::GeneralMarketDataListener(
GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue)
GeneralSecurityIndex &index,
MDProducer &md_producer,
api::Venue venue,
persistence::IDGenerator *sec_id_store)
: me(std::make_unique<PImpl>(
index,
md_producer,
bts::orderman::SimExecutionGatewayFactory::get()->get_getayeway(
venue))) {
venue),
sec_id_store)) {
LOG(md, Info,
"GeneralMarketDataListener::GeneralMarketDataListener venue: {}",
bts::api::Venue_Name(venue));
@@ -402,7 +407,7 @@ CCSecSearchResult GeneralMarketDataListener::_find_sec(
if (!me->use_given_sec_key) {
auto logical_key = get_logical_key(sec);
auto new_key_opt =
me->id_gen.get_id(me->cc_name_prefix, logical_key, true);
me->id_gen->get_id(me->cc_name_prefix, logical_key, true);
bts_assert(new_key_opt.has_value());
bts_id = new_key_opt.value();
}
@@ -550,7 +555,7 @@ uint64_t GeneralMarketDataListener::_generate_security_key(
auto logical_key = get_logical_key(sec);
// if (me->use_given_sec_key)
// return sec.security_common.id.value();
auto rv = me->id_gen.get_id(me->cc_name_prefix, logical_key, true);
auto rv = me->id_gen->get_id(me->cc_name_prefix, logical_key, true);
bts_assert(rv.has_value());
return rv.value();
}
@@ -5,6 +5,7 @@
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/Shm/FixedSecurity.h"
#include "spark/persistence/IDGenerator.h"
namespace bts::md {
class MDProducer;
@@ -16,7 +17,10 @@ namespace bts::md {
class GeneralMarketDataListener : public IMarketDataListener {
public:
GeneralMarketDataListener(
GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue);
GeneralSecurityIndex &index,
MDProducer &md_producer,
api::Venue venue,
persistence::IDGenerator *sec_id_store);
virtual ~GeneralMarketDataListener();
void on_securities(const std::vector<domain::Security const *>
&complete_security_list) override;