refactor: Split MarketDataGateway up into providers/sources

MarketDataGateway still exists, but is greatly simplified. It creates
several `IMarketDataProvider`s, each of which generates a list of
`IMarketDataSource`s. The market data sources provide market data to an
`IMarketDataListener`.

* EmumbaMarketDataProvider
* CCApiMarketDataProvider
* PyMarketDataProvider
* OnixsMarketDataProvider
* ReplayMarketDataProvider

As a side effect, individual py market data sources are now explicitly
limited to a single venue (it probably should have been this way all
along, since the shared memory writers do not expect updates to involve
multiple venues).
This commit is contained in:
John Colvin committed 2024-11-25 19:21:48 +00:00
1 parent d68fe93422
commit 0aaa6e18fd
60 files changed
+1724 -758

No files matched your search

+9 -1
View File
@@ -1,4 +1,5 @@
#include "Run.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/api/public/ServiceContext.h"
#include "spark/config/config.h"
#include "spark/config/protos/ports.pb.h"
@@ -15,6 +16,7 @@
#include "spark/orderman/OrderServiceServer.h"
#include "spark/orderman/events/TradeSummarySignalDispatcher.h"
#include "spark/orderman/order_factory/DynamicOrderRegistry.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/persistence/PersistenceServiceServer.h"
#include "spark/plugin/SparkServices.h"
#include "spark/plugin/plugin.h"
@@ -24,8 +26,10 @@
#include "md/cmds/Run.h"
#include "sparkd/algo/AlgoDefinitionServiceServer.h"
#include "sparkd/analytics/AnalyticsServiceServer.h"
#include "sparkd/cmds/md/cmds/Cleanup.h"
#include "sparkd/http/WebServer.h"
#include "sparkd/md/MarketDataAuxiliaryServiceServer.h"
#include "sparkd/md/MarketDataGateway.h"
#include "sparkd/md/recording/MDRecorder.h"
#include "sparkd/position/MarginServiceServer.h"
#include "sparkd/position/PositionServiceServer.h"
@@ -63,7 +67,11 @@ static void execute() {
bts::md::MBOProcessor mbo_processor;
bts::md::TradeSummarySignalDispatcher tss_dispatcher;
md::cmds::Run::run();
md::cmds::Cleanup::execute();
bts::md::MarketDataGateway md_gateway{
config::get<api::md::MarketDataConfig>(),
persistence::IDGenerator::instance_ptr()};
md_gateway.start();
api::synthetic::SyntheticSecurityService synthetic_service(sc);
bts::md::MDClient mdclient(
@@ -1,5 +1,7 @@
#include "LiveLadderDemo.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/config/config.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/MarketDataGateway.h"
#include "sparkd/md/listeners/LiveLadderPrinter.h"
@@ -7,13 +9,16 @@
namespace bts::cmds {
LiveLadderDemo::LiveLadderDemo(const api::md::MarketDataConfig &) {
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
LiveLadderDemo::LiveLadderDemo(const MarketDataConfig &config) {
char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt";
putenv(ssl_cert_file);
md::MarketDataGateway gw;
md::MarketDataGateway gw{config, IDGenerator::instance_ptr()};
md::listeners::LiveLadderPrinter ladder_printer;
gw.run();
gw.start();
const std::size_t n_loops = 16;
for (std::size_t n = 0; n < n_loops; n++) {
+11 -3
View File
@@ -1,5 +1,6 @@
#include "Run.h"
#include "Cleanup.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/config/protos/market_data.pb.h"
#include "spark/domain/SecurityCriteria.h"
#include "spark/logging/logging.h"
@@ -7,6 +8,7 @@
#include "spark/md/UserSymbol.h"
#include "spark/md/relay/RelayRecorder.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/thread_manager.h"
#include "sparkd/md/MarketDataGateway.h"
@@ -14,6 +16,7 @@
#include "sparkd/md/platforms/ice_onixs/ICEMain.h"
#include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
#include <string>
#include <thread>
namespace bts::cmds::md::cmds {
@@ -91,9 +94,14 @@ static void run_onixs(
static void run_md_gwy() {
LOG(md, Info, "Run::run_md_gwy t_id:{} t_name:{}", os::get_thread_id(),
os::get_thread_name());
bts::md::MarketDataGateway gwy = {};
// this keeps going in a loop
gwy.run();
bts::md::MarketDataGateway gwy{
config::get<api::md::MarketDataConfig>(),
persistence::IDGenerator::instance_ptr()};
gwy.start();
while (true) {
using namespace std::chrono_literals;
std::this_thread::sleep_for(2s);
}
}
void Run::run() {
@@ -0,0 +1,7 @@
#include "IMarketDataProvider.h"
namespace bts::md {
IMarketDataProvider::~IMarketDataProvider() = default;
} // namespace bts::md
@@ -0,0 +1,25 @@
#pragma once
#include "spark/api/domain/Venue.pb.h"
#include "sparkd/md/IMarketDataSource.h"
#include <vector>
namespace bts::md {
/**
* @brief This is a single function interface, whose sole purpose is to provide a list of market
* data sources (per venue). A good example implementation is the `CCApiMarketDataProvider`, which
* reads `MarketDataConfig` and returns an `IMarketDataSource` for each venue configuration (e.g.,
* `bts::api::md::BinanceMarketDataConfig`, `bts::api::md::CoinbaseMarketDataConfig`, etc.).
*/
class IMarketDataProvider {
public:
virtual ~IMarketDataProvider();
virtual std::vector<
std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() = 0;
};
} // namespace bts::md
@@ -0,0 +1,9 @@
#include "IMarketDataSource.h"
namespace bts::md {
IMarketDataSource::~IMarketDataSource() = default;
void IMarketDataSource::stop() {}
} // namespace bts::md
@@ -0,0 +1,20 @@
#pragma once
#include <memory>
namespace bts::md {
class IMarketDataListener;
/**
* @brief This interface represents a source of market data. Call `IMarketDataSource::start` to
* connect an `IMarketDataListener` to the source and begin receiving market data.
*/
class IMarketDataSource {
public:
virtual ~IMarketDataSource();
virtual void start(std::shared_ptr<IMarketDataListener> listener) = 0;
virtual void stop();
};
} // namespace bts::md
+56 -385
View File
@@ -1,75 +1,41 @@
#include "MarketDataGateway.h"
#include "spark/api/md/CboeMarketDataConfig.pb.h"
#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/api/md/MarketDataConfigUtils.h"
#include "spark/api/md/MarketDataReplayConfig.pb.h"
#include "spark/collections/Contains.h"
#include "spark/config/protos/system_config.pb.h"
#include "spark/core/FMT.h"
#include "spark/domain/DomainSecurityUtils.h"
#include "spark/domain/SecurityID.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/MDProducer.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/bts_fatal.h"
#include "spark/utils/thread_manager.h"
#include "sparkd/md/MarketDataListenerID.h"
#include "sparkd/md/MarketDataRegistration.h"
#include "sparkd/md/MarketDataRegistrationError.h"
#include "sparkd/md/MarketDataRegistrationFactory.h"
#include "sparkd/md/MarketDataSubscription.h"
#include "sparkd/md/IMarketDataProvider.h"
#include "sparkd/md/OnixsMarketDataProvider.h"
#include "sparkd/md/PyMarketDataProvider.h"
#include "sparkd/md/PyMarketDataSubscription.h"
#include "sparkd/md/binance/BinanceSubscription.h"
#include "sparkd/md/binance_us/BinanceUsSubscription.h"
#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
#include "sparkd/md/cboe/CboeMarketDataHandler.h"
#include "sparkd/md/coinbase/CoinbaseSubscription.h"
#include "sparkd/md/deribit/DeribitSubscription.h"
#include "sparkd/md/emumba/EmumbaDeribitData.h"
#include "sparkd/md/ftx/FTXSubscription.h"
#include "sparkd/md/glbx/GlbxMarketDataHandler.h"
#include "sparkd/md/okx/OkxSubscription.h"
#include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
#include "sparkd/md/recording/MDReplayer.h"
#include "sparkd/md/taifex/TaifexDataGateway.h"
#include "sparkd/md/ReplayMarketDataProvider.h"
#include "sparkd/md/ccapi/CCApiMarketDataProvider.h"
#include "sparkd/md/databento/DatabentoMarketDataProvider.h"
#include "sparkd/md/emumba/EmumbaMarketDataProvider.h"
#include "sparkd/md/taifex/TaifexMarketDataProvider.h"
#include <bits/stdint-uintn.h>
#include <boost/filesystem.hpp>
#include <boost/filesystem/path.hpp>
#include <ccapi_cpp/ccapi_event_dispatcher.h>
#include <emumba-deribit-connector/deribit_utils.h>
#include <memory>
#include <optional>
#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
#include <string>
#include <unordered_map>
#include <vector>
namespace py = pybind11;
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::config::SystemConfig;
//using bts::domain::Security;
using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::persistence::IDGenerator;
using os::sparkd_wd;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
using Path = boost::filesystem::path;
static bool validate(const MarketDataConfig &cfg, vector<string> &errs) {
auto orig_n_errs = errs.size();
if (!cfg.deribit().empty() && !cfg.emumba_deribit().empty()) {
@@ -79,361 +45,66 @@ static bool validate(const MarketDataConfig &cfg, vector<string> &errs) {
return errs.size() == orig_n_errs;
}
struct MarketDataListenerData {
MarketDataListenerData() = default;
MarketDataListenerData(const MarketDataListenerData &) noexcept = delete;
MarketDataListenerData(MarketDataListenerData &&) noexcept = default;
MarketDataListenerData &operator=(const MarketDataListenerData &) noexcept =
delete;
MarketDataListenerData &operator=(MarketDataListenerData &&) noexcept =
default;
~MarketDataListenerData() = default;
MarketDataListenerID id;
MarketDataSubscription subscription;
};
struct MDReplayData {
bts::api::md::MarketDataReplayConfig replay_conf;
bts::api::Venue venue;
bts::md::GeneralSecurityIndex sec_index;
bts::md::MDProducer md_producer;
bts::md::recording::MDReplayer md_replayer;
MDReplayData(const bts::api::md::MarketDataReplayConfig &replay_conf)
: replay_conf(replay_conf), venue(replay_conf.venue()), sec_index(),
md_replayer(replay_conf.venue(), sec_index, md_producer, replay_conf.replay_speed()) {}
};
struct VenueListenerData {
bts::md::GeneralSecurityIndex sec_index;
bts::md::MDProducer md_producer;
std::shared_ptr<bts::md::IMarketDataListener> listener;
VenueListenerData(bts::api::Venue venue)
: sec_index(),
listener(std::make_shared<bts::md::GeneralMarketDataListener>(
sec_index, md_producer, venue)) {}
};
struct MarketDataGateway::PImpl {
api::md::MarketDataConfig md_config =
config::get<api::md::MarketDataConfig>();
ccapi::EventDispatcher dispatcher;
MarketDataRegistrationFactory registration_factory;
std::shared_ptr<IDGenerator> sec_id_store = IDGenerator::instance_ptr();
unordered_map<MarketDataListenerID, MarketDataListenerData> listeners;
std::vector<std::unique_ptr<MDReplayData>> replays = {};
// we create these and pass them to the cc md sources
std::vector<std::shared_ptr<VenueListenerData>> venue_listeners = {};
std::vector<std::shared_ptr<PyMarketDataSubscription>> py_md_subs = {};
struct MdSink {
GeneralSecurityIndex sec_index;
MDProducer md_producer;
shared_ptr<IMarketDataListener> listener;
MdSink(api::Venue venue, IDGenerator *sec_id_store)
: sec_index(), listener(std::make_shared<GeneralMarketDataListener>(
sec_index, md_producer, venue, sec_id_store)) {}
};
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
vector<unique_ptr<IMarketDataProvider>> providers;
vector<unique_ptr<MdSink>> sinks;
vector<unique_ptr<IMarketDataSource>> sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {
providers.emplace_back(std::make_unique<CCApiMarketDataProvider>(
this->config, this->sec_id_store));
providers.emplace_back(
std::make_unique<EmumbaMarketDataProvider>(this->config));
providers.emplace_back(
std::make_unique<OnixsMarketDataProvider>(this->config));
providers.emplace_back(std::make_unique<DatabentoMarketDataProvider>(
this->config, this->sec_id_store));
providers.emplace_back(std::make_unique<TaifexMarketDataProvider>(
this->config, this->sec_id_store));
providers.emplace_back(std::make_unique<PyMarketDataProvider>(
this->config, this->sec_id_store));
providers.emplace_back(std::make_unique<ReplayMarketDataProvider>(
this->config, this->sec_id_store));
}
};
MarketDataGateway::MarketDataGateway()
: me_(std::make_unique<MarketDataGateway::PImpl>()) {}
MarketDataGateway::MarketDataGateway(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<MarketDataGateway::PImpl>(
std::move(config), std::move(sec_id_store))) {}
MarketDataGateway::~MarketDataGateway() = default;
// starts the market data sources and then keeps the function going
void MarketDataGateway::run() {
void MarketDataGateway::start() {
vector<string> cfg_errs;
if (!validate(me_->md_config, cfg_errs)) {
if (!validate(me_->config, cfg_errs)) {
BTS_FATAL_MSGF("Invalid MarketDataConfig:", FMT_JOIN(cfg_errs, "\n"));
return;
}
LOG(md, Info, "MarketDataGateway::run starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
_start_cc(me_->md_config);
if (!me_->md_config.emumba_deribit().empty()) {
bts_assert(me_->md_config.emumba_deribit().size() == 1);
_start_emumba_deribit(me_->md_config.emumba_deribit().at(0));
}
start_glbx(me_->md_config);
start_taifex(me_->md_config);
start_cboe(me_->md_config);
_start_replays(me_->md_config);
start_py_();
while (true) {
std::this_thread::sleep_for(std::chrono::seconds(1));
}
LOG(md, Info, "MarketDataGateway::run ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
void MarketDataGateway::start_py_() {
for (const auto &py_plugin : me_->md_config.python_md_providers()) {
try {
me_->py_md_subs.push_back(std::make_shared<PyMarketDataSubscription>(
py_plugin, me_->sec_id_store.get()));
} catch (const std::exception &ex) {
bts_assert_msgf(
false, "PyMarketDataSubscription threw: {}",
std::string{ex.what()});
for (auto &provider : me_->providers) {
for (auto &[venue, source] : provider->get_sources()) {
const auto v = domain::DomainSecurityUtils::convert(venue);
auto &sink = me_->sinks.emplace_back(
std::make_unique<PImpl::MdSink>(v, me_->sec_id_store.get()));
source->start(sink->listener);
}
}
}
// CCAPI //////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
MarketDataRegistration MarketDataGateway::_start_cc(
const MarketDataConfig &cfg) noexcept {
LOG(md, Info, "MarketDataGateway::_start_cc starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
auto reg = me_->registration_factory.create();
bts_assert(reg.listener_id.has_value());
bts_assert(!CONTAINS_KEY(me_->listeners, *reg.listener_id));
LOG(md, Info, "MarketDataGateway subscribing to data:\n{}",
ProtoPrinter::to_string(cfg));
vector<string> cfg_errs;
if (!validate(cfg, cfg_errs)) {
return MarketDataRegistration{
{MarketDataRegistrationError::Type::CONFIG_ERROR,
fmt::to_string(fmt::join(cfg_errs, "\n"))}};
}
MarketDataListenerData &listener_data = me_->listeners[*reg.listener_id];
listener_data.id = *reg.listener_id;
// FTX
for (const auto &ftx : cfg.ftx()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_FTX));
listener_data.subscription.add(ftx::FTXSubscription{
&me_->dispatcher, ftx, me_->sec_id_store.get(), p->listener.get()});
}
// Binance
for (const auto &binance : cfg.binance()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_BINANCE));
listener_data.subscription.add(binance::BinanceSubscription{
&me_->dispatcher, binance, me_->sec_id_store.get(),
p->listener.get()});
}
// Binance US
for (const auto &binance_us : cfg.binance_us()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_BINANCE_US));
listener_data.subscription.add(binance_us::BinanceUsSubscription{
&me_->dispatcher, binance_us, me_->sec_id_store.get(),
p->listener.get()});
}
// Coinbase
for (const auto &coinbase : cfg.coinbase()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_COINBASE));
listener_data.subscription.add(coinbase::CoinbaseSubscription{
&me_->dispatcher, coinbase, me_->sec_id_store.get(),
p->listener.get()});
}
// Binance USDS Futures
for (const auto &binance_ufs : cfg.binance_usds_futures()) {
auto p =
me_->venue_listeners.emplace_back(std::make_shared<VenueListenerData>(
bts::api::VENUE_BINANCE_USDS_FUTURES));
listener_data.subscription.add(
binance_usds_futures::BinanceUsdsFuturesSubscription{
&me_->dispatcher, binance_ufs, me_->sec_id_store.get(),
p->listener.get()});
}
// Deribit
for (const auto &deribit : cfg.deribit()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_DERIBIT));
listener_data.subscription.add(deribit::DeribitSubscription{
&me_->dispatcher, deribit, me_->sec_id_store.get(),
p->listener.get()});
}
// Okx
for (const auto &okx : cfg.okx()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_OKX));
listener_data.subscription.add(okx::OkxSubscription{
&me_->dispatcher, okx, me_->sec_id_store.get(), p->listener.get()});
}
LOG(md, Info, "MarketDataGateway::_start_cc ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
return reg;
}
void MarketDataGateway::unsubscribe(MarketDataListenerID id) {
LOG(md, Info, "Unsubscribing listener {}", id);
me_->listeners.erase(id);
}
/// glbx //////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////
void MarketDataGateway::start_glbx(const bts::api::md::MarketDataConfig &cfg){
for (const auto &glbx : cfg.glbx()) {
if(!glbx.use_databento()){
_start_onix_glbx(glbx);
return;
}
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_GLBX));
// Capture `p->listener` and `me_->sec_id_store` by value
auto listener = p->listener;
auto sec_id_store = me_->sec_id_store;
std::string tid_name = "glbx";
bts::threading::make_thread(tid_name, [listener, sec_id_store, glbx](){
api::md::glbx::GlbxMarketDatahandler::run(
listener,
sec_id_store,
glbx);
}).detach();
}
}
static void run_onixs() {
LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name",
os::get_thread_id(), os::get_thread_name());
bts::md::MBOWriter *mbo = nullptr;
bts::md::TradeSummarySignalDispatcher *tssd = nullptr;
std::vector<std::shared_ptr<bts::md::IMDPlatform>> platforms;
// get all glbx symbols
auto cme_symbols = SymbolLoader::load(
bts::domain::SecurityCriteria::create({api::domain::VENUE_GLBX}));
if (!cme_symbols.empty()) {
auto *main = new bts::md::MDP3Main();
main->start({cme_symbols.begin(), cme_symbols.end()}, mbo, tssd);
platforms.emplace_back(main);
}
// deribit
// auto *d_platform = new bts::md::EmMDListener()
// Timers
auto &disp = utils::Dispatcher::current();
utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() {
for (auto &p : platforms)
p->send_heartbeats();
});
heartbeat_timer.start();
disp.run();
}
void MarketDataGateway::_start_onix_glbx(
bts::api::md::GlbxMarketDataConfig cfg) {
LOG(md, Info,
"MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. "
"cfg:\n{}",
os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
bts::threading::make_thread("Onix CME", [&]() {
bts::md::run_onixs();
}).detach();
}
// deribit ////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
void MarketDataGateway::_start_emumba_deribit(
bts::api::md::EmumbaDeribitMarketDataConfig cfg) {
LOG(md, Info,
"MarketDataGateway::_start_emumba_deribit starting t_id:{} t_name:{}. "
"cfg:\n{}",
os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
auto start_deribit_thr = [=]() {
const auto config_path = Path{config::get<SystemConfig>().config_path()} /
cfg.json_conf_file_name();
const auto conf =
bts::cmds::DeribitDemoConfiguration::read_file(config_path.string());
bts_assert_msg(conf.has_value(), "Failed to parse deribit config");
// log
for (const auto &s : conf.value().symbol_config_list) {
LOG(md, Info, "sub.symbol: {}", s.symbol_name);
LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future,
s.interested_in_option);
for (const auto &i : s.interested_instruments) {
LOG(md, Info, "sub.instrument: {}", i);
}
}
EmumbaDeribitData ddata_obj;
EmumbaDeribitData *ddata = &ddata_obj;
// end log
bts_assert(conf.has_value());
LOG(md, Info, "Deribit conf: {}", conf.value());
ddata->real_handler =
std::make_shared<emumba::deribit_connector::deribit_handler>(
conf.value().symbol_config_list, conf.value().trading_config,
conf.value().credential_map, conf.value().logging_configuration,
conf.value().deribit_url, &ddata->listener);
// since the trading interface is housed inside the EmMDListener, we need to
// create and send emumba's trading_session_interface
for (const auto &p : conf.value().credential_map) {
const auto &sess = p.second;
auto ts_ptr = std::make_shared<
emumba::deribit_connector::trading_session_interface>(
ddata->real_handler->create_new_trading_session(sess));
ddata->trading_session_vec.push_back({sess, ts_ptr});
}
bts_assert(!ddata->trading_session_vec.empty());
ddata->listener.set_handler(
ddata->real_handler.get(), ddata->trading_session_vec);
ddata->real_handler->run();
};
threading::make_thread("Emumba", [&]() { start_deribit_thr(); }).detach();
LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
// replays ////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
void MarketDataGateway::_start_replays(
const bts::api::md::MarketDataConfig &cfg) {
LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
for (const auto &r :
MarketDataConfigUtils::get_effective_replay_config(cfg)) {
me_->replays.emplace_back(std::make_unique<MDReplayData>(r));
}
for (const auto &replayer : me_->replays) {
std::string tid_name = "MDReplayer_" + Venue_Name(replayer->venue);
bts::threading::make_thread(tid_name, [&]() {
replayer->md_replayer.run();
}).detach();
}
LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) {
// taifex
for (const auto &taifex : cfg.taifex()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_TAIFEX));
// Capture `p->listener` and `me_->sec_id_store` by value
auto listener = p->listener;
auto sec_id_store = me_->sec_id_store;
std::string tid_name = "taifex";
bts::threading::make_thread(tid_name, [listener, sec_id_store, taifex]() {
bts::md::taifex::TaifexDataGateway::run(
taifex, listener, sec_id_store);
}).detach();
}
}
void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg) {
for (const api::md::CboeMarketDataConfig &cboe : cfg.cboe()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_CBOE));
std::string tid_name = "CBOE";
// Capture `p->listener` and `me_->sec_id_store` by value
auto listener = p->listener;
auto sec_id_store = me_->sec_id_store;
bts::threading::make_thread(tid_name, [listener, sec_id_store, cboe]() {
api::md::cboe::CboeMarketDatahandler::run(
listener,
sec_id_store,
cboe);
}).detach();
}
}
} // namespace bts::md
+13 -25
View File
@@ -1,45 +1,33 @@
#pragma once
#include "spark/api/md/GlbxMarketDataConfig.pb.h"
#include "sparkd/md/MarketDataListenerID.h"
#include <memory>
namespace bts::api::md {
class EmumbaDeribitMarketDataConfig;
class MarketDataConfig;
} // namespace bts::api::md
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
class IMarketDataListener;
class MarketDataRegistration;
/**
* @brief The `MarketDataGateway` is responsible for starting all configured market data sources.
* It connects each source to a special `IMarketDataListener` that writes the data to shared memory.
*/
class MarketDataGateway {
public:
MarketDataGateway();
explicit MarketDataGateway(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~MarketDataGateway();
/**
* @brief Connects a market data listener to a market data source
* @return MarketDataRegistration
*/
void run();
void unsubscribe(MarketDataListenerID id);
static void test_py();
void start();
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
MarketDataRegistration _start_cc(const api::md::MarketDataConfig &) noexcept;
void start_py_();
static void _start_emumba_deribit(
bts::api::md::EmumbaDeribitMarketDataConfig cfg);
static void _start_onix_glbx(bts::api::md::GlbxMarketDataConfig cfg);
void _start_replays(const bts::api::md::MarketDataConfig &cfg);
void start_glbx(const bts::api::md::MarketDataConfig &cfg);
void start_cboe(const bts::api::md::MarketDataConfig &cfg);
void start_taifex(const bts::api::md::MarketDataConfig &cfg);
};
} // namespace bts::md
@@ -1,60 +1,60 @@
#include "MarketDataSubscription.h"
#include "sparkd/md/binance/BinanceSubscription.h"
#include "sparkd/md/binance_us/BinanceUsSubscription.h"
#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
#include "sparkd/md/coinbase/CoinbaseSubscription.h"
#include "sparkd/md/deribit/DeribitSubscription.h"
#include "sparkd/md/ftx/FTXSubscription.h"
#include "sparkd/md/okx/OkxSubscription.h"
// #include "MarketDataSubscription.h"
// #include "sparkd/md/binance/BinanceSubscription.h"
// #include "sparkd/md/binance_us/BinanceUsSubscription.h"
// #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
// #include "sparkd/md/coinbase/CoinbaseSubscription.h"
// #include "sparkd/md/deribit/DeribitSubscription.h"
// #include "sparkd/md/ftx/FTXSubscription.h"
// #include "sparkd/md/okx/OkxSubscription.h"
#include <vector>
// #include <vector>
namespace bts::md {
// namespace bts::md {
using std::vector;
// using std::vector;
struct MarketDataSubscription::PImpl {
vector<ftx::FTXSubscription> ftx_subs;
vector<binance::BinanceSubscription> binance_subs;
vector<binance_us::BinanceUsSubscription> binance_us_subs;
vector<coinbase::CoinbaseSubscription> coinbase_subs;
vector<binance_usds_futures::BinanceUsdsFuturesSubscription>
binance_usds_futures_subs;
vector<deribit::DeribitSubscription> deribit_subs;
vector<okx::OkxSubscription> okx_subs;
// TODO: add more exchanges
};
// struct MarketDataSubscription::PImpl {
// vector<ftx::FTXSubscription> ftx_subs;
// vector<binance::BinanceSubscription> binance_subs;
// vector<binance_us::BinanceUsSubscription> binance_us_subs;
// vector<coinbase::CoinbaseSubscription> coinbase_subs;
// vector<binance_usds_futures::BinanceUsdsFuturesSubscription>
// binance_usds_futures_subs;
// vector<deribit::DeribitSubscription> deribit_subs;
// vector<okx::OkxSubscription> okx_subs;
// // TODO: add more exchanges
// };
MarketDataSubscription::MarketDataSubscription()
: me_(std::make_unique<PImpl>()) {}
// MarketDataSubscription::MarketDataSubscription()
// : me_(std::make_unique<PImpl>()) {}
MarketDataSubscription::~MarketDataSubscription() = default;
// MarketDataSubscription::~MarketDataSubscription() = default;
void MarketDataSubscription::add(binance::BinanceSubscription sub) {
me_->binance_subs.push_back(std::move(sub));
}
// void MarketDataSubscription::add(binance::BinanceSubscription sub) {
// me_->binance_subs.push_back(std::move(sub));
// }
void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) {
me_->binance_us_subs.push_back(std::move(sub));
}
void MarketDataSubscription::add(coinbase::CoinbaseSubscription sub) {
me_->coinbase_subs.push_back(std::move(sub));
}
void MarketDataSubscription::add(
binance_usds_futures::BinanceUsdsFuturesSubscription sub) {
me_->binance_usds_futures_subs.push_back(std::move(sub));
}
// void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) {
// me_->binance_us_subs.push_back(std::move(sub));
// }
// void MarketDataSubscription::add(coinbase::CoinbaseSubscription sub) {
// me_->coinbase_subs.push_back(std::move(sub));
// }
// void MarketDataSubscription::add(
// binance_usds_futures::BinanceUsdsFuturesSubscription sub) {
// me_->binance_usds_futures_subs.push_back(std::move(sub));
// }
void MarketDataSubscription::add(deribit::DeribitSubscription sub) {
me_->deribit_subs.push_back(std::move(sub));
}
// void MarketDataSubscription::add(deribit::DeribitSubscription sub) {
// me_->deribit_subs.push_back(std::move(sub));
// }
void MarketDataSubscription::add(ftx::FTXSubscription sub) {
me_->ftx_subs.push_back(std::move(sub));
}
// void MarketDataSubscription::add(ftx::FTXSubscription sub) {
// me_->ftx_subs.push_back(std::move(sub));
// }
void MarketDataSubscription::add(okx::OkxSubscription sub) {
me_->okx_subs.push_back(std::move(sub));
}
// void MarketDataSubscription::add(okx::OkxSubscription sub) {
// me_->okx_subs.push_back(std::move(sub));
// }
} // namespace bts::md
// } // namespace bts::md
@@ -1,59 +1,59 @@
#pragma once
// #pragma once
#include <memory>
// #include <memory>
namespace bts::md::binance {
class BinanceSubscription;
}
// namespace bts::md::binance {
// class BinanceSubscription;
// }
namespace bts::md::binance_us {
class BinanceUsSubscription;
}
namespace bts::md::coinbase {
class CoinbaseSubscription;
}
namespace bts::md::binance_usds_futures {
class BinanceUsdsFuturesSubscription;
}
// namespace bts::md::binance_us {
// class BinanceUsSubscription;
// }
// namespace bts::md::coinbase {
// class CoinbaseSubscription;
// }
// namespace bts::md::binance_usds_futures {
// class BinanceUsdsFuturesSubscription;
// }
namespace bts::md::deribit {
class DeribitSubscription;
}
// namespace bts::md::deribit {
// class DeribitSubscription;
// }
namespace bts::md::ftx {
class FTXSubscription;
}
// namespace bts::md::ftx {
// class FTXSubscription;
// }
namespace bts::md::okx {
class OkxSubscription;
}
// namespace bts::md::okx {
// class OkxSubscription;
// }
namespace bts::md {
// namespace bts::md {
class MarketDataSubscription {
public:
MarketDataSubscription();
// class MarketDataSubscription {
// public:
// MarketDataSubscription();
MarketDataSubscription(const MarketDataSubscription &) noexcept = delete;
MarketDataSubscription(MarketDataSubscription &&) noexcept = default;
MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept =
delete;
MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept =
default;
// MarketDataSubscription(const MarketDataSubscription &) noexcept = delete;
// MarketDataSubscription(MarketDataSubscription &&) noexcept = default;
// MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept =
// delete;
// MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept =
// default;
~MarketDataSubscription();
// ~MarketDataSubscription();
void add(binance::BinanceSubscription);
void add(binance_us::BinanceUsSubscription);
void add(coinbase::CoinbaseSubscription);
void add(binance_usds_futures::BinanceUsdsFuturesSubscription);
void add(deribit::DeribitSubscription);
void add(ftx::FTXSubscription);
void add(okx::OkxSubscription);
// void add(binance::BinanceSubscription);
// void add(binance_us::BinanceUsSubscription);
// void add(coinbase::CoinbaseSubscription);
// void add(binance_usds_futures::BinanceUsdsFuturesSubscription);
// void add(deribit::DeribitSubscription);
// void add(ftx::FTXSubscription);
// void add(okx::OkxSubscription);
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
// private:
// struct PImpl;
// std::unique_ptr<PImpl> me_;
// };
} // namespace bts::md
// } // namespace bts::md
@@ -0,0 +1,44 @@
#include "OnixsMarketDataProvider.h"
#include "spark/api/domain/Venue.pb.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/glbx/OnixsGlbxSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::domain::Venue;
using bts::api::md::MarketDataConfig;
using bts::md::OnixsMarketDataProvider;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct OnixsMarketDataProvider::PImpl {
MarketDataConfig config;
std::shared_ptr<IDGenerator> sec_id_store;
vector<shared_ptr<OnixsGlbxSubscription>> sources;
};
OnixsMarketDataProvider::OnixsMarketDataProvider(MarketDataConfig config)
: me_(std::make_unique<PImpl>()) {
me_->config = std::move(config);
}
OnixsMarketDataProvider::~OnixsMarketDataProvider() = default;
vector<pair<Venue, shared_ptr<IMarketDataSource>>>
OnixsMarketDataProvider::get_sources() {
vector<pair<Venue, shared_ptr<IMarketDataSource>>> rv;
for (const auto &glbx : me_->config.glbx()) {
me_->sources.push_back(
std::make_shared<OnixsGlbxSubscription>(me_->sec_id_store, glbx));
rv.push_back({Venue::VENUE_GLBX, me_->sources.back()});
}
return rv;
}
} // namespace bts::md
@@ -0,0 +1,26 @@
#pragma once
#include "spark/api/md/MarketDataConfig.pb.h"
#include "sparkd/md/IMarketDataProvider.h"
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::md {
class OnixsMarketDataProvider : public IMarketDataProvider {
public:
OnixsMarketDataProvider();
explicit OnixsMarketDataProvider(api::md::MarketDataConfig config);
~OnixsMarketDataProvider();
std::vector<
std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,63 @@
#include "PyMarketDataProvider.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_assert.h"
#include "sparkd/md/IMarketDataSource.h"
#include "sparkd/md/MarketDataSubscription.h"
#include "sparkd/md/PyMarketDataSubscription.h"
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::os::sparkd_wd;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
using Path = boost::filesystem::path;
struct PyMarketDataProvider::PImpl {
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
vector<shared_ptr<PyMarketDataSubscription>> py_md_subs;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
PyMarketDataProvider::PyMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
PyMarketDataProvider::~PyMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
PyMarketDataProvider::get_sources() {
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> rv;
const Path wd{sparkd_wd()};
vector<string> module_paths{
wd.string(),
(wd / "python_md_providers").string(),
};
for (const auto &py_plugin : me_->config.python_md_providers()) {
try {
auto source = std::make_shared<PyMarketDataSubscription>(
py_plugin, me_->sec_id_store.get());
me_->py_md_subs.push_back(source);
rv.push_back(
std::make_pair<api::domain::Venue, shared_ptr<IMarketDataSource>>(
py_plugin.venue(), source));
} catch (const std::exception &ex) {
bts_assert_msgf(
false, "PyMarketDataSubscription threw: {}",
std::string{ex.what()});
}
}
return rv;
}
} // namespace bts::md
@@ -0,0 +1,27 @@
#pragma once
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/IMarketDataProvider.h"
#include <memory>
namespace bts::md {
class PyMarketDataProvider : public IMarketDataProvider {
public:
explicit PyMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~PyMarketDataProvider();
std::vector<
std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -6,21 +6,19 @@
#include "python_md/py_security_list.h"
#include "python_md/py_trade.h"
#include "spark/api/domain/CallPut.pb.h"
#include "spark/api/domain/SecurityType.h"
#include "spark/api/domain/Venue.h"
#include "spark/api/domain/Venue.pb.h"
#include "spark/api/md/PyMarketDataProviderConfig.pb.h"
#include "spark/core/TimeStamp.h"
#include "spark/domain/AggressorSide.h"
#include "spark/domain/DomainSecurityUtils.h"
#include "spark/domain/FutureSecurity.h"
#include "spark/domain/Security.h"
#include "spark/domain/SecurityID.h"
#include "spark/domain/SpotSecurity.h"
#include "spark/domain/TradeRecord.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/MDProducer.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_assert.h"
@@ -29,7 +27,7 @@
#include <optional>
#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
#include <unordered_map>
#include <utility>
#include <vector>
namespace bts::md {
@@ -44,23 +42,12 @@ using bts::md::py_interop::PySecurityList;
using bts::md::py_interop::PyTrade;
using bts::md::py_interop::PyTradeList;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
using Path = boost::filesystem::path;
struct VenueSink {
VenueSink(api::Venue venue, GeneralSecurityIndex *index)
: producer(), listener(*index, producer, venue) {}
MDProducer producer;
GeneralMarketDataListener listener;
};
struct TradeTuple {
py_interop::PyTrade py_trade;
domain::TradeRecord trade;
@@ -74,41 +61,40 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
{this->config.module_path().begin(),
this->config.module_path().end()},
this),
sec_id_store_(sec_id_store),
index_(std::make_unique<GeneralSecurityIndex>()) {}
sec_id_store_(sec_id_store) {}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener_ = std::move(listener);
LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
py_md_exec_.start();
}
void start() { py_md_exec_.start(); }
void stop() { py_md_exec_.stop(); }
private:
PyMarketDataProviderConfig config;
PyMdExecutor py_md_exec_;
unordered_map<api::Venue, shared_ptr<VenueSink>> sinks_;
shared_ptr<IMarketDataListener> listener_;
IDGenerator *sec_id_store_;
unique_ptr<GeneralSecurityIndex> index_;
// The main store for securities.
vector<domain::Security> securities_;
// We also maintain Security pointers per venue (the GeneralMarketDataListener does not support
// writing arbitrary groups of securities, they must share a common venue when batched together).
unordered_map<api::domain::Venue, vector<const domain::Security *>>
securities_by_venue_;
vector<const domain::Security *> security_ptrs_;
domain::Book book_;
// Since the python md provider isn't tethered to a single venue, we
// must know the venue for each individual trade.
vector<TradeTuple> trades_;
unordered_map<
SecurityID,
pair<api::domain::Venue, vector<const domain::TradeRecord *>>>
trades_by_security_;
vector<domain::TradeRecord> trades_;
vector<const domain::TradeRecord *> trade_ptrs_;
void on_securities(const PySecurityList &py_securities) override {
LOG(md, Info, "Received securities from python plugin. count: {}",
py_securities.size());
// `securities_` and `securities_by_venue_` should already be empty, but no harm in clearing
// them again. We keep them as member variables to avoid (re)allocating memory in each callback.
// `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them
// again. We keep them as member variables to avoid (re)allocating memory in each callback.
securities_.clear();
securities_by_venue_.clear();
security_ptrs_.clear();
for (const auto &py_sec : py_securities.items()) {
LOG(md, Info, "PyMarketDataSubscription adding security {}",
py_sec.name());
@@ -119,27 +105,23 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
py_sec.name());
continue;
}
securities_.emplace_back(std::move(*sec));
securities_.push_back(std::move(*sec));
}
// Populate vectors of security ptrs per each venue.
for (auto &sec : securities_) {
securities_by_venue_[sec.venue()].push_back(&sec);
security_ptrs_.push_back(&sec);
}
// Pass securities to the sinks (per venue).
for (const auto &[venue, secs] : securities_by_venue_) {
get_listener_(venue).on_securities(secs);
}
listener_->on_securities(security_ptrs_);
securities_.clear();
securities_by_venue_.clear();
security_ptrs_.clear();
}
void on_books(const PyBookList &py_books) override {
for (auto &py_book : py_books.items()) {
api::domain::Venue venue;
bts_assert(api::domain::venue_parse_smart(py_book.venue(), &venue));
bts_assert(book_.count_populated() == 0);
to_book_(py_book, book_);
get_listener_(venue).on_book(book_);
listener_->on_book(book_);
book_.clear();
}
}
@@ -147,50 +129,33 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
void on_trades(const PyTradeList &py_trades) override {
LOG(md, Info, "Received trades from python plugin. count: {}",
py_trades.size());
// `trades_` and `trades_by_security_` should already be empty, but no harm in clearing them
// again. We keep them as member variables to avoid (re)allocating memory in each callback.
// `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again.
// We keep them as member variables to avoid (re)allocating memory in each callback.
trades_.clear();
trades_by_security_.clear();
trade_ptrs_.clear();
for (const auto &py_trade : py_trades.items()) {
LOG(md, Info, "PyMarketDataSubscription received trade {}",
py_trade.to_string());
auto &trade = trades_.emplace_back();
bts_assert(
api::domain::venue_parse_smart(py_trade.venue(), &trade.venue));
to_bts_trade_(py_trade, trade.trade);
to_bts_trade_(py_trade, trades_.emplace_back());
}
// Populate vectors of trade ptrs per each security.
for (const auto &trade : trades_) {
auto &sec_trades = trades_by_security_[trade.trade.sec_id];
sec_trades.first = trade.venue;
sec_trades.second.push_back(&trade.trade);
}
// Pass trades to the sinks (per venue)
for (const auto &[sec_id, p] : trades_by_security_) {
get_listener_(p.first).on_trades(p.second);
for (const auto &t : trades_) {
trade_ptrs_.push_back(&t);
}
listener_->on_trades(trade_ptrs_);
trades_.clear();
trades_by_security_.clear();
trade_ptrs_.clear();
}
GeneralMarketDataListener &get_listener_(api::domain::Venue venue) {
const auto api_venue = domain::DomainSecurityUtils::convert(venue);
return sinks_
.try_emplace(
api_venue, std::make_shared<VenueSink>(api_venue, index_.get()))
.first->second->listener;
}
SecurityID get_sec_id_(const string &venue_name, const string &sec_name) {
api::domain::Venue venue;
// Although an overload of 'get_id' accepts a string, we parse `venue_name` to the enum to
// normalize different spellings of the same venue. For example, "deribt", "DERIBIT", and
// "VENUE_deribit" all represent the VENUE_DERIBIT enum (see `venue_parse_smart` for details).
bts_assert(api::domain::venue_parse_smart(venue_name, &venue));
auto sec_id = sec_id_store_->get_id(venue, sec_name, true);
SecurityID get_sec_id_(
api::domain::Venue venue,
api::domain::SecurityType sec_type,
const string &sec_name) {
auto sec_id = sec_id_store_->get_id(venue, sec_type, sec_name, true);
bts_assert_msgf(
sec_id, "Failed to get security id for py_book. venue: {}; name: {}",
venue, sec_name);
sec_id,
"Failed to get security id for py_book. venue: {}, type: {}, name: "
"{}",
venue, sec_type, sec_name);
return SecurityID{*sec_id};
}
@@ -205,7 +170,6 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
LOG(md, Error, "Failed to parse venue from '{}'", py_sec.venue());
return std::nullopt;
}
sec_comm.id = get_sec_id_(py_sec.venue(), py_sec.exchange_id());
sec_comm.exchange_sec_id = py_sec.exchange_id();
sec_comm.currency = py_sec.currency();
sec_comm.asset = py_sec.asset();
@@ -236,11 +200,18 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
"option)");
return std::nullopt;
}
sec_comm.id =
get_sec_id_(sec_comm.venue, sec.type(), sec_comm.exchange_sec_id);
return sec;
}
void to_bts_trade_(const PyTrade &source, domain::TradeRecord &dest) {
dest.sec_id = get_sec_id_(source.venue(), source.exchange_security_id());
api::domain::Venue venue;
bts_assert(api::domain::venue_parse_smart(source.venue(), &venue));
api::domain::SecurityType sec_type;
bts_assert(api::domain::security_type_parse_smart(
source.security_type(), &sec_type));
dest.sec_id = get_sec_id_(venue, sec_type, source.exchange_security_id());
dest.time_stamp = core::TimeStamp::from_nanos(source.timestamp());
dest.agg_side = source.aggressor_side().starts_with('B') ||
source.aggressor_side().starts_with('b')
@@ -266,20 +237,27 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
void to_book_(const PyBook &source, domain::Book &dest) {
to_book_side_(source.bids(), dest.bids());
to_book_side_(source.asks(), dest.asks());
api::domain::Venue venue;
bts_assert(api::domain::venue_parse_smart(source.venue(), &venue));
api::domain::SecurityType type;
bts_assert(api::domain::security_type_parse_smart(
source.security_type(), &type));
dest.set_security_id(
get_sec_id_(source.venue(), source.exchange_security_id()));
get_sec_id_(venue, type, source.exchange_security_id()));
}
};
PyMarketDataSubscription::PyMarketDataSubscription(
PyMarketDataProviderConfig config, IDGenerator *sec_id_store)
: me_(std::make_unique<PyMarketDataSubscription::PImpl>(
std::move(config), sec_id_store)) {
LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
me_->start();
}
std::move(config), sec_id_store)) {}
PyMarketDataSubscription::~PyMarketDataSubscription() { me_->stop(); }
void PyMarketDataSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
void PyMarketDataSubscription::stop() { me_->stop(); }
} // namespace bts::md
@@ -1,5 +1,7 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::api::md {
@@ -12,13 +14,16 @@ class IDGenerator;
namespace bts::md {
class PyMarketDataSubscription {
class PyMarketDataSubscription : public IMarketDataSource {
public:
explicit PyMarketDataSubscription(
api::md::PyMarketDataProviderConfig config,
persistence::IDGenerator *sec_id_store);
~PyMarketDataSubscription();
void start(std::shared_ptr<IMarketDataListener> listener) override;
void stop() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
@@ -0,0 +1,52 @@
#include "ReplayMarketDataProvider.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/api/md/MarketDataConfigUtils.h"
#include "spark/config/config.h"
#include "spark/domain/DomainSecurityUtils.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/IMarketDataSource.h"
#include "sparkd/md/recording/MDReplayer.h"
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::domain::DomainSecurityUtils;
using bts::md::recording::MDReplayer;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct ReplayMarketDataProvider::PImpl {
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
ReplayMarketDataProvider::ReplayMarketDataProvider()
: ReplayMarketDataProvider(
config::get<MarketDataConfig>(), IDGenerator::instance_ptr()) {}
ReplayMarketDataProvider::ReplayMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
ReplayMarketDataProvider::~ReplayMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
ReplayMarketDataProvider::get_sources() {
const auto replay_cfgs =
MarketDataConfigUtils::get_effective_replay_config(me_->config);
for (const auto &r : replay_cfgs) {
const auto v = DomainSecurityUtils::convert(r.venue());
me_->sources.push_back({v, std::make_shared<MDReplayer>(r.venue())});
}
return me_->sources;
}
} // namespace bts::md
@@ -0,0 +1,28 @@
#pragma once
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/IMarketDataProvider.h"
#include <memory>
namespace bts::md {
class ReplayMarketDataProvider : public IMarketDataProvider {
public:
ReplayMarketDataProvider();
explicit ReplayMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~ReplayMarketDataProvider();
std::vector<
std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,47 @@
#include "ReplayMarketDataSubscription.h"
#include "spark/api/md/MarketDataReplayConfig.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/MDProducer.h"
#include "spark/utils/thread_manager.h"
#include "sparkd/md/recording/MDReplayer.h"
#include <ccapi_cpp/ccapi_event_dispatcher.h>
#include <optional>
namespace bts::md {
using bts::api::md::MarketDataReplayConfig;
using bts::md::recording::MDReplayer;
using std::string;
struct ReplayMarketDataSource::PImpl {
MarketDataReplayConfig config;
PImpl(const MarketDataReplayConfig &config) : config(config) {}
};
ReplayMarketDataSource::ReplayMarketDataSource(MarketDataReplayConfig config)
: me_(std::make_unique<PImpl>(config)) {
me_->config = std::move(config);
}
ReplayMarketDataSource::~ReplayMarketDataSource() = default;
void ReplayMarketDataSource::start(
std::shared_ptr<IMarketDataListener> listener) {
LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
auto md_replayer = std::make_shared<MDReplayer>(me_->config.venue());
std::string tid_name = "MDReplayer_" + Venue_Name(me_->config.venue());
bts::threading::make_thread(tid_name, [&, listener, md_replayer]() {
md_replayer->start(listener);
}).detach();
LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
void ReplayMarketDataSource::stop() {}
} // namespace bts::md
@@ -0,0 +1,25 @@
#pragma once
#include "spark/api/md/MarketDataReplayConfig.pb.h"
#include "spark/md/IMarketDataListener.h"
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::md {
class ReplayMarketDataSource : public IMarketDataSource {
public:
ReplayMarketDataSource();
explicit ReplayMarketDataSource(api::md::MarketDataReplayConfig config);
~ReplayMarketDataSource();
void start(std::shared_ptr<IMarketDataListener> listener) override;
void stop() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -171,7 +171,7 @@ struct BinanceSubscription::PImpl {
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
std::shared_ptr<IMarketDataListener> listener;
BinanceSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
@@ -194,11 +194,10 @@ struct BinanceSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
BinanceMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
session(dispatcher, mk_session_cfg(this->cfg), {}),
sec_requestor(
&session,
this->criteria,
@@ -209,6 +208,11 @@ struct BinanceSubscription::PImpl {
LOG(md, Info, "Initializing BinanceSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(std::shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
@@ -216,7 +220,7 @@ struct BinanceSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert(is_new);
return it->second;
}
@@ -225,7 +229,7 @@ struct BinanceSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -242,7 +246,7 @@ struct BinanceSubscription::PImpl {
}
auto [it, is_new] = stats_handlers.try_emplace(
hnd_id,
std::make_unique<StatsHandler>(sec_desc_id_tuples, listener));
std::make_unique<StatsHandler>(sec_desc_id_tuples, listener.get()));
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", hnd_id);
return *it->second;
@@ -333,10 +337,8 @@ struct BinanceSubscription::PImpl {
BinanceSubscription::BinanceSubscription(
ccapi::EventDispatcher *dispatcher,
BinanceMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
BinanceSubscription::BinanceSubscription(BinanceSubscription &&) noexcept =
default;
@@ -345,4 +347,8 @@ BinanceSubscription &BinanceSubscription::operator=(
BinanceSubscription::~BinanceSubscription() = default;
void BinanceSubscription::start(std::shared_ptr<IMarketDataListener> listener) {
me_->start(listener);
}
} // namespace bts::md::binance
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,13 +26,12 @@ namespace bts::md::binance {
*
* @a https://binance-docs.github.io/apidocs/spot/en/#exchange-information
*/
class BinanceSubscription {
class BinanceSubscription : public IMarketDataSource {
public:
BinanceSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::BinanceMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
BinanceSubscription(const BinanceSubscription &) noexcept = delete;
BinanceSubscription(BinanceSubscription &&) noexcept;
@@ -39,10 +39,9 @@ class BinanceSubscription {
delete;
BinanceSubscription &operator=(BinanceSubscription &&) noexcept;
~BinanceSubscription();
~BinanceSubscription() override;
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -47,6 +47,7 @@ using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::tuple;
using std::uint32_t;
@@ -174,7 +175,7 @@ struct BinanceUsSubscription::PImpl {
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
BinanceUsSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
@@ -191,17 +192,16 @@ struct BinanceUsSubscription::PImpl {
// different map, 'stats_handlers' (unique ids are generated w/ a counter,
// 'stats_handlers_id_counter').
unordered_map<string, StatsHandler *> sec_stats_handler_map;
unordered_map<uint32_t, std::unique_ptr<StatsHandler>> stats_handlers;
unordered_map<uint32_t, unique_ptr<StatsHandler>> stats_handlers;
uint32_t stats_handlers_id_counter = 0;
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
BinanceUsMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
session(dispatcher, mk_session_cfg(this->cfg), {}),
sec_requestor(
&session,
this->criteria,
@@ -212,6 +212,11 @@ struct BinanceUsSubscription::PImpl {
LOG(md, Info, "Initializing BinanceUsSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(std::shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
@@ -219,7 +224,7 @@ struct BinanceUsSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert(is_new);
return it->second;
}
@@ -228,7 +233,7 @@ struct BinanceUsSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -245,7 +250,7 @@ struct BinanceUsSubscription::PImpl {
}
auto [it, is_new] = stats_handlers.try_emplace(
hnd_id,
std::make_unique<StatsHandler>(sec_desc_id_tuples, listener));
std::make_unique<StatsHandler>(sec_desc_id_tuples, listener.get()));
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", hnd_id);
return *it->second;
@@ -335,10 +340,8 @@ struct BinanceUsSubscription::PImpl {
BinanceUsSubscription::BinanceUsSubscription(
ccapi::EventDispatcher *dispatcher,
BinanceUsMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
BinanceUsSubscription::BinanceUsSubscription(
BinanceUsSubscription &&) noexcept = default;
@@ -347,4 +350,8 @@ BinanceUsSubscription &BinanceUsSubscription::operator=(
BinanceUsSubscription::~BinanceUsSubscription() = default;
void BinanceUsSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::binance_us
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,13 +26,12 @@ namespace bts::md::binance_us {
*
* @a https://docs.binance.us/#get-exchange-information
*/
class BinanceUsSubscription {
class BinanceUsSubscription : public IMarketDataSource {
public:
BinanceUsSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::BinanceUsMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
BinanceUsSubscription(const BinanceUsSubscription &) noexcept = delete;
BinanceUsSubscription(BinanceUsSubscription &&) noexcept;
@@ -41,8 +41,7 @@ class BinanceUsSubscription {
~BinanceUsSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -41,6 +41,7 @@ using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::uint32_t;
using std::unique_ptr;
@@ -167,7 +168,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
SecurityCriteria criteria;
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
std::shared_ptr<IMarketDataListener> listener;
BinanceUsdsFuturesSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
@@ -183,15 +184,19 @@ struct BinanceUsdsFuturesSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
BinanceUsdsFuturesMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
session(dispatcher, mk_session_cfg(this->cfg), {}),
sec_requestor(&session, this->cfg, sec_id_store) {
LOG(md, Info, "Initializing BinanceUsdsFuturesSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
@@ -218,7 +223,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const auto sec_id = get_sec_id(sec_type, sec_desc);
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Depth handler already created for '{}'", sec_desc);
return it->second;
@@ -228,7 +233,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const auto sec_id = get_sec_id(sec_type, sec_desc);
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -238,7 +243,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const auto sec_id = get_sec_id(sec_type, sec_desc);
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
stats_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Stats handler already created for '{}'", sec_desc);
return it->second;
@@ -310,10 +315,8 @@ struct BinanceUsdsFuturesSubscription::PImpl {
BinanceUsdsFuturesSubscription::BinanceUsdsFuturesSubscription(
ccapi::EventDispatcher *dispatcher,
BinanceUsdsFuturesMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
BinanceUsdsFuturesSubscription::BinanceUsdsFuturesSubscription(
BinanceUsdsFuturesSubscription &&) noexcept = default;
@@ -322,4 +325,9 @@ BinanceUsdsFuturesSubscription &BinanceUsdsFuturesSubscription::operator=(
BinanceUsdsFuturesSubscription::~BinanceUsdsFuturesSubscription() = default;
void BinanceUsdsFuturesSubscription::start(
shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::binance_usds_futures
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,13 +26,12 @@ namespace bts::md::binance_usds_futures {
*
* @a https://binance-docs.github.io/apidocs/futures/en/#exchange-information
*/
class BinanceUsdsFuturesSubscription {
class BinanceUsdsFuturesSubscription : public IMarketDataSource {
public:
BinanceUsdsFuturesSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::BinanceUsdsFuturesMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
BinanceUsdsFuturesSubscription(
const BinanceUsdsFuturesSubscription &) noexcept = delete;
@@ -43,8 +43,7 @@ class BinanceUsdsFuturesSubscription {
~BinanceUsdsFuturesSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -0,0 +1,140 @@
#include "CCApiMarketDataProvider.h"
#include "spark/api/domain/Venue.pb.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/config/config.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/MDProducer.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/MarketDataSubscription.h"
#include "sparkd/md/binance/BinanceSubscription.h"
#include "sparkd/md/binance_us/BinanceUsSubscription.h"
#include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
#include "sparkd/md/coinbase/CoinbaseSubscription.h"
#include "sparkd/md/deribit/DeribitSubscription.h"
#include "sparkd/md/ftx/FTXSubscription.h"
#include "sparkd/md/okx/OkxSubscription.h"
#include <ccapi_cpp/ccapi_event_dispatcher.h>
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
static bool validate(
const MarketDataConfig & /*cfg*/, vector<string> & /*errs*/) {
return true;
}
struct CCApiMarketDataProvider::PImpl {
struct VenueSink {
GeneralSecurityIndex sec_index;
MDProducer md_producer;
GeneralMarketDataListener listener;
VenueSink(api::Venue venue, shared_ptr<IDGenerator> sec_id_store)
: sec_index(),
listener(sec_index, md_producer, venue, sec_id_store.get()) {}
};
MarketDataConfig config;
ccapi::EventDispatcher dispatcher;
shared_ptr<IDGenerator> sec_id_store;
unordered_map<api::Venue, unique_ptr<VenueSink>> venue_sinks;
vector<std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
VenueSink *get_venue_sink(api::Venue venue) {
auto it = venue_sinks.find(venue);
if (it != venue_sinks.end()) {
return it->second.get();
}
return venue_sinks
.try_emplace(venue, std::make_unique<VenueSink>(venue, sec_id_store))
.first->second.get();
}
};
CCApiMarketDataProvider::CCApiMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
CCApiMarketDataProvider::~CCApiMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
CCApiMarketDataProvider::get_sources() {
auto &cfg = me_->config;
vector<string> cfg_errs;
if (!validate(cfg, cfg_errs)) {
LOG(md, Error,
"CCApiMarketDataProvider::get_sources failed. Invalid config: {}",
fmt::to_string(fmt::join(cfg_errs, "\n")));
}
// Binance
for (const auto &binance : cfg.binance()) {
me_->sources.emplace_back(
api::domain::VENUE_BINANCE,
std::make_shared<binance::BinanceSubscription>(
&me_->dispatcher, binance, me_->sec_id_store.get()));
}
// Binance US
for (const auto &binance_us : cfg.binance_us()) {
me_->sources.emplace_back(
api::domain::VENUE_BINANCE_US,
std::make_shared<binance_us::BinanceUsSubscription>(
&me_->dispatcher, binance_us, me_->sec_id_store.get()));
}
// Binance USDS Futures
for (const auto &binance_ufs : cfg.binance_usds_futures()) {
me_->sources.emplace_back(
api::domain::VENUE_BINANCE_USDS_FUTURES,
std::make_shared<
binance_usds_futures::BinanceUsdsFuturesSubscription>(
&me_->dispatcher, binance_ufs, me_->sec_id_store.get()));
}
// Coinbase
for (const auto &coinbase : cfg.coinbase()) {
me_->sources.emplace_back(
api::domain::VENUE_COINBASE,
std::make_shared<coinbase::CoinbaseSubscription>(
&me_->dispatcher, coinbase, me_->sec_id_store.get()));
}
// Deribit
for (const auto &deribit : cfg.deribit()) {
me_->sources.emplace_back(
api::domain::VENUE_DERIBIT,
std::make_shared<deribit::DeribitSubscription>(
&me_->dispatcher, deribit, me_->sec_id_store.get()));
}
// FTX
for (const auto &ftx : cfg.ftx()) {
me_->sources.emplace_back(
api::domain::VENUE_FTX,
std::make_shared<ftx::FTXSubscription>(
&me_->dispatcher, ftx, me_->sec_id_store.get()));
}
// Okx
for (const auto &okx : cfg.okx()) {
me_->sources.emplace_back(
api::domain::VENUE_OKX,
std::make_shared<okx::OkxSubscription>(
&me_->dispatcher, okx, me_->sec_id_store.get()));
}
return me_->sources;
}
} // namespace bts::md
@@ -0,0 +1,27 @@
#pragma once
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/IMarketDataProvider.h"
#include <memory>
namespace bts::md {
class CCApiMarketDataProvider : public IMarketDataProvider {
public:
explicit CCApiMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~CCApiMarketDataProvider();
std::vector<
std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -5,8 +5,10 @@
#include "spark/domain/SecurityCriteria.h"
#include "spark/io/ProtoPrinter.h"
#include "spark/io/QueryParams.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/common.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_assert.h"
#include "sparkd/md/ccapi/CCApiMarketDepthAdaptor.h"
#include "sparkd/md/ccapi/CCApiSession.h"
#include "sparkd/md/ccapi/CCApiTradeAdaptor.h"
@@ -36,6 +38,7 @@ using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
@@ -151,7 +154,7 @@ struct CoinbaseSubscription::PImpl {
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
CoinbaseSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
@@ -172,11 +175,10 @@ struct CoinbaseSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
CoinbaseMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(dispatcher, mk_session_cfg(this->cfg), {}), listener(listener),
session(dispatcher, mk_session_cfg(this->cfg), {}),
sec_requestor(
&session,
this->criteria,
@@ -187,6 +189,11 @@ struct CoinbaseSubscription::PImpl {
LOG(md, Info, "Initializing CoinbaseSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
@@ -195,7 +202,7 @@ struct CoinbaseSubscription::PImpl {
auto sec_id =
SecurityID{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
stats_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", sec_id);
return it->second;
@@ -204,7 +211,7 @@ struct CoinbaseSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert(is_new);
return it->second;
}
@@ -213,7 +220,7 @@ struct CoinbaseSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -285,10 +292,8 @@ struct CoinbaseSubscription::PImpl {
CoinbaseSubscription::CoinbaseSubscription(
ccapi::EventDispatcher *dispatcher,
CoinbaseMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
CoinbaseSubscription::CoinbaseSubscription(CoinbaseSubscription &&) noexcept =
default;
@@ -297,4 +302,8 @@ CoinbaseSubscription &CoinbaseSubscription::operator=(
CoinbaseSubscription::~CoinbaseSubscription() = default;
void CoinbaseSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::coinbase
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -20,13 +21,12 @@ class IDGenerator;
namespace bts::md::coinbase {
class CoinbaseSubscription {
class CoinbaseSubscription : public IMarketDataSource {
public:
CoinbaseSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::CoinbaseMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
CoinbaseSubscription(const CoinbaseSubscription &) noexcept = delete;
CoinbaseSubscription(CoinbaseSubscription &&) noexcept;
@@ -36,8 +36,7 @@ class CoinbaseSubscription {
~CoinbaseSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -0,0 +1,51 @@
#include "DatabentoMarketDataProvider.h"
#include "spark/api/domain/Venue.pb.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/IMarketDataSource.h"
#include "sparkd/md/databento/DatabentoDataHandler.h"
#include "sparkd/md/glbx/DatabentoGlbxSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct DatabentoMarketDataProvider::PImpl {
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
vector<shared_ptr<IMarketDataSource>> glbx_sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
DatabentoMarketDataProvider::DatabentoMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
me_->config = std::move(config);
}
DatabentoMarketDataProvider::~DatabentoMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
DatabentoMarketDataProvider::get_sources() {
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> rv;
for (const auto &glbx : me_->config.glbx()) {
if (glbx.use_databento()) {
me_->glbx_sources.push_back(
std::make_shared<DatabentoGlbxSubscription>(glbx, me_->sec_id_store));
rv.push_back({api::domain::VENUE_GLBX, me_->glbx_sources.back()});
}
}
return rv;
}
} // namespace bts::md
@@ -0,0 +1,33 @@
#pragma once
#include "sparkd/md/IMarketDataProvider.h"
#include <memory>
namespace bts::persistence {
class IDGenerator;
}
namespace bts::api::md {
class MarketDataConfig;
}
namespace bts::md {
class DatabentoMarketDataProvider : public IMarketDataProvider {
public:
explicit DatabentoMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~DatabentoMarketDataProvider();
std::vector<
std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -52,6 +52,7 @@ using bts::io::websocket::JsonRpcId;
using bts::io::websocket::JsonRpcRequest;
using bts::persistence::IDGenerator;
using nlohmann::json;
using std::shared_ptr;
using std::string;
using std::uint32_t;
using std::unique_ptr;
@@ -134,14 +135,13 @@ struct DeribitSubscription::PImpl {
const bool subscribe_trades = true;
size_t sub_id_counter = 0;
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
DeribitChannelDemuxer channel_demuxer;
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
DeribitMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
excluded_secs(
@@ -159,11 +159,15 @@ struct DeribitSubscription::PImpl {
criteria,
&instrument_store,
{this->cfg.currencies().begin(), this->cfg.currencies().end()}),
session(dispatcher, mk_session_cfg(this->cfg), {}),
listener(listener) {
session(dispatcher, mk_session_cfg(this->cfg), {}) {
// DON'T reference the ctr arg 'cfg' (it has been moved to 'this->cfg')
_INFO("ctr. config:\n{}", ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(std::shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request_all();
subscribe_instrument_state();
}
@@ -172,7 +176,7 @@ struct DeribitSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] = security_data_map.try_emplace(
sec_desc, std::make_unique<SecurityData>(sid, listener));
sec_desc, std::make_unique<SecurityData>(sid, listener.get()));
_ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc);
_INFO("add_security_data. sec: '{}'", sec_desc);
return *it->second;
@@ -303,10 +307,8 @@ struct DeribitSubscription::PImpl {
DeribitSubscription::DeribitSubscription(
ccapi::EventDispatcher *dispatcher,
DeribitMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
DeribitSubscription::DeribitSubscription(DeribitSubscription &&) noexcept =
default;
@@ -315,4 +317,8 @@ DeribitSubscription &DeribitSubscription::operator=(
DeribitSubscription::~DeribitSubscription() = default;
void DeribitSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::deribit
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -24,13 +25,12 @@ namespace bts::md::deribit {
* @brief Subscribes to market data at Deribit
* @a https://docs.deribit.com/?#market-data
*/
class DeribitSubscription {
class DeribitSubscription : public IMarketDataSource {
public:
DeribitSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::DeribitMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
DeribitSubscription(const DeribitSubscription &) noexcept = delete;
DeribitSubscription(DeribitSubscription &&) noexcept;
@@ -40,9 +40,8 @@ class DeribitSubscription {
~DeribitSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
@@ -0,0 +1,77 @@
#include "EmumbaDeribitSubscription.h"
#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h"
#include "spark/config/protos/system_config.pb.h"
#include "spark/utils/thread_manager.h"
#include "sparkd/md/emumba/EmumbaDeribitData.h"
namespace bts::md {
using bts::config::SystemConfig;
using Path = boost::filesystem::path;
struct EmumbaDeribitSubscription::PImpl {
api::md::EmumbaDeribitMarketDataConfig config;
};
EmumbaDeribitSubscription::EmumbaDeribitSubscription(
api::md::EmumbaDeribitMarketDataConfig config)
: me_(std::make_unique<PImpl>()) {
me_->config = std::move(config);
}
EmumbaDeribitSubscription::~EmumbaDeribitSubscription() = default;
void EmumbaDeribitSubscription::start(
std::shared_ptr<IMarketDataListener> /*listener*/) {
auto &cfg = me_->config;
LOG(md, Info, "EmumbaMarketDataProvider::start t_id:{} t_name:{}. cfg:\n{}",
os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
auto start_deribit_thr = [=]() {
const auto config_path = Path{config::get<SystemConfig>().config_path()} /
cfg.json_conf_file_name();
const auto conf =
bts::cmds::DeribitDemoConfiguration::read_file(config_path.string());
bts_assert_msg(conf.has_value(), "Failed to parse deribit config");
// log
for (const auto &s : conf.value().symbol_config_list) {
LOG(md, Info, "sub.symbol: {}", s.symbol_name);
LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future,
s.interested_in_option);
for (const auto &i : s.interested_instruments) {
LOG(md, Info, "sub.instrument: {}", i);
}
}
EmumbaDeribitData ddata_obj;
EmumbaDeribitData *ddata = &ddata_obj;
// end log
bts_assert(conf.has_value());
LOG(md, Info, "Deribit conf: {}", conf.value());
ddata->real_handler =
std::make_shared<emumba::deribit_connector::deribit_handler>(
conf.value().symbol_config_list, conf.value().trading_config,
conf.value().credential_map, conf.value().logging_configuration,
conf.value().deribit_url, &ddata->listener);
// since the trading interface is housed inside the EmMDListener, we need to
// create and send emumba's trading_session_interface
for (const auto &p : conf.value().credential_map) {
const auto &sess = p.second;
auto ts_ptr = std::make_shared<
emumba::deribit_connector::trading_session_interface>(
ddata->real_handler->create_new_trading_session(sess));
ddata->trading_session_vec.push_back({sess, ts_ptr});
}
bts_assert(!ddata->trading_session_vec.empty());
ddata->listener.set_handler(
ddata->real_handler.get(), ddata->trading_session_vec);
ddata->real_handler->run();
};
threading::make_thread("Emumba", [&]() { start_deribit_thr(); }).detach();
LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
void EmumbaDeribitSubscription::stop() {
//
}
} // namespace bts::md
@@ -0,0 +1,24 @@
#pragma once
#include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h"
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::md {
class EmumbaDeribitSubscription : public IMarketDataSource {
public:
explicit EmumbaDeribitSubscription(
api::md::EmumbaDeribitMarketDataConfig config);
~EmumbaDeribitSubscription() override;
void start(std::shared_ptr<IMarketDataListener> listener) override;
void stop() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,40 @@
#include "EmumbaMarketDataProvider.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/api/md/MarketDataConfigUtils.h"
#include "sparkd/md/IMarketDataSource.h"
#include "sparkd/md/emumba/EmumbaDeribitSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::md::MarketDataConfig;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct EmumbaMarketDataProvider::PImpl {
MarketDataConfig config;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> deribit_subs;
PImpl(MarketDataConfig config) : config(std::move(config)) {}
};
EmumbaMarketDataProvider::EmumbaMarketDataProvider(MarketDataConfig config)
: me_(std::make_unique<PImpl>(std::move(config))) {}
EmumbaMarketDataProvider::~EmumbaMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
EmumbaMarketDataProvider::get_sources() {
for (const auto &cfg : me_->config.emumba_deribit()) {
me_->deribit_subs.push_back(
{api::domain::VENUE_DERIBIT,
std::make_shared<EmumbaDeribitSubscription>(cfg)});
}
return me_->deribit_subs;
}
} // namespace bts::md
@@ -0,0 +1,25 @@
#pragma once
#include "spark/api/md/MarketDataConfig.pb.h"
#include "sparkd/md/IMarketDataProvider.h"
#include <memory>
namespace bts::md {
class EmumbaMarketDataProvider : public IMarketDataProvider {
public:
EmumbaMarketDataProvider();
explicit EmumbaMarketDataProvider(api::md::MarketDataConfig config);
~EmumbaMarketDataProvider();
std::vector<
std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -38,6 +38,7 @@ using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
@@ -105,7 +106,7 @@ struct FTXSubscription::PImpl {
FTXSecurityRequestor sec_requestor;
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
// These aliases are useful for quickly switching the handler types (e.g.,
// a special handler that logs messages for testing/diagnostic purposes)
@@ -120,16 +121,20 @@ struct FTXSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
FTXMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener)
persistence::IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(domain::SecurityCriteria::from_proto(
this->cfg.security_criteria())),
sec_requestor(dispatcher, criteria, sec_id_store), session(dispatcher),
listener(listener) {
sec_requestor(dispatcher, criteria, sec_id_store),
session(dispatcher) {
LOG(md, Info, "Initializing FTXSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
@@ -137,7 +142,7 @@ struct FTXSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert(is_new);
return it->second;
}
@@ -146,7 +151,7 @@ struct FTXSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -156,7 +161,7 @@ struct FTXSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
stats_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Statistics handler already created for '{}'", sec_desc);
return it->second;
@@ -228,10 +233,8 @@ struct FTXSubscription::PImpl {
FTXSubscription::FTXSubscription(
ccapi::EventDispatcher *dispatcher,
FTXMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
FTXSubscription::FTXSubscription(FTXSubscription &&) noexcept = default;
FTXSubscription &FTXSubscription::operator=(FTXSubscription &&) noexcept =
@@ -239,4 +242,8 @@ FTXSubscription &FTXSubscription::operator=(FTXSubscription &&) noexcept =
FTXSubscription::~FTXSubscription() = default;
void FTXSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::ftx
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,20 +26,21 @@ namespace bts::md::ftx {
*
* @details Uses ccapi infrastructure to request/subscribe data
*/
class FTXSubscription {
class FTXSubscription : public IMarketDataSource {
public:
FTXSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::FTXMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
FTXSubscription(const FTXSubscription &) noexcept = delete;
FTXSubscription(FTXSubscription &&) noexcept;
FTXSubscription &operator=(const FTXSubscription &) noexcept = delete;
FTXSubscription &operator=(FTXSubscription &&) noexcept;
~FTXSubscription();
~FTXSubscription() override;
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -0,0 +1,41 @@
#include "DatabentoGlbxSubscription.h"
#include "spark/api/md/GlbxMarketDataConfig.pb.h"
#include "spark/utils/thread_manager.h"
#include "sparkd/md/databento/DatabentoDataHandler.h"
#include "sparkd/md/glbx/GlbxMarketDataHandler.h"
namespace bts::md {
using bts::api::md::GlbxMarketDataConfig;
using bts::persistence::IDGenerator;
using std::shared_ptr;
struct DatabentoGlbxSubscription::PImpl {
GlbxMarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
PImpl(GlbxMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
DatabentoGlbxSubscription::DatabentoGlbxSubscription(
GlbxMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
DatabentoGlbxSubscription::~DatabentoGlbxSubscription() = default;
void DatabentoGlbxSubscription::start(
std::shared_ptr<IMarketDataListener> listener) {
bts::threading::make_thread(
"glbx",
[config = me_->config, sec_id_store = me_->sec_id_store,
listener = std::move(listener)]() {
api::md::glbx::GlbxMarketDatahandler::run(
std::move(listener), std::move(sec_id_store), config);
})
.detach();
// TODO: implement 'stop' (signal the thread above to terminate, then join it).
}
} // namespace bts::md
@@ -0,0 +1,47 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::api::md {
class GlbxMarketDataConfig;
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
/**
* @brief Subscribes to market data at Glbx via Databento
*
* @a https://www.Databento.biz/
*/
class DatabentoGlbxSubscription : public IMarketDataSource {
public:
DatabentoGlbxSubscription(
api::md::GlbxMarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
DatabentoGlbxSubscription(const DatabentoGlbxSubscription &) noexcept =
delete;
DatabentoGlbxSubscription(DatabentoGlbxSubscription &&) noexcept;
DatabentoGlbxSubscription &operator=(
const DatabentoGlbxSubscription &) noexcept = delete;
DatabentoGlbxSubscription &operator=(DatabentoGlbxSubscription &&) noexcept;
~DatabentoGlbxSubscription() override;
/**
* Starts market data subscription on a background thread.
*
* @note Does not block.
*/
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,118 @@
#include "OnixsGlbxSubscription.h"
#include "spark/api/md/GlbxMarketDataConfig.pb.h"
#include "spark/core/LOCK_GUARD.h"
#include "spark/logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/SymbolLoader.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/Dispatcher.h"
#include "sparkd/md/platforms/IMDPlatform.h"
#include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
#include <atomic>
#include <memory>
#include <thread>
#include <vector>
namespace bts::md {
using bts::api::md::GlbxMarketDataConfig;
using bts::domain::SecurityCriteria;
using bts::md::IMDPlatform;
using bts::persistence::IDGenerator;
using std::shared_ptr;
using std::string;
using std::vector;
struct OnixsGlbxSubscription::PImpl {
GlbxMarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
PImpl(shared_ptr<IDGenerator> sec_id_store, GlbxMarketDataConfig config)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
std::thread dispatcher_thread;
utils::Dispatcher *dispatcher{nullptr};
std::optional<utils::DispatcherTimer> heartbeat_timer;
bool is_running = false;
std::mutex mx;
~PImpl() { stop(); }
/**
* Starts market data subscription on this thread.
*
* @note Designed to be called from a thread callback inside `OnixsGlbxSubscription::start`.
* @note Blocks until subscription stops.
*/
void run() {
{
BTS_LOCK_GUARD(mx);
if (!is_running) {
return;
}
const auto symbols = SymbolLoader::load(
SecurityCriteria::create({api::domain::VENUE_GLBX}));
if (symbols.empty()) {
LOG(md, Info, "OnixsGlbxSubscription aborting. No symbols loaded");
return;
}
LOG(md, Info,
"OnixsGlbxSubscription starting. t_id: {}; t_name: {}; config: "
"{}; symbols: {}",
os::get_thread_id(), os::get_thread_name(), config.DebugString(),
FMT_JOIN(symbols, ", "));
vector<shared_ptr<IMDPlatform>> platforms;
auto main = shared_ptr<MDP3Main>();
main->start({symbols.begin(), symbols.end()}, nullptr, nullptr);
platforms.emplace_back(std::move(main));
dispatcher = &utils::Dispatcher::current();
heartbeat_timer.emplace(*dispatcher, 250, [&]() {
for (auto &p : platforms) {
p->send_heartbeats();
}
});
heartbeat_timer->start();
is_running = true;
}
// This will block until dispatcher->terminate() is called.
// There is a slim, but non-zero chance that this is executed after
// 'stop' is called, since we 'run' outside of the lock.
dispatcher->run();
}
void stop() {
BTS_LOCK_GUARD(mx);
if (!is_running) {
return;
}
is_running = false;
if (dispatcher != nullptr) {
LOG(md, Diag, "OnixsGlbxSubscription stopping.");
dispatcher->io_service()->stop();
}
if (dispatcher_thread.joinable()) {
LOG(md, Diag, "OnixsGlbxSubscription joining dispatcher thread.");
dispatcher_thread.join();
}
}
};
OnixsGlbxSubscription::OnixsGlbxSubscription(
shared_ptr<IDGenerator> sec_id_store, GlbxMarketDataConfig config)
: me_(std::make_unique<PImpl>(std::move(sec_id_store), std::move(config))) {
}
OnixsGlbxSubscription::~OnixsGlbxSubscription() = default;
void OnixsGlbxSubscription::start(
shared_ptr<IMarketDataListener> /*listener*/) {
std::atomic_thread_fence(std::memory_order_release);
bts_assert(!me_->dispatcher_thread.joinable());
me_->dispatcher_thread = std::thread{[&]() { me_->run(); }};
}
void OnixsGlbxSubscription::stop() { me_->stop(); }
} // namespace bts::md
@@ -0,0 +1,48 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::api::md {
class GlbxMarketDataConfig;
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
/**
* @brief Subscribes to market data at Glbx via Onixs
*
* @a https://www.onixs.biz/
*/
class OnixsGlbxSubscription : public IMarketDataSource {
public:
OnixsGlbxSubscription(
std::shared_ptr<persistence::IDGenerator> sec_id_store,
api::md::GlbxMarketDataConfig config);
OnixsGlbxSubscription(const OnixsGlbxSubscription &) noexcept = delete;
OnixsGlbxSubscription(OnixsGlbxSubscription &&) noexcept;
OnixsGlbxSubscription &operator=(const OnixsGlbxSubscription &) noexcept =
delete;
OnixsGlbxSubscription &operator=(OnixsGlbxSubscription &&) noexcept;
~OnixsGlbxSubscription() override;
/**
* Starts market data subscription on a background thread.
*
* @note Does not block.
*/
void start(std::shared_ptr<IMarketDataListener> listener) override;
void stop() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -46,6 +46,7 @@ using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::persistence::IDGenerator;
using std::optional;
using std::shared_ptr;
using std::size_t;
using std::string;
using std::unique_ptr;
@@ -136,14 +137,13 @@ struct OkxSubscription::PImpl {
const bool subscribe_trades = true;
size_t sub_id_counter = 0;
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
OkxChannelDemuxer channel_demuxer;
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
OkxMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
excluded_secs(
@@ -157,10 +157,16 @@ struct OkxSubscription::PImpl {
!CONTAINS(excluded_secs, s.desc());
}),
session(
dispatcher, mk_session_cfg(this->cfg), mk_session_opts(this->cfg)),
listener(listener) {
dispatcher,
mk_session_cfg(this->cfg),
mk_session_opts(this->cfg)) {
// DON'T reference the ctr arg 'cfg' (it has been moved to 'this->cfg')
_INFO("ctr. config:\n{}", ProtoPrinter::to_string(this->cfg));
}
void start(std::shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = listener;
sec_sub.emplace(
&session, &instrument_store, this->criteria,
[this](const auto &secs) { on_securities(secs); });
@@ -170,7 +176,7 @@ struct OkxSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] = security_data_map.try_emplace(
sec_desc, std::make_unique<SecurityData>(sid, listener));
sec_desc, std::make_unique<SecurityData>(sid, listener.get()));
_ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc);
_INFO("add_security_data. sec: '{}'", sec_desc);
return *it->second;
@@ -253,10 +259,8 @@ struct OkxSubscription::PImpl {
OkxSubscription::OkxSubscription(
ccapi::EventDispatcher *dispatcher,
OkxMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
OkxSubscription::OkxSubscription(OkxSubscription &&) noexcept = default;
OkxSubscription &OkxSubscription::operator=(OkxSubscription &&) noexcept =
@@ -264,4 +268,9 @@ OkxSubscription &OkxSubscription::operator=(OkxSubscription &&) noexcept =
OkxSubscription::~OkxSubscription() = default;
void OkxSubscription::start(
std::shared_ptr<bts::md::IMarketDataListener> listener) {
me_->start(listener);
}
} // namespace bts::md::okx
@@ -1,5 +1,6 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,25 +26,22 @@ namespace bts::md::okx {
*
* @a https://www.okx.com/docs-v5/en/#websocket-api-public-channel
*/
class OkxSubscription {
class OkxSubscription : public IMarketDataSource {
public:
OkxSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::OkxMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
OkxSubscription(const OkxSubscription &) noexcept = delete;
OkxSubscription(OkxSubscription &&) noexcept;
OkxSubscription &operator=(const OkxSubscription &) noexcept =
delete;
OkxSubscription &operator=(const OkxSubscription &) noexcept = delete;
OkxSubscription &operator=(OkxSubscription &&) noexcept;
~OkxSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
@@ -12,9 +12,11 @@
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/GeneralMarketDataListener.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/MDProducer.h"
#include "spark/md/protos/co2.pb.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/PBUtils.h"
#include "spark/utils/bts_assert.h"
#include "sparkd/md/recording/MDRecorder.h"
@@ -52,27 +54,26 @@ void copy_books(const bts::co2::BookMessage &from, bts::domain::Book &to) {
struct MDReplayer ::PImpl {
bts::api::Venue venue;
std::set<uint32_t> sec_keys = {};
uint64_t start_time = bts::core::TimeStamp::now_ns();
GeneralMarketDataListener listener;
uint64_t start_time = 0;
std::shared_ptr<IMarketDataListener> listener{nullptr};
double replay_speed;
PImpl(
bts::api::Venue venue,
bts::md::GeneralSecurityIndex &index,
bts::md::MDProducer &md_producer, double replay_speed)
: venue(venue), listener(index, md_producer, venue), replay_speed(replay_speed) {}
PImpl(bts::api::Venue venue, double replay_speed)
: venue(venue), replay_speed(replay_speed) {}
};
MDReplayer::MDReplayer(
bts::api::Venue venue,
bts::md::GeneralSecurityIndex &index,
bts::md::MDProducer &md_producer,
double replay_speed)
: me(new PImpl(venue, index, md_producer, replay_speed)) {
MDReplayer::MDReplayer(bts::api::Venue venue, double replay_speed)
: me(new PImpl(venue, replay_speed)) {
LOG(md, Info, "MDReplayer::MDReplayer venue: {}", Venue_Name(venue));
}
MDReplayer::~MDReplayer() {}
void MDReplayer::run() {
void MDReplayer::start(std::shared_ptr<IMarketDataListener> listener) {
LOG(md, Info, "MDReplayer::start venue: {}", Venue_Name(me->venue));
bts_assert(listener != nullptr);
me->listener = std::move(listener);
me->start_time = bts::core::TimeStamp::now_ns();
std::string directory = MDRecorder::recording_path();
auto include_file = [&](const std::string &file_name) {
return MDRecorder::is_md_recording_file(file_name, me->venue);
@@ -132,7 +133,7 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) {
me->sec_keys.insert(m.security().id());
std::vector<domain::Security const *> sec_vec = {};
sec_vec.push_back(&new_sec);
me->listener.on_securities(sec_vec);
me->listener->on_securities(sec_vec);
break;
}
case (co2::RecordedMDMessage::MDRecordCase::kTotalBook): {
@@ -146,20 +147,20 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) {
bts::domain::Book book = {};
copy_books(tot_book.direct_book(), book);
book.set_security_id(bts::domain::SecurityID(m.sec_key()));
me->listener.on_book(book);
me->listener->on_book(book);
}
break;
}
case (co2::RecordedMDMessage::MDRecordCase::kMisc): {
bts::domain::SecurityMarketDataStatistics stats(m.sec_key(), m.misc());
me->listener.on_security_statistics(stats);
me->listener->on_security_statistics(stats);
break;
}
case (co2::RecordedMDMessage::MDRecordCase::kTrade): {
bts::domain::TradeRecord tr(m.trade());
std::vector<domain::TradeRecord const *> trades;
trades.push_back(&tr);
me->listener.on_trades(trades);
me->listener->on_trades(trades);
break;
}
default: {
@@ -1,7 +1,8 @@
#pragma once
#include "spark/domain_model/Security.pb.h"
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/IMarketDataListener.h"
#include "sparkd/md/IMarketDataSource.h"
namespace bts::md {
class MDProducer;
@@ -14,15 +15,12 @@ class RecordedMDMessage;
namespace bts::md::recording {
class MDReplayer {
class MDReplayer : public IMarketDataSource {
public:
MDReplayer(
bts::api::Venue venue,
bts::md::GeneralSecurityIndex &index,
bts::md::MDProducer &md_producer,
double replay_speed=1);
~MDReplayer();
void run();
explicit MDReplayer(bts::api::Venue venue, double replay_speed = 1);
~MDReplayer() override;
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -0,0 +1,50 @@
#include "TaifexMarketDataProvider.h"
#include "spark/api/domain/Venue.pb.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/persistence/IDGenerator.h"
#include "sparkd/md/taifex/TaifexSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::domain::Venue;
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct TaifexMarketDataProvider::PImpl {
MarketDataConfig config;
std::shared_ptr<IDGenerator> sec_id_store;
vector<shared_ptr<TaifexSubscription>> sources;
PImpl(
MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
TaifexMarketDataProvider::TaifexMarketDataProvider(
MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
TaifexMarketDataProvider::~TaifexMarketDataProvider() = default;
vector<pair<Venue, shared_ptr<IMarketDataSource>>>
TaifexMarketDataProvider::get_sources() {
vector<pair<Venue, shared_ptr<IMarketDataSource>>> rv;
for (const auto &taifex : me_->config.taifex()) {
me_->sources.push_back(
std::make_shared<TaifexSubscription>(taifex, me_->sec_id_store));
rv.push_back({Venue::VENUE_TAIFEX, me_->sources.back()});
}
return rv;
}
} // namespace bts::md
@@ -0,0 +1,33 @@
#pragma once
#include "sparkd/md/IMarketDataProvider.h"
#include <memory>
namespace bts::persistence {
class IDGenerator;
}
namespace bts::api::md {
class MarketDataConfig;
}
namespace bts::md {
class TaifexMarketDataProvider : public IMarketDataProvider {
public:
explicit TaifexMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~TaifexMarketDataProvider();
std::vector<
std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,41 @@
#include "TaifexSubscription.h"
#include "spark/api/md/TaifexMarketDataConfig.pb.h"
#include "spark/utils/thread_manager.h"
#include "sparkd/md/databento/DatabentoDataHandler.h"
#include "sparkd/md/glbx/GlbxMarketDataHandler.h"
#include "sparkd/md/taifex/TaifexDataGateway.h"
namespace bts::md {
using bts::api::md::TaifexMarketDataConfig;
using bts::persistence::IDGenerator;
using std::shared_ptr;
struct TaifexSubscription::PImpl {
TaifexMarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
PImpl(TaifexMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
TaifexSubscription::TaifexSubscription(
TaifexMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
TaifexSubscription::~TaifexSubscription() = default;
void TaifexSubscription::start(std::shared_ptr<IMarketDataListener> listener) {
bts::threading::make_thread(
"taifex",
[config = me_->config, sec_id_store = me_->sec_id_store,
listener = std::move(listener)]() {
md::taifex::TaifexDataGateway::run(
config, std::move(listener), std::move(sec_id_store));
})
.detach();
// TODO: implement 'stop' (signal the thread above to terminate, then join it).
}
} // namespace bts::md
@@ -0,0 +1,43 @@
#pragma once
#include "sparkd/md/IMarketDataSource.h"
#include <memory>
namespace bts::api::md {
class TaifexMarketDataConfig;
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
/**
* @brief Subscribes to market data at Taifex
*/
class TaifexSubscription : public IMarketDataSource {
public:
TaifexSubscription(
api::md::TaifexMarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
TaifexSubscription(const TaifexSubscription &) noexcept = delete;
TaifexSubscription(TaifexSubscription &&) noexcept;
TaifexSubscription &operator=(const TaifexSubscription &) noexcept = delete;
TaifexSubscription &operator=(TaifexSubscription &&) noexcept;
~TaifexSubscription() override;
/**
* Starts market data subscription on a background thread.
*
* @note Does not block.
*/
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
+24 -3
View File
@@ -20,19 +20,32 @@ class PyBook {
const std::string &venue() const;
void set_venue(std::string value);
/**
* @brief Gets the name of the security type (e.g., "SPOT", "FUTURE", "OPTION").
*
* @return const std::string& The name of the security type.
*/
const std::string &security_type() const;
/**
* @brief Sets the security type.
*
* @param value The new security type (should be "SPOT", "FUTURE", or "OPTION").
*/
void set_security_type(std::string value);
/**
* @brief Gets the exchange security id. This is typically the security's name
* (e.g., "BTS-PERPETUAL", or "ESH5").
*
* @note Must be unique for each security at a particular venue (i.e., a single
* venue cannot have 2 securities with the same name).
* @note Should be unique for each security at a particular venue (i.e., a single
* venue cannot have 2 securities with the same name). MUST be unique for each security
* of the same type (e.g., SPOT, FUTURE, OPTION) at a particular venue.
*
* @note There is no "security_id" member because that is an internal id that
* we generate elsewhere in c++ code. The python plugins just need to set the
* venue and exchange_security_id.
*/
const std::string &exchange_security_id() const;
/**
* @brief Sets the exchange security id.
*
@@ -57,6 +70,7 @@ class PyBook {
private:
std::string venue_;
std::string security_type_;
std::string exchange_security_id_;
PyBookSide bids_;
PyBookSide asks_;
@@ -69,6 +83,13 @@ class PyBook {
inline const std::string &PyBook::venue() const { return venue_; }
inline void PyBook::set_venue(std::string value) { venue_ = std::move(value); }
inline const std::string &PyBook::security_type() const {
return security_type_;
}
inline void PyBook::set_security_type(std::string value) {
security_type_ = std::move(value);
}
inline const std::string &PyBook::exchange_security_id() const {
return exchange_security_id_;
}
+3 -3
View File
@@ -37,9 +37,6 @@ namespace py = pybind11;
// Binds getter -> <name>, and setter -> set_<name>.
#define PROP(type, name) \
def(_STR(name), &type::name).def(_STR(set_##name), &type::set_##name)
// namespace blah {
// typedef bts::md::py_interop::PyBookSide & (bts::md::py_interop::PyBook::*bids());
// }
// Define python module with c++ classes.
PYBIND11_MODULE(python_md_module, m) {
@@ -93,6 +90,7 @@ PYBIND11_MODULE(python_md_module, m) {
py::class_<bts::md::py_interop::PyBook>(m, "PyBook")
.def(py::init<>())
.PROP(PyBook, venue)
.PROP(PyBook, security_type)
.PROP(PyBook, exchange_security_id)
.EXPLICIT_DEF(PyBook, bids, BookSideGetter)
.DEF(PyBook, set_bids)
@@ -110,6 +108,7 @@ PYBIND11_MODULE(python_md_module, m) {
py::class_<PyTrade>(m, "PyTrade")
.def(py::init<>())
.PROP(PyTrade, venue)
.PROP(PyTrade, security_type)
.PROP(PyTrade, exchange_security_id)
.PROP(PyTrade, price)
.PROP(PyTrade, quantity)
@@ -142,5 +141,6 @@ PYBIND11_MODULE(python_md_module, m) {
}
#undef PROP
#undef EXPLICIT_DEF
#undef DEF
#undef _STR
+7
View File
@@ -12,6 +12,9 @@ class PyTrade {
const std::string &venue() const;
void set_venue(std::string value);
const std::string &security_type() const;
void set_security_type(std::string value);
const std::string &exchange_security_id() const;
void set_exchange_security_id(std::string value);
@@ -37,6 +40,7 @@ class PyTrade {
private:
std::string venue_;
std::string security_type_;
std::string exchange_security_id_;
double price_;
double quantity_;
@@ -51,6 +55,9 @@ class PyTrade {
inline const std::string &PyTrade::venue() const { return venue_; }
inline void PyTrade::set_venue(std::string value) { venue_ = std::move(value); }
inline const std::string &PyTrade::security_type() const { return security_type_; }
inline void PyTrade::set_security_type(std::string value) { security_type_ = std::move(value); }
inline const std::string &PyTrade::exchange_security_id() const {
return exchange_security_id_;
}
@@ -54,6 +54,7 @@ def _get_book_side(levels: list[(float, float)], is_bid_side: bool):
def _get_trade(
venue: str,
security_type: str,
exchange_security_id: str,
price: float,
quantity: float,
@@ -61,6 +62,7 @@ def _get_trade(
):
trade = pymd.PyTrade()
trade.set_venue(venue)
trade.set_security_type(security_type)
trade.set_exchange_security_id(exchange_security_id)
trade.set_price(price)
trade.set_quantity(quantity)
@@ -94,6 +96,7 @@ class DummyMdProvider(pymd.MdProvider):
asks = _get_book_side([(100.25, 3)], is_bid_side=False)
book = pymd.PyBook()
book.set_venue("Glbx")
book.set_security_type("FUTURE")
book.set_exchange_security_id("ESH5")
book.set_bids(bids)
book.set_asks(asks)
@@ -105,6 +108,7 @@ class DummyMdProvider(pymd.MdProvider):
def add_fake_trades():
trade = _get_trade(
venue="Glbx",
security_type="FUTURE",
exchange_security_id="ESH5",
price=100,
quantity=2,
@@ -119,4 +123,4 @@ class DummyMdProvider(pymd.MdProvider):
add_fake_trades()
def stop(self):
print("Stopping python MdProviderImpl")
print("Stopping python DummyMdProvider")
+7
View File
@@ -0,0 +1,7 @@
#pragma once
#define BTS_LOCK_GUARD(lockable) \
std::lock_guard<decltype(lockable)> __lock_guard { lockable }
#define BTS_LOCK_GUARD_AS(lockable, name) \
std::lock_guard<decltype(lockable)> __lock_guard_##name { lockable }
@@ -277,8 +277,7 @@ struct GeneralMarketDataListener::PImpl {
const bool use_given_sec_key = false;
const std::string cc_name_prefix = "cc_security";
size_t n_bk_updates = 0;
bts::persistence::IDGenerator &id_gen =
bts::persistence::IDGenerator::instance();
bts::persistence::IDGenerator *id_gen;
GeneralSecurityIndex &index;
MDProducer &md_producer;
const std::shared_ptr<bts::orderman::SimExecutionGateway> sim_ex_gwy =
@@ -296,10 +295,10 @@ struct GeneralMarketDataListener::PImpl {
api::domain::Venue venue,
GeneralSecurityIndex &index,
MDProducer &md_producer,
std::shared_ptr<bts::orderman::SimExecutionGateway> sim_ex_gwy)
: venue(venue), index(index),
md_producer(md_producer),
sim_ex_gwy(std::move(sim_ex_gwy)) {
std::shared_ptr<bts::orderman::SimExecutionGateway> sim_ex_gwy,
bts::persistence::IDGenerator *sec_id_store)
: venue(venue), id_gen(sec_id_store), index(index),
md_producer(md_producer), sim_ex_gwy(std::move(sim_ex_gwy)) {
if (this->sim_ex_gwy != nullptr) {
this->sim_ex_gwy->set_producers(&this->index);
}
@@ -361,13 +360,17 @@ void copy_books(
// interface functions ////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
GeneralMarketDataListener::GeneralMarketDataListener(
GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue)
GeneralSecurityIndex &index,
MDProducer &md_producer,
api::Venue venue,
persistence::IDGenerator *sec_id_store)
: me(std::make_unique<PImpl>(
domain::DomainSecurityUtils::convert(venue),
index,
md_producer,
bts::orderman::SimExecutionGatewayFactory::get()->get_getayeway(
venue))) {
venue),
sec_id_store)) {
LOG(md, Info,
"GeneralMarketDataListener::GeneralMarketDataListener venue: {}",
bts::api::Venue_Name(venue));
@@ -504,7 +507,7 @@ CCSecSearchResult GeneralMarketDataListener::_find_sec(
if (add_if_absent) {
uint64_t bts_id = cc_id.value();
if (!me->use_given_sec_key) {
bts_id = me->id_gen.get_or_add(me->venue, sec.type(), sec.desc());
bts_id = me->id_gen->get_or_add(me->venue, sec.type(), sec.desc());
}
me->cc_sec_id_map.insert({cc_id, {bts_id}});
LOG(md, Info,
@@ -651,7 +654,7 @@ uint64_t GeneralMarketDataListener::_generate_security_key(
auto logical_key = get_logical_key(sec);
// if (me->use_given_sec_key)
// return sec.security_common.id.value();
return me->id_gen.get_or_add(me->venue, sec.type(), sec.desc());
return me->id_gen->get_or_add(me->venue, sec.type(), sec.desc());
}
void GeneralMarketDataListener::_possibly_match_with_sim_orders(
@@ -5,6 +5,7 @@
#include "spark/md/GeneralSecurityIndex.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/Shm/FixedSecurity.h"
#include "spark/persistence/IDGenerator.h"
namespace bts::md {
class MDProducer;
@@ -16,7 +17,10 @@ namespace bts::md {
class GeneralMarketDataListener : public IMarketDataListener {
public:
GeneralMarketDataListener(
GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue);
GeneralSecurityIndex &index,
MDProducer &md_producer,
api::Venue venue,
persistence::IDGenerator *sec_id_store);
virtual ~GeneralMarketDataListener();
void on_securities(const std::vector<domain::Security const *>
&complete_security_list) override;