refactor: Split MarketDataGateway up into providers/sources

MarketDataGateway still exists, but is greatly simplified. It creates
several `IMarketDataProvider`s, each of which generates a list of
`IMarketDataSource`s. The market data sources provide market data to an
`IMarketDataListener`.

* EmumbaMarketDataProvider
* CCApiMarketDataProvider
* PyMarketDataProvider
* OnixsMarketDataProvider
* ReplayMarketDataProvider

As a side effect, individual py market data sources are now explicitly
limited to a single venue (it probably should have been this way all
along, since the shared memory writers do not expect updates to involve
multiple venues).
This commit is contained in:
John Colvin committed 2025-05-20 20:20:07 +00:00
1 parent 33093f3497
commit 9930b65ddc
53 files changed
+1482 -785

No files matched your search

+7
View File
@@ -0,0 +1,7 @@
#pragma once
#define BTS_LOCK_GUARD(lockable) \
std::lock_guard<decltype(lockable)> __lock_guard { lockable }
#define BTS_LOCK_GUARD_AS(lockable, name) \
std::lock_guard<decltype(lockable)> __lock_guard_##name { lockable }
+13 -9
View File
@@ -315,8 +315,7 @@ struct GeneralMarketDataListener::PImpl {
true; // with true, replay works, with orders goign in ((did not push it in, since it was always false - need to test))
const std::string cc_name_prefix = "cc_security";
size_t n_bk_updates = 0;
bts::persistence::IDGenerator &id_gen =
bts::persistence::IDGenerator::instance();
bts::persistence::IDGenerator *id_gen;
GeneralSecurityIndex &index;
MDProducer &md_producer;
const std::shared_ptr<bts::orderman::SimExecutionGateway> sim_ex_gwy =
@@ -334,9 +333,10 @@ struct GeneralMarketDataListener::PImpl {
api::domain::Venue venue,
GeneralSecurityIndex &index,
MDProducer &md_producer,
std::shared_ptr<bts::orderman::SimExecutionGateway> sim_ex_gwy)
: venue(venue), index(index), md_producer(md_producer),
sim_ex_gwy(std::move(sim_ex_gwy)) {
std::shared_ptr<bts::orderman::SimExecutionGateway> sim_ex_gwy,
bts::persistence::IDGenerator *sec_id_store)
: venue(venue), id_gen(sec_id_store), index(index),
md_producer(md_producer), sim_ex_gwy(std::move(sim_ex_gwy)) {
if (this->sim_ex_gwy != nullptr) {
this->sim_ex_gwy->set_producers(&this->index);
}
@@ -398,13 +398,17 @@ void copy_books(
// interface functions ////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
GeneralMarketDataListener::GeneralMarketDataListener(
GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue)
GeneralSecurityIndex &index,
MDProducer &md_producer,
api::Venue venue,
persistence::IDGenerator *sec_id_store)
: me(std::make_unique<PImpl>(
domain::DomainSecurityUtils::convert(venue),
index,
md_producer,
bts::orderman::SimExecutionGatewayFactory::get()->get_getayeway(
venue))) {
venue),
sec_id_store)) {
LOG(md, Info,
"GeneralMarketDataListener::GeneralMarketDataListener venue: {}",
bts::api::Venue_Name(venue));
@@ -677,7 +681,7 @@ CCSecSearchResult GeneralMarketDataListener::_find_sec(
if (add_if_absent) {
uint64_t bts_id = cc_id.value();
if (!me->use_given_sec_key) {
bts_id = me->id_gen.get_or_add(me->venue, sec.type(), sec.desc());
bts_id = me->id_gen->get_or_add(me->venue, sec.type(), sec.desc());
}
me->cc_sec_id_map.insert({cc_id, {bts_id}});
LOG(md, Info,
@@ -929,7 +933,7 @@ uint64_t GeneralMarketDataListener::_generate_security_key(
auto logical_key = get_logical_key(sec);
// if (me->use_given_sec_key)
// return sec.security_common.id.value();
return me->id_gen.get_or_add(me->venue, sec.type(), sec.desc());
return me->id_gen->get_or_add(me->venue, sec.type(), sec.desc());
}
void GeneralMarketDataListener::_possibly_match_with_sim_orders(
+5 -1
View File
@@ -5,6 +5,7 @@
#include "lib/md/GeneralSecurityIndex.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/Shm/FixedSecurity.h"
#include "lib/persistence/IDGenerator.h"
namespace bts::md {
class MDProducer;
@@ -16,7 +17,10 @@ namespace bts::md {
class GeneralMarketDataListener : public IMarketDataListener {
public:
GeneralMarketDataListener(
GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue);
GeneralSecurityIndex &index,
MDProducer &md_producer,
api::Venue venue,
persistence::IDGenerator *sec_id_store);
virtual ~GeneralMarketDataListener();
void on_securities(const std::vector<domain::Security const *>
&complete_security_list) override;
+7
View File
@@ -0,0 +1,7 @@
#include "IMarketDataProvider.h"
namespace bts::md {
IMarketDataProvider::~IMarketDataProvider() = default;
} // namespace bts::md
+25
View File
@@ -0,0 +1,25 @@
#pragma once
#include "lib/api/domain/Venue.pb.h"
#include "lib/md/IMarketDataSource.h"
#include <vector>
namespace bts::md {
/**
* @brief This is a single function interface, whose sole purpose is to provide a list of market
* data sources (per venue). A good example implementation is the `CCApiMarketDataProvider`, which
* reads `MarketDataConfig` and returns an `IMarketDataSource` for each venue configuration (e.g.,
* `bts::api::md::BinanceMarketDataConfig`, `bts::api::md::CoinbaseMarketDataConfig`, etc.).
*/
class IMarketDataProvider {
public:
virtual ~IMarketDataProvider();
virtual std::vector<
std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() = 0;
};
} // namespace bts::md
+9
View File
@@ -0,0 +1,9 @@
#include "IMarketDataSource.h"
namespace bts::md {
IMarketDataSource::~IMarketDataSource() = default;
void IMarketDataSource::stop() {}
} // namespace bts::md
+20
View File
@@ -0,0 +1,20 @@
#pragma once
#include <memory>
namespace bts::md {
class IMarketDataListener;
/**
* @brief This interface represents a source of market data. Call `IMarketDataSource::start` to
* connect an `IMarketDataListener` to the source and begin receiving market data.
*/
class IMarketDataSource {
public:
virtual ~IMarketDataSource();
virtual void start(std::shared_ptr<IMarketDataListener> listener) = 0;
virtual void stop();
};
} // namespace bts::md
@@ -1,19 +1,24 @@
#include "LiveLadderDemo.h"
#include "lib/api/md/MarketDataConfig.pb.h"
#include "lib/config/config.h"
#include "lib/persistence/IDGenerator.h"
#include "server/sparkd/md/MarketDataGateway.h"
#include "server/sparkd/md/MarketDataRegistration.h"
#include "server/sparkd/md/listeners/LiveLadderPrinter.h"
#include <vector>
#include <thread>
namespace bts::cmds {
LiveLadderDemo::LiveLadderDemo(const api::md::MarketDataConfig &) {
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
LiveLadderDemo::LiveLadderDemo(const MarketDataConfig &config) {
char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt";
putenv(ssl_cert_file);
md::MarketDataGateway gw;
md::MarketDataGateway gw{config, IDGenerator::instance_ptr()};
md::listeners::LiveLadderPrinter ladder_printer;
gw.run();
gw.start();
const std::size_t n_loops = 16;
for (std::size_t n = 0; n < n_loops; n++) {
+11 -3
View File
@@ -1,5 +1,6 @@
#include "Run.h"
#include "Cleanup.h"
#include "lib/api/md/MarketDataConfig.pb.h"
#include "lib/config/protos/market_data.pb.h"
#include "lib/domain/SecurityCriteria.h"
#include "lib/logging/logging.h"
@@ -7,6 +8,7 @@
#include "lib/md/UserSymbol.h"
#include "lib/md/relay/RelayRecorder.h"
#include "lib/os/os.h"
#include "lib/persistence/IDGenerator.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/thread_manager.h"
#include "server/sparkd/md/MarketDataGateway.h"
@@ -14,6 +16,7 @@
#include "server/sparkd/md/platforms/ice_onixs/ICEMain.h"
#include "server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
#include <string>
#include <thread>
namespace bts::cmds::md::cmds {
@@ -91,9 +94,14 @@ static void run_onixs(
static void run_md_gwy() {
LOG(md, Info, "Run::run_md_gwy t_id:{} t_name:{}", os::get_thread_id(),
os::get_thread_name());
bts::md::MarketDataGateway gwy = {};
// this keeps going in a loop
gwy.run();
bts::md::MarketDataGateway gwy{
config::get<api::md::MarketDataConfig>(),
persistence::IDGenerator::instance_ptr()};
gwy.start();
while (true) {
using namespace std::chrono_literals;
std::this_thread::sleep_for(2s);
}
}
void Run::run() {
+78 -424
View File
@@ -1,70 +1,44 @@
#include "MarketDataGateway.h"
#include "lib/api/domain/Venue.pb.h"
#include "lib/api/md/EmumbaDeribitMarketDataConfig.pb.h"
#include "lib/api/domain/Venue.h"
#include "lib/api/md/MarketDataConfig.pb.h"
#include "lib/api/md/MarketDataConfigUtils.h"
#include "lib/api/md/MarketDataReplayConfig.pb.h"
#include "lib/collections/Contains.h"
#include "lib/config/protos/system_config.pb.h"
#include "lib/core/FMT.h"
#include "lib/core/TimeSpan.h"
#include "lib/domain/DomainSecurityUtils.h"
#include "lib/domain/SecurityID.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/io/ProtoPrinter.h"
#include "lib/domain_model/Venue.h"
#include "lib/logging/logging.h"
#include "lib/md/GeneralMarketDataListener.h"
#include "lib/md/GeneralSecurityIndex.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/IMarketDataProvider.h"
#include "lib/md/IMarketDataSource.h"
#include "lib/md/MDProducer.h"
#include "lib/os/os.h"
#include "lib/persistence/IDGenerator.h"
#include "lib/utils/Dispatcher.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/bts_fatal.h"
#include "lib/utils/thread_manager.h"
#include "server/sparkd/md/MarketDataListenerID.h"
#include "server/sparkd/md/MarketDataRegistration.h"
#include "server/sparkd/md/MarketDataRegistrationError.h"
#include "server/sparkd/md/MarketDataRegistrationFactory.h"
#include "server/sparkd/md/MarketDataSubscription.h"
#include "server/sparkd/md/binance/BinanceSubscription.h"
#include "server/sparkd/md/binance_coin_futures/binance_coin_futures_subscription.h"
#include "server/sparkd/md/binance_us/BinanceUsSubscription.h"
#include "server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
#include "server/sparkd/md/cboe/CboeMarketDataHandler.h"
#include "server/sparkd/md/coinbase/CoinbaseSubscription.h"
#include "server/sparkd/md/deribit/DeribitSubscription.h"
#include "server/sparkd/md/emumba/EmumbaDeribitData.h"
#include "server/sparkd/md/ftx/FTXSubscription.h"
#include "server/sparkd/md/glbx/GlbxMarketDataHandler.h"
#include "server/sparkd/md/okx/OkxSubscription.h"
#include "server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
#include "server/sparkd/md/recording/MDReplayer.h"
#include "server/sparkd/md/taifex/TaifexDataGateway.h"
#include "server/sparkd/md/twse/TwseDataGateway.h"
#include "server/sparkd/md/OnixsMarketDataProvider.h"
#include "server/sparkd/md/ReplayMarketDataProvider.h"
#include "server/sparkd/md/ccapi/CCApiMarketDataProvider.h"
#include "server/sparkd/md/databento/DatabentoMarketDataProvider.h"
#include "server/sparkd/md/emumba/EmumbaMarketDataProvider.h"
#include "server/sparkd/md/taifex/TaifexMarketDataProvider.h"
#include <bits/stdint-uintn.h>
#include <boost/filesystem.hpp>
#include <ccapi_cpp/ccapi_event_dispatcher.h>
#include <emumba-deribit-connector/deribit_utils.h>
#include <memory>
#include <optional>
#include <string>
#include <unordered_map>
#include <vector>
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::config::SystemConfig;
//using bts::domain::Security;
using bts::domain::DomainSecurityUtils;
using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::persistence::IDGenerator;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
using Path = boost::filesystem::path;
static bool validate(const MarketDataConfig &cfg, vector<string> &errs) {
auto orig_n_errs = errs.size();
if (!cfg.deribit().empty() && !cfg.emumba_deribit().empty()) {
@@ -74,393 +48,73 @@ static bool validate(const MarketDataConfig &cfg, vector<string> &errs) {
return errs.size() == orig_n_errs;
}
struct MarketDataListenerData {
MarketDataListenerData() = default;
MarketDataListenerData(const MarketDataListenerData &) noexcept = delete;
MarketDataListenerData(MarketDataListenerData &&) noexcept = default;
MarketDataListenerData &operator=(const MarketDataListenerData &) noexcept =
delete;
MarketDataListenerData &operator=(MarketDataListenerData &&) noexcept =
default;
~MarketDataListenerData() = default;
MarketDataListenerID id;
MarketDataSubscription subscription;
};
struct MDReplayData {
bts::api::md::MarketDataReplayConfig replay_conf;
bts::api::Venue venue;
bts::md::GeneralSecurityIndex sec_index;
bts::md::MDProducer md_producer;
bts::md::recording::MDReplayer md_replayer;
MDReplayData(const bts::api::md::MarketDataReplayConfig &replay_conf)
: replay_conf(replay_conf), venue(replay_conf.venue()), sec_index(),
md_replayer(
replay_conf.venue(),
sec_index,
md_producer,
replay_conf.publish_boostrap_books_only(),
replay_conf.replay_speed()) {}
};
struct VenueListenerData {
bts::md::GeneralSecurityIndex sec_index;
bts::md::MDProducer md_producer;
std::shared_ptr<bts::md::IMarketDataListener> listener;
std::optional<ccapi::EventDispatcher> ccapi_dispatcher;
utils::DispatcherTimer timer;
VenueListenerData(bts::api::Venue venue)
: sec_index(),
listener(std::make_shared<bts::md::GeneralMarketDataListener>(
sec_index, md_producer, venue)),
timer(utils::Dispatcher::background(), 200, [&]() {
md_producer.heartbeat();
}) {}
};
struct MarketDataGateway::PImpl {
api::md::MarketDataConfig md_config =
config::get<api::md::MarketDataConfig>();
MarketDataRegistrationFactory registration_factory;
std::shared_ptr<IDGenerator> sec_id_store = IDGenerator::instance_ptr();
unordered_map<MarketDataListenerID, MarketDataListenerData> listeners;
std::vector<std::unique_ptr<MDReplayData>> replays = {};
// we create these and pass them to the cc md sources
std::vector<std::shared_ptr<VenueListenerData>> venue_listeners = {};
struct MdSink {
GeneralSecurityIndex sec_index;
MDProducer md_producer;
shared_ptr<IMarketDataListener> listener;
MdSink(api::Venue venue, IDGenerator *sec_id_store)
: sec_index(), listener(std::make_shared<GeneralMarketDataListener>(
sec_index, md_producer, venue, sec_id_store)) {}
};
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
vector<unique_ptr<IMarketDataProvider>> providers;
vector<unique_ptr<MdSink>> sinks;
vector<shared_ptr<IMarketDataSource>> sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {
providers.push_back(std::make_unique<CCApiMarketDataProvider>(
this->config, this->sec_id_store));
providers.push_back(
std::make_unique<EmumbaMarketDataProvider>(this->config));
providers.push_back(
std::make_unique<OnixsMarketDataProvider>(this->config));
providers.push_back(std::make_unique<DatabentoMarketDataProvider>(
this->config, this->sec_id_store));
providers.push_back(std::make_unique<TaifexMarketDataProvider>(
this->config, this->sec_id_store));
providers.push_back(std::make_unique<ReplayMarketDataProvider>(
this->config, this->sec_id_store));
}
void start() {
bts_assert(sources.empty());
vector<string> cfg_errs;
if (!validate(config, cfg_errs)) {
BTS_FATAL_MSGF("Invalid MarketDataConfig:", FMT_JOIN(cfg_errs, "\n"));
return;
}
LOG(md, Info, "MarketDataGateway::run starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
for (auto &provider : providers) {
for (auto &[venue, source] : provider->get_sources()) {
sources.push_back(source);
auto &sink = sinks.emplace_back(std::make_unique<PImpl::MdSink>(
DomainSecurityUtils::convert(venue), sec_id_store.get()));
LOG(md, Info, "MarketDataGateway::run. Starting {} md", venue);
source->start(sink->listener);
}
}
}
void stop() {
for (auto &source : sources) {
source->stop();
}
}
};
MarketDataGateway::MarketDataGateway()
: me_(std::make_unique<MarketDataGateway::PImpl>()) {}
MarketDataGateway::MarketDataGateway(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<MarketDataGateway::PImpl>(
std::move(config), std::move(sec_id_store))) {}
MarketDataGateway::~MarketDataGateway() = default;
// starts the market data sources and then keeps the function going
void MarketDataGateway::run() {
vector<string> cfg_errs;
if (!validate(me_->md_config, cfg_errs)) {
BTS_FATAL_MSGF("Invalid MarketDataConfig:", FMT_JOIN(cfg_errs, "\n"));
return;
}
LOG(md, Info, "MarketDataGateway::run starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
_start_cc(me_->md_config);
if (!me_->md_config.emumba_deribit().empty()) {
bts_assert(me_->md_config.emumba_deribit().size() == 1);
_start_emumba_deribit(me_->md_config.emumba_deribit().at(0));
}
start_glbx(me_->md_config);
start_taifex(me_->md_config);
start_twse(me_->md_config);
start_cboe(me_->md_config);
_start_replays(me_->md_config);
while (true) {
std::this_thread::sleep_for(std::chrono::seconds(1));
}
LOG(md, Info, "MarketDataGateway::run ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
// CCAPI //////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
MarketDataRegistration MarketDataGateway::_start_cc(
const MarketDataConfig &cfg) noexcept {
LOG(md, Info, "MarketDataGateway::_start_cc starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
auto reg = me_->registration_factory.create();
bts_assert(reg.listener_id.has_value());
bts_assert(!CONTAINS_KEY(me_->listeners, *reg.listener_id));
LOG(md, Info, "MarketDataGateway subscribing to data:\n{}",
ProtoPrinter::to_string(cfg));
vector<string> cfg_errs;
if (!validate(cfg, cfg_errs)) {
return MarketDataRegistration{
{MarketDataRegistrationError::Type::CONFIG_ERROR,
fmt::to_string(fmt::join(cfg_errs, "\n"))}};
}
MarketDataListenerData &listener_data = me_->listeners[*reg.listener_id];
listener_data.id = *reg.listener_id;
// FTX
for (const auto &ftx : cfg.ftx()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_FTX));
listener_data.subscription.add(ftx::FTXSubscription{
&p->ccapi_dispatcher.emplace(), ftx, me_->sec_id_store.get(),
p->listener.get()});
}
// Binance
for (const auto &binance : cfg.binance()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_BINANCE));
listener_data.subscription.add(binance::BinanceSubscription{
&p->ccapi_dispatcher.emplace(), binance, me_->sec_id_store.get(),
p->listener.get()});
}
// Binance Coin Futures
for (const auto &binance_cfs : cfg.binance_coin_futures()) {
auto p =
me_->venue_listeners.emplace_back(std::make_shared<VenueListenerData>(
bts::api::VENUE_BINANCE_COIN_FUTURES));
listener_data.subscription.add(
binance_coin_futures::BinanceCoinFuturesSubscription{
&p->ccapi_dispatcher.emplace(), binance_cfs,
me_->sec_id_store.get(), p->listener.get()});
}
// Binance US
for (const auto &binance_us : cfg.binance_us()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_BINANCE_US));
listener_data.subscription.add(binance_us::BinanceUsSubscription{
&p->ccapi_dispatcher.emplace(), binance_us, me_->sec_id_store.get(),
p->listener.get()});
}
// Coinbase
for (const auto &coinbase : cfg.coinbase()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_COINBASE));
listener_data.subscription.add(coinbase::CoinbaseSubscription{
&p->ccapi_dispatcher.emplace(), coinbase, me_->sec_id_store.get(),
p->listener.get()});
}
// Binance USDS Futures
for (const auto &binance_ufs : cfg.binance_usds_futures()) {
auto p =
me_->venue_listeners.emplace_back(std::make_shared<VenueListenerData>(
bts::api::VENUE_BINANCE_USDS_FUTURES));
listener_data.subscription.add(
binance_usds_futures::BinanceUsdsFuturesSubscription{
&p->ccapi_dispatcher.emplace(), binance_ufs,
me_->sec_id_store.get(), p->listener.get()});
}
// Deribit
for (const auto &deribit : cfg.deribit()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_DERIBIT));
listener_data.subscription.add(deribit::DeribitSubscription{
&p->ccapi_dispatcher.emplace(), deribit, me_->sec_id_store.get(),
p->listener.get()});
}
// Okx
for (const auto &okx : cfg.okx()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_OKX));
listener_data.subscription.add(okx::OkxSubscription{
&p->ccapi_dispatcher.emplace(), okx, me_->sec_id_store.get(),
p->listener.get()});
}
LOG(md, Info, "MarketDataGateway::_start_cc ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
return reg;
}
void MarketDataGateway::unsubscribe(MarketDataListenerID id) {
LOG(md, Info, "Unsubscribing listener {}", id);
me_->listeners.erase(id);
}
/// glbx //////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////
void MarketDataGateway::start_glbx(const bts::api::md::MarketDataConfig &cfg) {
for (const auto &glbx : cfg.glbx()) {
if (!glbx.use_databento()) {
_start_onix_glbx(glbx);
return;
}
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_GLBX));
// Capture `p->listener` and `me_->sec_id_store` by value
auto listener = p->listener;
auto sec_id_store = me_->sec_id_store;
std::string tid_name = "glbx";
bts::threading::make_thread(tid_name, [listener, sec_id_store, glbx]() {
api::md::glbx::GlbxMarketDatahandler::run(
listener, sec_id_store, glbx);
}).detach();
}
}
static void run_onixs() {
LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name",
os::get_thread_id(), os::get_thread_name());
bts::md::MBOWriter *mbo = nullptr;
bts::md::TradeSummarySignalDispatcher *tssd = nullptr;
std::vector<std::shared_ptr<bts::md::IMDPlatform>> platforms;
// get all glbx symbols
auto cme_symbols = SymbolLoader::load(
bts::domain::SecurityCriteria::create({api::domain::VENUE_GLBX}));
if (!cme_symbols.empty()) {
auto *main = new bts::md::MDP3Main();
main->start({cme_symbols.begin(), cme_symbols.end()}, mbo, tssd);
platforms.emplace_back(main);
}
// deribit
// auto *d_platform = new bts::md::EmMDListener()
// Timers
auto &disp = utils::Dispatcher::current();
utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() {
for (auto &p : platforms)
p->send_heartbeats();
});
heartbeat_timer.start();
disp.run();
}
void MarketDataGateway::_start_onix_glbx(
bts::api::md::GlbxMarketDataConfig cfg) {
LOG(md, Info,
"MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. "
"cfg:\n{}",
os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
bts::threading::make_thread("Onix CME", [&]() {
bts::md::run_onixs();
}).detach();
}
// deribit ////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
void MarketDataGateway::_start_emumba_deribit(
bts::api::md::EmumbaDeribitMarketDataConfig cfg) {
LOG(md, Info,
"MarketDataGateway::_start_emumba_deribit starting t_id:{} t_name:{}. "
"cfg:\n{}",
os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
auto start_deribit_thr = [cfg = std::move(cfg)]() {
const auto config_path = Path{config::get<SystemConfig>().config_path()} /
cfg.json_conf_file_name();
const auto conf =
bts::cmds::DeribitDemoConfiguration::read_file(config_path.string());
bts_assert_msgf(
conf.has_value(), "Failed to parse deribit config '{}'",
config_path.string());
// log
for (const auto &s : conf.value().symbol_config_list) {
LOG(md, Info, "sub.symbol: {}", s.symbol_name);
LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future,
s.interested_in_option);
for (const auto &i : s.interested_instruments) {
LOG(md, Info, "sub.instrument: {}", i);
}
}
EmumbaDeribitData ddata_obj;
EmumbaDeribitData *ddata = &ddata_obj;
// end log
bts_assert(conf.has_value());
LOG(md, Info, "Deribit conf: {}", conf.value());
ddata->real_handler =
std::make_shared<emumba::deribit_connector::deribit_handler>(
conf.value().symbol_config_list, conf.value().trading_config,
conf.value().credential_map, conf.value().logging_configuration,
conf.value().deribit_url, &ddata->listener);
// since the trading interface is housed inside the EmMDListener, we need to
// create and send emumba's trading_session_interface
for (const auto &p : conf.value().credential_map) {
const auto &sess = p.second;
auto ts_ptr = std::make_shared<
emumba::deribit_connector::trading_session_interface>(
ddata->real_handler->create_new_trading_session(sess));
ddata->trading_session_vec.push_back({sess, ts_ptr});
}
bts_assert(!ddata->trading_session_vec.empty());
ddata->listener.set_handler(
ddata->real_handler.get(), ddata->trading_session_vec);
ddata->real_handler->run();
};
threading::make_thread("Emumba", start_deribit_thr).detach();
LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
// replays ////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
void MarketDataGateway::_start_replays(
const bts::api::md::MarketDataConfig &cfg) {
LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
for (const auto &r :
MarketDataConfigUtils::get_effective_replay_config(cfg)) {
me_->replays.emplace_back(std::make_unique<MDReplayData>(r));
}
for (const auto &replayer : me_->replays) {
std::string tid_name = "MDReplayer_" + Venue_Name(replayer->venue);
bts::threading::make_thread(tid_name, [&]() {
replayer->md_replayer.run();
}).detach();
}
LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
void MarketDataGateway::start_twse(const bts::api::md::MarketDataConfig &cfg) {
// twse
// int counter = 0;
// LOG(md, Info, "In start, mdgtway.cpp");
// for (const auto &twse : cfg.twse()) {
// counter++;
// auto p = me_->venue_listeners.emplace_back(
// std::make_shared<VenueListenerData>(bts::api::VENUE_TWSE));
// std::string tid_name = "twse" + std::to_string(counter);
// bts::threading::make_thread(tid_name, [&]() {
// bts::md::twse::TwseDataGateway::run(
// twse, p->listener, me_->sec_id_store);
// }).detach();
// }
for (const auto &twse : cfg.twse()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_TWSE));
// Capture `p->listener` and `me_->sec_id_store` by value
auto listener = p->listener;
auto sec_id_store = me_->sec_id_store;
std::string tid_name = "twse";
bts::threading::make_thread(tid_name, [listener, sec_id_store, twse]() {
bts::md::twse::TwseDataGateway::run(
twse, listener, sec_id_store);
}).detach();
}
}
void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) {
// taifex
for (const auto &taifex : cfg.taifex()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_TAIFEX));
// Capture `p->listener` and `me_->sec_id_store` by value
auto listener = p->listener;
auto sec_id_store = me_->sec_id_store;
std::string tid_name = "taifex";
bts::threading::make_thread(tid_name, [listener, sec_id_store, taifex]() {
bts::md::taifex::TaifexDataGateway::run(
taifex, listener, sec_id_store);
}).detach();
}
}
void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg) {
for (const api::md::CboeMarketDataConfig &cboe : cfg.cboe()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_CBOE));
std::string tid_name = "CBOE";
// Capture `p->listener` and `me_->sec_id_store` by value
auto listener = p->listener;
auto sec_id_store = me_->sec_id_store;
bts::threading::make_thread(tid_name, [listener, sec_id_store, cboe]() {
api::md::cboe::CboeMarketDatahandler::run(
listener, sec_id_store, cboe);
}).detach();
}
}
void MarketDataGateway::start() { me_->start(); }
} // namespace bts::md
+13 -25
View File
@@ -1,45 +1,33 @@
#pragma once
#include "lib/api/md/GlbxMarketDataConfig.pb.h"
#include "server/sparkd/md/MarketDataListenerID.h"
#include <memory>
namespace bts::api::md {
class EmumbaDeribitMarketDataConfig;
class MarketDataConfig;
} // namespace bts::api::md
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
class IMarketDataListener;
class MarketDataRegistration;
/**
* @brief The `MarketDataGateway` is responsible for starting all configured market data sources.
* It connects each source to a special `IMarketDataListener` that writes the data to shared memory.
*/
class MarketDataGateway {
public:
MarketDataGateway();
explicit MarketDataGateway(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~MarketDataGateway();
/**
* @brief Connects a market data listener to a market data source
* @return MarketDataRegistration
*/
void run();
void unsubscribe(MarketDataListenerID id);
void start();
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
MarketDataRegistration _start_cc(const api::md::MarketDataConfig &) noexcept;
static void _start_emumba_deribit(
bts::api::md::EmumbaDeribitMarketDataConfig cfg);
static void _start_onix_glbx(
bts::api::md::GlbxMarketDataConfig cfg);
void _start_replays(const bts::api::md::MarketDataConfig &cfg);
void start_glbx(const bts::api::md::MarketDataConfig &cfg);
void start_cboe(const bts::api::md::MarketDataConfig &cfg);
void start_taifex(const bts::api::md::MarketDataConfig &cfg);
void start_twse(const bts::api::md::MarketDataConfig &cfg);
};
} // namespace bts::md
@@ -1,68 +0,0 @@
#include "MarketDataSubscription.h"
#include "server/sparkd/md/binance/BinanceSubscription.h"
#include "server/sparkd/md/binance_coin_futures/binance_coin_futures_subscription.h"
#include "server/sparkd/md/binance_us/BinanceUsSubscription.h"
#include "server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
#include "server/sparkd/md/coinbase/CoinbaseSubscription.h"
#include "server/sparkd/md/deribit/DeribitSubscription.h"
#include "server/sparkd/md/ftx/FTXSubscription.h"
#include "server/sparkd/md/okx/OkxSubscription.h"
#include <vector>
namespace bts::md {
using std::vector;
struct MarketDataSubscription::PImpl {
vector<ftx::FTXSubscription> ftx_subs;
vector<binance::BinanceSubscription> binance_subs;
vector<binance_coin_futures::BinanceCoinFuturesSubscription>
binance_coin_futures_subs;
vector<binance_us::BinanceUsSubscription> binance_us_subs;
vector<coinbase::CoinbaseSubscription> coinbase_subs;
vector<binance_usds_futures::BinanceUsdsFuturesSubscription>
binance_usds_futures_subs;
vector<deribit::DeribitSubscription> deribit_subs;
vector<okx::OkxSubscription> okx_subs;
// TODO: add more exchanges
};
MarketDataSubscription::MarketDataSubscription()
: me_(std::make_unique<PImpl>()) {}
MarketDataSubscription::~MarketDataSubscription() = default;
void MarketDataSubscription::add(binance::BinanceSubscription sub) {
me_->binance_subs.push_back(std::move(sub));
}
void MarketDataSubscription::add(
binance_coin_futures::BinanceCoinFuturesSubscription sub) {
me_->binance_coin_futures_subs.push_back(std::move(sub));
}
void MarketDataSubscription::add(binance_us::BinanceUsSubscription sub) {
me_->binance_us_subs.push_back(std::move(sub));
}
void MarketDataSubscription::add(coinbase::CoinbaseSubscription sub) {
me_->coinbase_subs.push_back(std::move(sub));
}
void MarketDataSubscription::add(
binance_usds_futures::BinanceUsdsFuturesSubscription sub) {
me_->binance_usds_futures_subs.push_back(std::move(sub));
}
void MarketDataSubscription::add(deribit::DeribitSubscription sub) {
me_->deribit_subs.push_back(std::move(sub));
}
void MarketDataSubscription::add(ftx::FTXSubscription sub) {
me_->ftx_subs.push_back(std::move(sub));
}
void MarketDataSubscription::add(okx::OkxSubscription sub) {
me_->okx_subs.push_back(std::move(sub));
}
} // namespace bts::md
@@ -1,64 +0,0 @@
#pragma once
#include <memory>
namespace bts::md::binance {
class BinanceSubscription;
}
namespace bts::md::binance_coin_futures {
class BinanceCoinFuturesSubscription;
}
namespace bts::md::binance_us {
class BinanceUsSubscription;
}
namespace bts::md::coinbase {
class CoinbaseSubscription;
}
namespace bts::md::binance_usds_futures {
class BinanceUsdsFuturesSubscription;
}
namespace bts::md::deribit {
class DeribitSubscription;
}
namespace bts::md::ftx {
class FTXSubscription;
}
namespace bts::md::okx {
class OkxSubscription;
}
namespace bts::md {
class MarketDataSubscription {
public:
MarketDataSubscription();
MarketDataSubscription(const MarketDataSubscription &) noexcept = delete;
MarketDataSubscription(MarketDataSubscription &&) noexcept = default;
MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept =
delete;
MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept =
default;
~MarketDataSubscription();
void add(binance::BinanceSubscription);
void add(binance_coin_futures::BinanceCoinFuturesSubscription);
void add(binance_us::BinanceUsSubscription);
void add(coinbase::CoinbaseSubscription);
void add(binance_usds_futures::BinanceUsdsFuturesSubscription);
void add(deribit::DeribitSubscription);
void add(ftx::FTXSubscription);
void add(okx::OkxSubscription);
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,44 @@
#include "OnixsMarketDataProvider.h"
#include "lib/api/domain/Venue.pb.h"
#include "lib/persistence/IDGenerator.h"
#include "server/sparkd/md/glbx/OnixsGlbxSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::domain::Venue;
using bts::api::md::MarketDataConfig;
using bts::md::OnixsMarketDataProvider;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct OnixsMarketDataProvider::PImpl {
MarketDataConfig config;
std::shared_ptr<IDGenerator> sec_id_store;
vector<shared_ptr<OnixsGlbxSubscription>> sources;
};
OnixsMarketDataProvider::OnixsMarketDataProvider(MarketDataConfig config)
: me_(std::make_unique<PImpl>()) {
me_->config = std::move(config);
}
OnixsMarketDataProvider::~OnixsMarketDataProvider() = default;
vector<pair<Venue, shared_ptr<IMarketDataSource>>>
OnixsMarketDataProvider::get_sources() {
vector<pair<Venue, shared_ptr<IMarketDataSource>>> rv;
for (const auto &glbx : me_->config.glbx()) {
me_->sources.push_back(
std::make_shared<OnixsGlbxSubscription>(me_->sec_id_store, glbx));
rv.push_back({Venue::VENUE_GLBX, me_->sources.back()});
}
return rv;
}
} // namespace bts::md
@@ -0,0 +1,26 @@
#pragma once
#include "lib/api/md/MarketDataConfig.pb.h"
#include "lib/md/IMarketDataProvider.h"
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace bts::md {
class OnixsMarketDataProvider : public IMarketDataProvider {
public:
OnixsMarketDataProvider();
explicit OnixsMarketDataProvider(api::md::MarketDataConfig config);
~OnixsMarketDataProvider();
std::vector<
std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,52 @@
#include "ReplayMarketDataProvider.h"
#include "lib/api/md/MarketDataConfig.pb.h"
#include "lib/api/md/MarketDataConfigUtils.h"
#include "lib/config/config.h"
#include "lib/domain/DomainSecurityUtils.h"
#include "lib/md/IMarketDataSource.h"
#include "lib/persistence/IDGenerator.h"
#include "server/sparkd/md/recording/MDReplayer.h"
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::domain::DomainSecurityUtils;
using bts::md::recording::MDReplayer;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct ReplayMarketDataProvider::PImpl {
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
ReplayMarketDataProvider::ReplayMarketDataProvider()
: ReplayMarketDataProvider(
config::get<MarketDataConfig>(), IDGenerator::instance_ptr()) {}
ReplayMarketDataProvider::ReplayMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
ReplayMarketDataProvider::~ReplayMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
ReplayMarketDataProvider::get_sources() {
const auto replay_cfgs =
MarketDataConfigUtils::get_effective_replay_config(me_->config);
for (const auto &r : replay_cfgs) {
const auto v = DomainSecurityUtils::convert(r.venue());
me_->sources.push_back({v, std::make_shared<MDReplayer>(r.venue())});
}
return me_->sources;
}
} // namespace bts::md
@@ -0,0 +1,28 @@
#pragma once
#include "lib/api/md/MarketDataConfig.pb.h"
#include "lib/md/IMarketDataProvider.h"
#include "lib/persistence/IDGenerator.h"
#include <memory>
namespace bts::md {
class ReplayMarketDataProvider : public IMarketDataProvider {
public:
ReplayMarketDataProvider();
explicit ReplayMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~ReplayMarketDataProvider();
std::vector<
std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,47 @@
#include "ReplayMarketDataSubscription.h"
#include "lib/api/md/MarketDataReplayConfig.pb.h"
#include "lib/logging/logging.h"
#include "lib/md/GeneralMarketDataListener.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/MDProducer.h"
#include "lib/utils/thread_manager.h"
#include "server/sparkd/md/recording/MDReplayer.h"
#include <ccapi_cpp/ccapi_event_dispatcher.h>
#include <optional>
namespace bts::md {
using bts::api::md::MarketDataReplayConfig;
using bts::md::recording::MDReplayer;
using std::string;
struct ReplayMarketDataSource::PImpl {
MarketDataReplayConfig config;
PImpl(const MarketDataReplayConfig &config) : config(config) {}
};
ReplayMarketDataSource::ReplayMarketDataSource(MarketDataReplayConfig config)
: me_(std::make_unique<PImpl>(config)) {
me_->config = std::move(config);
}
ReplayMarketDataSource::~ReplayMarketDataSource() = default;
void ReplayMarketDataSource::start(
std::shared_ptr<IMarketDataListener> listener) {
LOG(md, Info, "MarketDataGateway::_start_replays starting t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
auto md_replayer = std::make_shared<MDReplayer>(me_->config.venue());
std::string tid_name = "MDReplayer_" + Venue_Name(me_->config.venue());
bts::threading::make_thread(tid_name, [&, listener, md_replayer]() {
md_replayer->start(listener);
}).detach();
LOG(md, Info, "MarketDataGateway::_start_replays ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
void ReplayMarketDataSource::stop() {}
} // namespace bts::md
@@ -0,0 +1,25 @@
#pragma once
#include "lib/api/md/MarketDataReplayConfig.pb.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace bts::md {
class ReplayMarketDataSource : public IMarketDataSource {
public:
ReplayMarketDataSource();
explicit ReplayMarketDataSource(api::md::MarketDataReplayConfig config);
~ReplayMarketDataSource();
void start(std::shared_ptr<IMarketDataListener> listener) override;
void stop() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -193,7 +193,7 @@ struct BinanceSubscription::PImpl {
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
std::shared_ptr<IMarketDataListener> listener;
// These aliases are useful for quickly switching the handler types (e.g.,
// a special handler that logs messages for testing/diagnostic purposes)
@@ -216,20 +216,23 @@ struct BinanceSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
BinanceMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(
dispatcher,
mk_session_cfg(this->cfg),
mk_session_opts(),
{ccapi::BtsVenues::BINANCE}),
listener(listener) {
{ccapi::BtsVenues::BINANCE}) {
LOG(md, Info, "Initializing BinanceSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
// Request the list of all spots at Binance. Match the results against our SecurityCriteria,
// and subscribe to md for each matching spot.
}
void start(std::shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
// Request the list of all spots at Binance. Match the results against our
// SecurityCriteria, and subscribe to md for each matching spot.
start_thread = std::thread{[this]() {
LOG(md, Info, "Requesting Binance securities on background thread");
std::string exch_info_url = this->cfg.url_rest_base();
@@ -266,7 +269,7 @@ struct BinanceSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert(is_new);
return it->second;
}
@@ -275,7 +278,7 @@ struct BinanceSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -291,7 +294,8 @@ struct BinanceSubscription::PImpl {
sec_desc_id_tuples.emplace_back(type, std::move(name), id);
}
auto [it, is_new] = stats_handlers.try_emplace(
hnd_id, std::make_unique<StatsHandler>(sec_desc_id_tuples, listener));
hnd_id,
std::make_unique<StatsHandler>(sec_desc_id_tuples, listener.get()));
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", hnd_id);
return *it->second;
@@ -401,10 +405,8 @@ struct BinanceSubscription::PImpl {
BinanceSubscription::BinanceSubscription(
ccapi::EventDispatcher *dispatcher,
BinanceMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
BinanceSubscription::BinanceSubscription(BinanceSubscription &&) noexcept =
default;
@@ -413,4 +415,8 @@ BinanceSubscription &BinanceSubscription::operator=(
BinanceSubscription::~BinanceSubscription() = default;
void BinanceSubscription::start(std::shared_ptr<IMarketDataListener> listener) {
me_->start(listener);
}
} // namespace bts::md::binance
@@ -1,5 +1,7 @@
#pragma once
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,13 +27,12 @@ namespace bts::md::binance {
*
* @a https://binance-docs.github.io/apidocs/spot/en/#exchange-information
*/
class BinanceSubscription {
class BinanceSubscription : public IMarketDataSource {
public:
BinanceSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::BinanceMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
BinanceSubscription(const BinanceSubscription &) noexcept = delete;
BinanceSubscription(BinanceSubscription &&) noexcept;
@@ -39,10 +40,9 @@ class BinanceSubscription {
delete;
BinanceSubscription &operator=(BinanceSubscription &&) noexcept;
~BinanceSubscription();
~BinanceSubscription() override;
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -18,6 +18,7 @@
#include "server/sparkd/md/ccapi/CCApiSession.h"
#include "server/sparkd/md/ccapi/CCApiTradeAdaptor.h"
#include <ccapi_cpp/bts_session_options.h>
#include <ccapi_cpp/ccapi_event_dispatcher.h>
#include <ccapi_cpp/ccapi_macro.h>
#include <ccapi_cpp/ccapi_message.h>
@@ -41,6 +42,7 @@ using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::uint32_t;
using std::unique_ptr;
@@ -169,7 +171,7 @@ struct BinanceCoinFuturesSubscription::PImpl {
SecurityCriteria criteria;
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
std::shared_ptr<IMarketDataListener> listener;
BinanceCoinFuturesSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
@@ -185,8 +187,7 @@ struct BinanceCoinFuturesSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
BinanceCoinFuturesMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(
@@ -194,10 +195,15 @@ struct BinanceCoinFuturesSubscription::PImpl {
mk_session_cfg(this->cfg),
{},
{ccapi::BtsVenues::BINANCE_COIN_FUTURES}),
listener(listener), sec_requestor(&session, this->cfg, sec_id_store) {
LOG(md, Info, "Initializing BinanceCoinFuturesSubscription:\n{}",
sec_requestor(&session, this->cfg, sec_id_store) {
LOG(md, Info, "Initializing BinanceUsdsFuturesSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
@@ -224,7 +230,7 @@ struct BinanceCoinFuturesSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const auto sec_id = get_sec_id(sec_type, sec_desc);
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Depth handler already created for '{}'", sec_desc);
return it->second;
@@ -234,7 +240,7 @@ struct BinanceCoinFuturesSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const auto sec_id = get_sec_id(sec_type, sec_desc);
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -244,7 +250,7 @@ struct BinanceCoinFuturesSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const auto sec_id = get_sec_id(sec_type, sec_desc);
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
stats_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Stats handler already created for '{}'", sec_desc);
return it->second;
@@ -316,10 +322,8 @@ struct BinanceCoinFuturesSubscription::PImpl {
BinanceCoinFuturesSubscription::BinanceCoinFuturesSubscription(
ccapi::EventDispatcher *dispatcher,
BinanceCoinFuturesMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
BinanceCoinFuturesSubscription::BinanceCoinFuturesSubscription(
BinanceCoinFuturesSubscription &&) noexcept = default;
@@ -328,4 +332,9 @@ BinanceCoinFuturesSubscription &BinanceCoinFuturesSubscription::operator=(
BinanceCoinFuturesSubscription::~BinanceCoinFuturesSubscription() = default;
void BinanceCoinFuturesSubscription::start(
shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::binance_coin_futures
@@ -1,5 +1,7 @@
#pragma once
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,13 +27,12 @@ namespace bts::md::binance_coin_futures {
*
* @a https://developers.binance.com/docs/derivatives/coin-margined-futures/market-data/Exchange-Information
*/
class BinanceCoinFuturesSubscription {
class BinanceCoinFuturesSubscription : public IMarketDataSource {
public:
BinanceCoinFuturesSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::BinanceCoinFuturesMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
BinanceCoinFuturesSubscription(
const BinanceCoinFuturesSubscription &) noexcept = delete;
@@ -43,8 +44,7 @@ class BinanceCoinFuturesSubscription {
~BinanceCoinFuturesSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -47,6 +47,7 @@ using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::tuple;
using std::uint32_t;
@@ -174,7 +175,7 @@ struct BinanceUsSubscription::PImpl {
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
BinanceUsSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
@@ -191,14 +192,13 @@ struct BinanceUsSubscription::PImpl {
// different map, 'stats_handlers' (unique ids are generated w/ a counter,
// 'stats_handlers_id_counter').
unordered_map<string, StatsHandler *> sec_stats_handler_map;
unordered_map<uint32_t, std::unique_ptr<StatsHandler>> stats_handlers;
unordered_map<uint32_t, unique_ptr<StatsHandler>> stats_handlers;
uint32_t stats_handlers_id_counter = 0;
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
BinanceUsMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(
@@ -206,7 +206,6 @@ struct BinanceUsSubscription::PImpl {
mk_session_cfg(this->cfg),
{},
{ccapi::BtsVenues::BINANCE_US}),
listener(listener),
sec_requestor(
&session,
this->criteria,
@@ -217,6 +216,11 @@ struct BinanceUsSubscription::PImpl {
LOG(md, Info, "Initializing BinanceUsSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(std::shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
@@ -224,7 +228,7 @@ struct BinanceUsSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert(is_new);
return it->second;
}
@@ -233,7 +237,7 @@ struct BinanceUsSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -249,7 +253,8 @@ struct BinanceUsSubscription::PImpl {
sec_desc_id_tuples.emplace_back(type, std::move(name), id);
}
auto [it, is_new] = stats_handlers.try_emplace(
hnd_id, std::make_unique<StatsHandler>(sec_desc_id_tuples, listener));
hnd_id,
std::make_unique<StatsHandler>(sec_desc_id_tuples, listener.get()));
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", hnd_id);
return *it->second;
@@ -339,10 +344,8 @@ struct BinanceUsSubscription::PImpl {
BinanceUsSubscription::BinanceUsSubscription(
ccapi::EventDispatcher *dispatcher,
BinanceUsMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
BinanceUsSubscription::BinanceUsSubscription(
BinanceUsSubscription &&) noexcept = default;
@@ -351,4 +354,8 @@ BinanceUsSubscription &BinanceUsSubscription::operator=(
BinanceUsSubscription::~BinanceUsSubscription() = default;
void BinanceUsSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::binance_us
@@ -1,5 +1,7 @@
#pragma once
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,13 +27,12 @@ namespace bts::md::binance_us {
*
* @a https://docs.binance.us/#get-exchange-information
*/
class BinanceUsSubscription {
class BinanceUsSubscription : public IMarketDataSource {
public:
BinanceUsSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::BinanceUsMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
BinanceUsSubscription(const BinanceUsSubscription &) noexcept = delete;
BinanceUsSubscription(BinanceUsSubscription &&) noexcept;
@@ -41,8 +42,7 @@ class BinanceUsSubscription {
~BinanceUsSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -47,6 +47,7 @@ using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::uint32_t;
using std::unique_ptr;
@@ -187,9 +188,10 @@ struct BinanceUsdsFuturesSubscription::PImpl {
// (i.e., inspection in debugger, temp logs, etc.)
BinanceUsdsFuturesMarketDataConfig cfg;
SecurityCriteria criteria;
ccapi::EventDispatcher *dispatcher;
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
// These aliases are useful for quickly switching the handler types (e.g.,
// a special handler that logs messages for testing/diagnostic purposes)
@@ -205,31 +207,29 @@ struct BinanceUsdsFuturesSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
BinanceUsdsFuturesMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(
dispatcher,
mk_session_cfg(this->cfg),
mk_session_opts(),
{ccapi::BtsVenues::BINANCE_USDS_FUTURES}),
listener(listener) {
dispatcher(dispatcher), session(
dispatcher,
mk_session_cfg(this->cfg),
mk_session_opts(),
{ccapi::BtsVenues::BINANCE_USDS_FUTURES}) {
LOG(md, Info, "Initializing BinanceUsdsFuturesSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
// Request the list of all spots at Binance. Match the results against our SecurityCriteria,
// and subscribe to md for each matching spot.
start_thread = std::thread{[this, dispatcher]() {
start_thread = std::thread{[this]() {
std::string exch_info_url = this->cfg.url_rest_base();
exch_info_url.append("/fapi/v1/exchangeInfo");
bts::http::HttpResponse exch_info_res;
bts::http::HttpClient::get(exch_info_url, {}, &exch_info_res);
std::vector<domain::Security> secs;
// LOG(md, Info,
// "BinanceUsdsFuturesSubscription::start_thread. Received "
// "securities.\n{}",
// exch_info_res);
BinanceUsdsFuture::parse(exch_info_res.body, secs);
// Exclude secs that don't satisfy the SecurityCriteria
auto is_sec_excluded = [this](const Security &sec) {
@@ -267,7 +267,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const auto sec_id = get_sec_id(sec_type, sec_desc);
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Depth handler already created for '{}'", sec_desc);
return it->second;
@@ -277,7 +277,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const auto sec_id = get_sec_id(sec_type, sec_desc);
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -287,7 +287,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const auto sec_id = get_sec_id(sec_type, sec_desc);
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
stats_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Stats handler already created for '{}'", sec_desc);
return it->second;
@@ -365,10 +365,8 @@ struct BinanceUsdsFuturesSubscription::PImpl {
BinanceUsdsFuturesSubscription::BinanceUsdsFuturesSubscription(
ccapi::EventDispatcher *dispatcher,
BinanceUsdsFuturesMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
BinanceUsdsFuturesSubscription::BinanceUsdsFuturesSubscription(
BinanceUsdsFuturesSubscription &&) noexcept = default;
@@ -377,4 +375,9 @@ BinanceUsdsFuturesSubscription &BinanceUsdsFuturesSubscription::operator=(
BinanceUsdsFuturesSubscription::~BinanceUsdsFuturesSubscription() = default;
void BinanceUsdsFuturesSubscription::start(
std::shared_ptr<IMarketDataListener> listener) {
me_->start(listener);
}
} // namespace bts::md::binance_usds_futures
@@ -1,5 +1,7 @@
#pragma once
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,13 +27,12 @@ namespace bts::md::binance_usds_futures {
*
* @a https://binance-docs.github.io/apidocs/futures/en/#exchange-information
*/
class BinanceUsdsFuturesSubscription {
class BinanceUsdsFuturesSubscription : public IMarketDataSource {
public:
BinanceUsdsFuturesSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::BinanceUsdsFuturesMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
BinanceUsdsFuturesSubscription(
const BinanceUsdsFuturesSubscription &) noexcept = delete;
@@ -43,8 +44,7 @@ class BinanceUsdsFuturesSubscription {
~BinanceUsdsFuturesSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -0,0 +1,139 @@
#include "CCApiMarketDataProvider.h"
#include "lib/api/domain/Venue.pb.h"
#include "lib/api/md/MarketDataConfig.pb.h"
#include "lib/config/config.h"
#include "lib/io/ProtoPrinter.h"
#include "lib/logging/logging.h"
#include "lib/md/GeneralMarketDataListener.h"
#include "lib/md/GeneralSecurityIndex.h"
#include "lib/md/MDProducer.h"
#include "lib/persistence/IDGenerator.h"
#include "server/sparkd/md/binance/BinanceSubscription.h"
#include "server/sparkd/md/binance_us/BinanceUsSubscription.h"
#include "server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
#include "server/sparkd/md/coinbase/CoinbaseSubscription.h"
#include "server/sparkd/md/deribit/DeribitSubscription.h"
#include "server/sparkd/md/ftx/FTXSubscription.h"
#include "server/sparkd/md/okx/OkxSubscription.h"
#include <ccapi_cpp/ccapi_event_dispatcher.h>
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
static bool validate(
const MarketDataConfig & /*cfg*/, vector<string> & /*errs*/) {
return true;
}
struct CCApiMarketDataProvider::PImpl {
struct VenueSink {
GeneralSecurityIndex sec_index;
MDProducer md_producer;
GeneralMarketDataListener listener;
VenueSink(api::Venue venue, shared_ptr<IDGenerator> sec_id_store)
: sec_index(),
listener(sec_index, md_producer, venue, sec_id_store.get()) {}
};
MarketDataConfig config;
ccapi::EventDispatcher dispatcher;
shared_ptr<IDGenerator> sec_id_store;
unordered_map<api::Venue, unique_ptr<VenueSink>> venue_sinks;
vector<std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
VenueSink *get_venue_sink(api::Venue venue) {
auto it = venue_sinks.find(venue);
if (it != venue_sinks.end()) {
return it->second.get();
}
return venue_sinks
.try_emplace(venue, std::make_unique<VenueSink>(venue, sec_id_store))
.first->second.get();
}
};
CCApiMarketDataProvider::CCApiMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
CCApiMarketDataProvider::~CCApiMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
CCApiMarketDataProvider::get_sources() {
auto &cfg = me_->config;
vector<string> cfg_errs;
if (!validate(cfg, cfg_errs)) {
LOG(md, Error,
"CCApiMarketDataProvider::get_sources failed. Invalid config: {}",
fmt::to_string(fmt::join(cfg_errs, "\n")));
}
// Binance
for (const auto &binance : cfg.binance()) {
me_->sources.emplace_back(
api::domain::VENUE_BINANCE,
std::make_shared<binance::BinanceSubscription>(
&me_->dispatcher, binance, me_->sec_id_store.get()));
}
// Binance US
for (const auto &binance_us : cfg.binance_us()) {
me_->sources.emplace_back(
api::domain::VENUE_BINANCE_US,
std::make_shared<binance_us::BinanceUsSubscription>(
&me_->dispatcher, binance_us, me_->sec_id_store.get()));
}
// Binance USDS Futures
for (const auto &binance_ufs : cfg.binance_usds_futures()) {
me_->sources.emplace_back(
api::domain::VENUE_BINANCE_USDS_FUTURES,
std::make_shared<
binance_usds_futures::BinanceUsdsFuturesSubscription>(
&me_->dispatcher, binance_ufs, me_->sec_id_store.get()));
}
// Coinbase
for (const auto &coinbase : cfg.coinbase()) {
me_->sources.emplace_back(
api::domain::VENUE_COINBASE,
std::make_shared<coinbase::CoinbaseSubscription>(
&me_->dispatcher, coinbase, me_->sec_id_store.get()));
}
// Deribit
for (const auto &deribit : cfg.deribit()) {
me_->sources.emplace_back(
api::domain::VENUE_DERIBIT,
std::make_shared<deribit::DeribitSubscription>(
&me_->dispatcher, deribit, me_->sec_id_store.get()));
}
// FTX
for (const auto &ftx : cfg.ftx()) {
me_->sources.emplace_back(
api::domain::VENUE_FTX,
std::make_shared<ftx::FTXSubscription>(
&me_->dispatcher, ftx, me_->sec_id_store.get()));
}
// Okx
for (const auto &okx : cfg.okx()) {
me_->sources.emplace_back(
api::domain::VENUE_OKX,
std::make_shared<okx::OkxSubscription>(
&me_->dispatcher, okx, me_->sec_id_store.get()));
}
return me_->sources;
}
} // namespace bts::md
@@ -0,0 +1,27 @@
#pragma once
#include "lib/api/md/MarketDataConfig.pb.h"
#include "lib/md/IMarketDataProvider.h"
#include "lib/persistence/IDGenerator.h"
#include <memory>
namespace bts::md {
class CCApiMarketDataProvider : public IMarketDataProvider {
public:
explicit CCApiMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~CCApiMarketDataProvider();
std::vector<
std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -5,7 +5,9 @@
#include "lib/domain/SecurityCriteria.h"
#include "lib/io/ProtoPrinter.h"
#include "lib/io/QueryParams.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/persistence/IDGenerator.h"
#include "lib/utils/bts_assert.h"
#include "server/sparkd/md/ccapi/CCApiMarketDepthAdaptor.h"
#include "server/sparkd/md/ccapi/CCApiSession.h"
#include "server/sparkd/md/ccapi/CCApiTradeAdaptor.h"
@@ -35,6 +37,7 @@ using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
@@ -150,7 +153,7 @@ struct CoinbaseSubscription::PImpl {
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
CoinbaseSecurityRequestor sec_requestor;
// These aliases are useful for quickly switching the handler types (e.g.,
@@ -171,8 +174,7 @@ struct CoinbaseSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
CoinbaseMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
session(
@@ -180,7 +182,6 @@ struct CoinbaseSubscription::PImpl {
mk_session_cfg(this->cfg),
{},
{ccapi::BtsVenues::COINBASE}),
listener(listener),
sec_requestor(
&session,
this->criteria,
@@ -191,6 +192,11 @@ struct CoinbaseSubscription::PImpl {
LOG(md, Info, "Initializing CoinbaseSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
@@ -199,7 +205,7 @@ struct CoinbaseSubscription::PImpl {
auto sec_id =
SecurityID{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
stats_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", sec_id);
return it->second;
@@ -208,7 +214,7 @@ struct CoinbaseSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert(is_new);
return it->second;
}
@@ -217,7 +223,7 @@ struct CoinbaseSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -289,10 +295,8 @@ struct CoinbaseSubscription::PImpl {
CoinbaseSubscription::CoinbaseSubscription(
ccapi::EventDispatcher *dispatcher,
CoinbaseMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
CoinbaseSubscription::CoinbaseSubscription(CoinbaseSubscription &&) noexcept =
default;
@@ -301,4 +305,8 @@ CoinbaseSubscription &CoinbaseSubscription::operator=(
CoinbaseSubscription::~CoinbaseSubscription() = default;
void CoinbaseSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::coinbase
@@ -1,5 +1,7 @@
#pragma once
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -20,13 +22,12 @@ class IDGenerator;
namespace bts::md::coinbase {
class CoinbaseSubscription {
class CoinbaseSubscription : public IMarketDataSource {
public:
CoinbaseSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::CoinbaseMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
CoinbaseSubscription(const CoinbaseSubscription &) noexcept = delete;
CoinbaseSubscription(CoinbaseSubscription &&) noexcept;
@@ -36,8 +37,7 @@ class CoinbaseSubscription {
~CoinbaseSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -0,0 +1,52 @@
#include "DatabentoMarketDataProvider.h"
#include "lib/api/domain/Venue.pb.h"
#include "lib/api/md/MarketDataConfig.pb.h"
#include "lib/md/IMarketDataSource.h"
#include "lib/persistence/IDGenerator.h"
#include "server/sparkd/md/databento/DatabentoDataHandler.h"
#include "server/sparkd/md/glbx/DatabentoGlbxSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct DatabentoMarketDataProvider::PImpl {
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
vector<shared_ptr<IMarketDataSource>> glbx_sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
DatabentoMarketDataProvider::DatabentoMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
me_->config = std::move(config);
}
DatabentoMarketDataProvider::~DatabentoMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
DatabentoMarketDataProvider::get_sources() {
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> rv;
for (const auto &glbx : me_->config.glbx()) {
if (glbx.use_databento()) {
me_->glbx_sources.push_back(
std::make_shared<DatabentoGlbxSubscription>(
glbx, me_->sec_id_store));
rv.push_back({api::domain::VENUE_GLBX, me_->glbx_sources.back()});
}
}
return rv;
}
} // namespace bts::md
@@ -0,0 +1,33 @@
#pragma once
#include "lib/md/IMarketDataProvider.h"
#include <memory>
namespace bts::persistence {
class IDGenerator;
}
namespace bts::api::md {
class MarketDataConfig;
}
namespace bts::md {
class DatabentoMarketDataProvider : public IMarketDataProvider {
public:
explicit DatabentoMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~DatabentoMarketDataProvider();
std::vector<
std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -52,6 +52,7 @@ using bts::io::websocket::JsonRpcId;
using bts::io::websocket::JsonRpcRequest;
using bts::persistence::IDGenerator;
using nlohmann::json;
using std::shared_ptr;
using std::string;
using std::uint32_t;
using std::unique_ptr;
@@ -134,14 +135,13 @@ struct DeribitSubscription::PImpl {
const bool subscribe_trades = true;
size_t sub_id_counter = 0;
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
DeribitChannelDemuxer channel_demuxer;
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
DeribitMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
excluded_secs(
@@ -163,11 +163,15 @@ struct DeribitSubscription::PImpl {
dispatcher,
mk_session_cfg(this->cfg),
{},
{ccapi::BtsVenues::DERIBIT}),
listener(listener) {
{ccapi::BtsVenues::DERIBIT}) {
// DON'T reference the ctr arg 'cfg' (it has been moved to 'this->cfg')
_INFO("ctr. config:\n{}", ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(std::shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request_all();
subscribe_instrument_state();
}
@@ -176,7 +180,7 @@ struct DeribitSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] = security_data_map.try_emplace(
sec_desc, std::make_unique<SecurityData>(sid, listener));
sec_desc, std::make_unique<SecurityData>(sid, listener.get()));
_ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc);
_INFO("add_security_data. sec: '{}'", sec_desc);
return *it->second;
@@ -307,10 +311,8 @@ struct DeribitSubscription::PImpl {
DeribitSubscription::DeribitSubscription(
ccapi::EventDispatcher *dispatcher,
DeribitMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
DeribitSubscription::DeribitSubscription(DeribitSubscription &&) noexcept =
default;
@@ -319,4 +321,8 @@ DeribitSubscription &DeribitSubscription::operator=(
DeribitSubscription::~DeribitSubscription() = default;
void DeribitSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::deribit
@@ -1,5 +1,7 @@
#pragma once
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -24,13 +26,12 @@ namespace bts::md::deribit {
* @brief Subscribes to market data at Deribit
* @a https://docs.deribit.com/?#market-data
*/
class DeribitSubscription {
class DeribitSubscription : public IMarketDataSource {
public:
DeribitSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::DeribitMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
DeribitSubscription(const DeribitSubscription &) noexcept = delete;
DeribitSubscription(DeribitSubscription &&) noexcept;
@@ -40,9 +41,8 @@ class DeribitSubscription {
~DeribitSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
@@ -0,0 +1,78 @@
#include "EmumbaDeribitSubscription.h"
#include "lib/api/md/EmumbaDeribitMarketDataConfig.pb.h"
#include "lib/config/protos/system_config.pb.h"
#include "lib/utils/thread_manager.h"
#include "server/sparkd/md/emumba/EmumbaDeribitData.h"
namespace bts::md {
using bts::config::SystemConfig;
using Path = boost::filesystem::path;
struct EmumbaDeribitSubscription::PImpl {
api::md::EmumbaDeribitMarketDataConfig config;
};
EmumbaDeribitSubscription::EmumbaDeribitSubscription(
api::md::EmumbaDeribitMarketDataConfig config)
: me_(std::make_unique<PImpl>()) {
me_->config = std::move(config);
}
EmumbaDeribitSubscription::~EmumbaDeribitSubscription() = default;
void EmumbaDeribitSubscription::start(
std::shared_ptr<IMarketDataListener> /*listener*/) {
auto &cfg = me_->config;
LOG(md, Info, "EmumbaMarketDataProvider::start t_id:{} t_name:{}. cfg:\n{}",
os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
auto start_deribit_thr = [=]() {
const auto config_path = Path{config::get<SystemConfig>().config_path()} /
cfg.json_conf_file_name();
const auto conf =
bts::cmds::DeribitDemoConfiguration::read_file(config_path.string());
bts_assert_msg(conf.has_value(), "Failed to parse deribit config");
// log
for (const auto &s : conf.value().symbol_config_list) {
LOG(md, Info, "sub.symbol: {}", s.symbol_name);
LOG(md, Info, "sub.futs: {} sub.options: {}", s.interested_in_future,
s.interested_in_option);
for (const auto &i : s.interested_instruments) {
LOG(md, Info, "sub.instrument: {}", i);
}
}
EmumbaDeribitData ddata_obj;
EmumbaDeribitData *ddata = &ddata_obj;
// end log
bts_assert(conf.has_value());
LOG(md, Info, "Deribit conf: {}", conf.value());
ddata->real_handler =
std::make_shared<emumba::deribit_connector::deribit_handler>(
conf.value().symbol_config_list, conf.value().trading_config,
conf.value().credential_map, conf.value().logging_configuration,
conf.value().deribit_url, &ddata->listener);
// since the trading interface is housed inside the EmMDListener, we need to
// create and send emumba's trading_session_interface
for (const auto &p : conf.value().credential_map) {
const auto &sess = p.second;
auto ts_ptr = std::make_shared<
emumba::deribit_connector::trading_session_interface>(
ddata->real_handler->create_new_trading_session(sess));
ddata->trading_session_vec.push_back({sess, ts_ptr});
}
bts_assert(!ddata->trading_session_vec.empty());
ddata->listener.set_handler(
ddata->real_handler.get(), ddata->trading_session_vec);
ddata->real_handler->run();
};
threading::make_thread("Emumba", [&]() { start_deribit_thr(); }).detach();
LOG(md, Info, "MarketDataGateway::_start_deribit ending t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
}
void EmumbaDeribitSubscription::stop() {
//
}
} // namespace bts::md
@@ -0,0 +1,24 @@
#pragma once
#include "lib/api/md/EmumbaDeribitMarketDataConfig.pb.h"
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace bts::md {
class EmumbaDeribitSubscription : public IMarketDataSource {
public:
explicit EmumbaDeribitSubscription(
api::md::EmumbaDeribitMarketDataConfig config);
~EmumbaDeribitSubscription() override;
void start(std::shared_ptr<IMarketDataListener> listener) override;
void stop() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,40 @@
#include "EmumbaMarketDataProvider.h"
#include "lib/api/md/MarketDataConfig.pb.h"
#include "lib/api/md/MarketDataConfigUtils.h"
#include "lib/md/IMarketDataSource.h"
#include "server/sparkd/md/emumba/EmumbaDeribitSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::md::MarketDataConfig;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct EmumbaMarketDataProvider::PImpl {
MarketDataConfig config;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> deribit_subs;
PImpl(MarketDataConfig config) : config(std::move(config)) {}
};
EmumbaMarketDataProvider::EmumbaMarketDataProvider(MarketDataConfig config)
: me_(std::make_unique<PImpl>(std::move(config))) {}
EmumbaMarketDataProvider::~EmumbaMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
EmumbaMarketDataProvider::get_sources() {
for (const auto &cfg : me_->config.emumba_deribit()) {
me_->deribit_subs.push_back(
{api::domain::VENUE_DERIBIT,
std::make_shared<EmumbaDeribitSubscription>(cfg)});
}
return me_->deribit_subs;
}
} // namespace bts::md
@@ -0,0 +1,25 @@
#pragma once
#include "lib/api/md/MarketDataConfig.pb.h"
#include "lib/md/IMarketDataProvider.h"
#include <memory>
namespace bts::md {
class EmumbaMarketDataProvider : public IMarketDataProvider {
public:
EmumbaMarketDataProvider();
explicit EmumbaMarketDataProvider(api::md::MarketDataConfig config);
~EmumbaMarketDataProvider();
std::vector<
std::pair<bts::api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
+18 -12
View File
@@ -39,6 +39,7 @@ using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
using std::unordered_map;
@@ -106,7 +107,7 @@ struct FTXSubscription::PImpl {
FTXSecurityRequestor sec_requestor;
// The session automatically starts/stops in its ctr/dtr
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
// These aliases are useful for quickly switching the handler types (e.g.,
// a special handler that logs messages for testing/diagnostic purposes)
@@ -121,18 +122,21 @@ struct FTXSubscription::PImpl {
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
FTXMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener)
persistence::IDGenerator *sec_id_store)
: sec_id_store(sec_id_store), cfg(std::move(cfg)),
criteria(domain::SecurityCriteria::from_proto(
this->cfg.security_criteria())),
sec_requestor(dispatcher, criteria, sec_id_store),
session(
dispatcher, ccapi::SessionConfigs{}, {}, {ccapi::BtsVenues::FTX}),
listener(listener) {
dispatcher, ccapi::SessionConfigs{}, {}, {ccapi::BtsVenues::FTX}) {
LOG(md, Info, "Initializing FTXSubscription:\n{}",
ProtoPrinter::to_string(this->cfg));
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = std::move(listener);
sec_requestor.request();
}
@@ -140,7 +144,7 @@ struct FTXSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
depth_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert(is_new);
return it->second;
}
@@ -149,7 +153,7 @@ struct FTXSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
ticker_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Ticker handler already created for '{}'", sec_desc);
return it->second;
@@ -159,7 +163,7 @@ struct FTXSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
stats_handlers.try_emplace(sec_desc, sec_id, listener.get());
bts_assert_msgf(
is_new, "Statistics handler already created for '{}'", sec_desc);
return it->second;
@@ -231,10 +235,8 @@ struct FTXSubscription::PImpl {
FTXSubscription::FTXSubscription(
ccapi::EventDispatcher *dispatcher,
FTXMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
FTXSubscription::FTXSubscription(FTXSubscription &&) noexcept = default;
FTXSubscription &FTXSubscription::operator=(FTXSubscription &&) noexcept =
@@ -242,4 +244,8 @@ FTXSubscription &FTXSubscription::operator=(FTXSubscription &&) noexcept =
FTXSubscription::~FTXSubscription() = default;
void FTXSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
} // namespace bts::md::ftx
+7 -4
View File
@@ -1,5 +1,7 @@
#pragma once
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,20 +27,21 @@ namespace bts::md::ftx {
*
* @details Uses ccapi infrastructure to request/subscribe data
*/
class FTXSubscription {
class FTXSubscription : public IMarketDataSource {
public:
FTXSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::FTXMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
FTXSubscription(const FTXSubscription &) noexcept = delete;
FTXSubscription(FTXSubscription &&) noexcept;
FTXSubscription &operator=(const FTXSubscription &) noexcept = delete;
FTXSubscription &operator=(FTXSubscription &&) noexcept;
~FTXSubscription();
~FTXSubscription() override;
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -0,0 +1,41 @@
#include "DatabentoGlbxSubscription.h"
#include "lib/api/md/GlbxMarketDataConfig.pb.h"
#include "lib/utils/thread_manager.h"
#include "server/sparkd/md/databento/DatabentoDataHandler.h"
#include "server/sparkd/md/glbx/GlbxMarketDataHandler.h"
namespace bts::md {
using bts::api::md::GlbxMarketDataConfig;
using bts::persistence::IDGenerator;
using std::shared_ptr;
struct DatabentoGlbxSubscription::PImpl {
GlbxMarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
PImpl(GlbxMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
DatabentoGlbxSubscription::DatabentoGlbxSubscription(
GlbxMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
DatabentoGlbxSubscription::~DatabentoGlbxSubscription() = default;
void DatabentoGlbxSubscription::start(
std::shared_ptr<IMarketDataListener> listener) {
bts::threading::make_thread(
"glbx",
[config = me_->config, sec_id_store = me_->sec_id_store,
listener = std::move(listener)]() {
api::md::glbx::GlbxMarketDatahandler::run(
std::move(listener), std::move(sec_id_store), config);
})
.detach();
// TODO: implement 'stop' (signal the thread above to terminate, then join it).
}
} // namespace bts::md
@@ -0,0 +1,47 @@
#pragma once
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace bts::api::md {
class GlbxMarketDataConfig;
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
/**
* @brief Subscribes to market data at Glbx via Databento
*
* @a https://www.Databento.biz/
*/
class DatabentoGlbxSubscription : public IMarketDataSource {
public:
DatabentoGlbxSubscription(
api::md::GlbxMarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
DatabentoGlbxSubscription(const DatabentoGlbxSubscription &) noexcept =
delete;
DatabentoGlbxSubscription(DatabentoGlbxSubscription &&) noexcept;
DatabentoGlbxSubscription &operator=(
const DatabentoGlbxSubscription &) noexcept = delete;
DatabentoGlbxSubscription &operator=(DatabentoGlbxSubscription &&) noexcept;
~DatabentoGlbxSubscription() override;
/**
* Starts market data subscription on a background thread.
*
* @note Does not block.
*/
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,118 @@
#include "OnixsGlbxSubscription.h"
#include "lib/api/md/GlbxMarketDataConfig.pb.h"
#include "lib/core/LOCK_GUARD.h"
#include "lib/logging/logging.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/SymbolLoader.h"
#include "lib/os/os.h"
#include "lib/persistence/IDGenerator.h"
#include "lib/utils/Dispatcher.h"
#include "server/sparkd/md/platforms/IMDPlatform.h"
#include "server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
#include <atomic>
#include <memory>
#include <thread>
#include <vector>
namespace bts::md {
using bts::api::md::GlbxMarketDataConfig;
using bts::domain::SecurityCriteria;
using bts::md::IMDPlatform;
using bts::persistence::IDGenerator;
using std::shared_ptr;
using std::string;
using std::vector;
struct OnixsGlbxSubscription::PImpl {
GlbxMarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
PImpl(shared_ptr<IDGenerator> sec_id_store, GlbxMarketDataConfig config)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
std::thread dispatcher_thread;
utils::Dispatcher *dispatcher{nullptr};
std::optional<utils::DispatcherTimer> heartbeat_timer;
bool is_running = false;
std::mutex mx;
~PImpl() { stop(); }
/**
* Starts market data subscription on this thread.
*
* @note Designed to be called from a thread callback inside `OnixsGlbxSubscription::start`.
* @note Blocks until subscription stops.
*/
void run() {
{
BTS_LOCK_GUARD(mx);
if (!is_running) {
return;
}
const auto symbols = SymbolLoader::load(
SecurityCriteria::create({api::domain::VENUE_GLBX}));
if (symbols.empty()) {
LOG(md, Info, "OnixsGlbxSubscription aborting. No symbols loaded");
return;
}
LOG(md, Info,
"OnixsGlbxSubscription starting. t_id: {}; t_name: {}; config: "
"{}; symbols: {}",
os::get_thread_id(), os::get_thread_name(), config.DebugString(),
FMT_JOIN(symbols, ", "));
vector<shared_ptr<IMDPlatform>> platforms;
auto main = shared_ptr<MDP3Main>();
main->start({symbols.begin(), symbols.end()}, nullptr, nullptr);
platforms.emplace_back(std::move(main));
dispatcher = &utils::Dispatcher::current();
heartbeat_timer.emplace(*dispatcher, 250, [&]() {
for (auto &p : platforms) {
p->send_heartbeats();
}
});
heartbeat_timer->start();
is_running = true;
}
// This will block until dispatcher->terminate() is called.
// There is a slim, but non-zero chance that this is executed after
// 'stop' is called, since we 'run' outside of the lock.
dispatcher->run();
}
void stop() {
BTS_LOCK_GUARD(mx);
if (!is_running) {
return;
}
is_running = false;
if (dispatcher != nullptr) {
LOG(md, Diag, "OnixsGlbxSubscription stopping.");
dispatcher->io_service()->stop();
}
if (dispatcher_thread.joinable()) {
LOG(md, Diag, "OnixsGlbxSubscription joining dispatcher thread.");
dispatcher_thread.join();
}
}
};
OnixsGlbxSubscription::OnixsGlbxSubscription(
shared_ptr<IDGenerator> sec_id_store, GlbxMarketDataConfig config)
: me_(std::make_unique<PImpl>(std::move(sec_id_store), std::move(config))) {
}
OnixsGlbxSubscription::~OnixsGlbxSubscription() = default;
void OnixsGlbxSubscription::start(
shared_ptr<IMarketDataListener> /*listener*/) {
std::atomic_thread_fence(std::memory_order_release);
bts_assert(!me_->dispatcher_thread.joinable());
me_->dispatcher_thread = std::thread{[&]() { me_->run(); }};
}
void OnixsGlbxSubscription::stop() { me_->stop(); }
} // namespace bts::md
@@ -0,0 +1,48 @@
#pragma once
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace bts::api::md {
class GlbxMarketDataConfig;
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
/**
* @brief Subscribes to market data at Glbx via Onixs
*
* @a https://www.onixs.biz/
*/
class OnixsGlbxSubscription : public IMarketDataSource {
public:
OnixsGlbxSubscription(
std::shared_ptr<persistence::IDGenerator> sec_id_store,
api::md::GlbxMarketDataConfig config);
OnixsGlbxSubscription(const OnixsGlbxSubscription &) noexcept = delete;
OnixsGlbxSubscription(OnixsGlbxSubscription &&) noexcept;
OnixsGlbxSubscription &operator=(const OnixsGlbxSubscription &) noexcept =
delete;
OnixsGlbxSubscription &operator=(OnixsGlbxSubscription &&) noexcept;
~OnixsGlbxSubscription() override;
/**
* Starts market data subscription on a background thread.
*
* @note Does not block.
*/
void start(std::shared_ptr<IMarketDataListener> listener) override;
void stop() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
+17 -10
View File
@@ -47,6 +47,7 @@ using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::persistence::IDGenerator;
using std::optional;
using std::shared_ptr;
using std::size_t;
using std::string;
using std::unique_ptr;
@@ -137,14 +138,13 @@ struct OkxSubscription::PImpl {
const bool subscribe_trades = true;
size_t sub_id_counter = 0;
CCApiSession session;
IMarketDataListener *listener;
shared_ptr<IMarketDataListener> listener;
OkxChannelDemuxer channel_demuxer;
explicit PImpl(
ccapi::EventDispatcher *dispatcher,
OkxMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
IDGenerator *sec_id_store)
: cfg(std::move(cfg)),
criteria(SecurityCriteria::from_proto(this->cfg.security_criteria())),
excluded_secs(
@@ -161,10 +161,14 @@ struct OkxSubscription::PImpl {
dispatcher,
mk_session_cfg(this->cfg),
mk_session_opts(this->cfg),
{ccapi::BtsVenues::OKX}),
listener(listener) {
{ccapi::BtsVenues::OKX}) {
// DON'T reference the ctr arg 'cfg' (it has been moved to 'this->cfg')
_INFO("ctr. config:\n{}", ProtoPrinter::to_string(this->cfg));
}
void start(std::shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener = listener;
sec_sub.emplace(
&session, &instrument_store, this->criteria,
[this](const auto &secs) { on_securities(secs); });
@@ -174,7 +178,7 @@ struct OkxSubscription::PImpl {
SecurityType sec_type, const string &sec_desc) {
const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] = security_data_map.try_emplace(
sec_desc, std::make_unique<SecurityData>(sid, listener));
sec_desc, std::make_unique<SecurityData>(sid, listener.get()));
_ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc);
_INFO("add_security_data. sec: '{}'", sec_desc);
return *it->second;
@@ -257,10 +261,8 @@ struct OkxSubscription::PImpl {
OkxSubscription::OkxSubscription(
ccapi::EventDispatcher *dispatcher,
OkxMarketDataConfig cfg,
IDGenerator *sec_id_store,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>(
dispatcher, std::move(cfg), sec_id_store, listener)) {}
IDGenerator *sec_id_store)
: me_(std::make_unique<PImpl>(dispatcher, std::move(cfg), sec_id_store)) {}
OkxSubscription::OkxSubscription(OkxSubscription &&) noexcept = default;
OkxSubscription &OkxSubscription::operator=(OkxSubscription &&) noexcept =
@@ -268,4 +270,9 @@ OkxSubscription &OkxSubscription::operator=(OkxSubscription &&) noexcept =
OkxSubscription::~OkxSubscription() = default;
void OkxSubscription::start(
std::shared_ptr<bts::md::IMarketDataListener> listener) {
me_->start(listener);
}
} // namespace bts::md::okx
+7 -8
View File
@@ -1,5 +1,7 @@
#pragma once
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace ccapi {
@@ -25,25 +27,22 @@ namespace bts::md::okx {
*
* @a https://www.okx.com/docs-v5/en/#websocket-api-public-channel
*/
class OkxSubscription {
class OkxSubscription : public IMarketDataSource {
public:
OkxSubscription(
ccapi::EventDispatcher *dispatcher,
api::md::OkxMarketDataConfig cfg,
persistence::IDGenerator *sec_id_store,
IMarketDataListener *listener);
persistence::IDGenerator *sec_id_store);
OkxSubscription(const OkxSubscription &) noexcept = delete;
OkxSubscription(OkxSubscription &&) noexcept;
OkxSubscription &operator=(const OkxSubscription &) noexcept =
delete;
OkxSubscription &operator=(const OkxSubscription &) noexcept = delete;
OkxSubscription &operator=(OkxSubscription &&) noexcept;
~OkxSubscription();
void add_listener(IMarketDataListener *listener);
void remove_listener(IMarketDataListener *listener);
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
+20 -30
View File
@@ -1,10 +1,10 @@
#include "MDReplayer.h"
#include "lib/core/TimeStamp.h"
#include "lib/domain/Book.h"
#include "lib/domain/DomainSecurityUtils.h"
#include "lib/domain/Price.h"
#include "lib/domain/Quantity.h"
#include "lib/domain/Security.h"
#include "lib/domain/SecurityCommon.h"
#include "lib/domain/SecurityID.h"
#include "lib/domain/SecurityMarketDataStatistics.h"
#include "lib/domain/Side.h"
@@ -12,10 +12,11 @@
#include "lib/domain_model/Security.pb.h"
#include "lib/logging/logging.h"
#include "lib/md/GeneralMarketDataListener.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/MDProducer.h"
#include "lib/md/Shm/FixedSecurity.h"
#include "lib/md/protos/co2.pb.h"
#include "lib/os/os.h"
#include "lib/persistence/IDGenerator.h"
#include "lib/utils/PBUtils.h"
#include "lib/utils/bts_assert.h"
#include "server/sparkd/md/recording/MDRecorder.h"
@@ -55,38 +56,27 @@ struct MDReplayer ::PImpl {
std::map<SecurityKey, co2::TotalBook> books = {};
bts::api::Venue venue;
std::set<uint32_t> sec_keys = {};
int64_t start_time = bts::core::TimeStamp::now_ns();
GeneralMarketDataListener listener;
uint64_t start_time = 0;
std::shared_ptr<IMarketDataListener> listener{nullptr};
bool publish_boostrap_books_only;
double replay_speed;
PImpl(
bts::api::Venue venue,
bts::md::GeneralSecurityIndex &index,
bts::md::MDProducer &md_producer,
bool publish_boostrap_books_only_,
double replay_speed)
: venue(venue), listener(index, md_producer, venue),
publish_boostrap_books_only(publish_boostrap_books_only_),
replay_speed(replay_speed) {}
PImpl(bts::api::Venue venue, double replay_speed)
: venue(venue), replay_speed(replay_speed) {}
};
MDReplayer::MDReplayer(
bts::api::Venue venue,
bts::md::GeneralSecurityIndex &index,
bts::md::MDProducer &md_producer,
bool publish_boostrap_books_only,
double replay_speed)
: me(new PImpl(
venue,
index,
md_producer,
publish_boostrap_books_only,
replay_speed)) {
MDReplayer::MDReplayer(bts::api::Venue venue, double replay_speed)
: me(new PImpl(venue, replay_speed)) {
LOG(md, Info, "MDReplayer::MDReplayer venue: {}", Venue_Name(venue));
}
MDReplayer::~MDReplayer() {}
void MDReplayer::run() {
void MDReplayer::start(std::shared_ptr<IMarketDataListener> listener) {
LOG(md, Info, "MDReplayer::start venue: {}", Venue_Name(me->venue));
bts_assert(listener != nullptr);
me->listener = std::move(listener);
me->start_time = bts::core::TimeStamp::now_ns();
std::string directory = MDRecorder::recording_path();
auto include_file = [&](const std::string &file_name) {
return MDRecorder::is_md_recording_file(file_name, me->venue);
@@ -179,7 +169,7 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) {
me->sec_keys.insert(m.security().id());
std::vector<domain::Security const *> sec_vec = {};
sec_vec.push_back(&new_sec);
me->listener.on_securities(sec_vec);
me->listener->on_securities(sec_vec);
break;
}
case (co2::RecordedMDMessage::MDRecordCase::kTotalBook): {
@@ -194,19 +184,19 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) {
bts::domain::Book book = {};
copy_books(tot_book.direct_book(), book);
book.set_security_id(bts::domain::SecurityID(m.sec_key()));
me->listener.on_book(book);
me->listener->on_book(book);
}
break;
}
case (co2::RecordedMDMessage::MDRecordCase::kMisc): {
bts::domain::SecurityMarketDataStatistics stats(m.sec_key(), m.misc());
me->listener.on_security_statistics(stats);
me->listener->on_security_statistics(stats);
break;
}
case (co2::RecordedMDMessage::MDRecordCase::kTrade): {
domain::TradeRecord tr;
domain::DomainSecurityUtils::convert(m.trade(), tr);
me->listener.on_trades({&tr});
me->listener->on_trades({&tr});
break;
}
default: {
+7 -10
View File
@@ -1,7 +1,8 @@
#pragma once
#include "lib/domain_model/Security.pb.h"
#include "lib/md/GeneralSecurityIndex.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/IMarketDataSource.h"
namespace bts::md {
class MDProducer;
@@ -14,16 +15,12 @@ class RecordedMDMessage;
namespace bts::md::recording {
class MDReplayer {
class MDReplayer : public IMarketDataSource {
public:
MDReplayer(
bts::api::Venue venue,
bts::md::GeneralSecurityIndex &index,
bts::md::MDProducer &md_producer,
bool publish_boostrap_books_only,
double replay_speed = 1);
~MDReplayer();
void run();
explicit MDReplayer(bts::api::Venue venue, double replay_speed = 1);
~MDReplayer() override;
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
@@ -0,0 +1,47 @@
#include "TaifexMarketDataProvider.h"
#include "lib/api/domain/Venue.pb.h"
#include "lib/api/md/MarketDataConfig.pb.h"
#include "lib/persistence/IDGenerator.h"
#include "server/sparkd/md/taifex/TaifexSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::domain::Venue;
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct TaifexMarketDataProvider::PImpl {
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
vector<shared_ptr<TaifexSubscription>> sources;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
TaifexMarketDataProvider::TaifexMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
TaifexMarketDataProvider::~TaifexMarketDataProvider() = default;
vector<pair<Venue, shared_ptr<IMarketDataSource>>>
TaifexMarketDataProvider::get_sources() {
vector<pair<Venue, shared_ptr<IMarketDataSource>>> rv;
for (const auto &taifex : me_->config.taifex()) {
me_->sources.push_back(
std::make_shared<TaifexSubscription>(taifex, me_->sec_id_store));
rv.push_back({Venue::VENUE_TAIFEX, me_->sources.back()});
}
return rv;
}
} // namespace bts::md
@@ -0,0 +1,33 @@
#pragma once
#include "lib/md/IMarketDataProvider.h"
#include <memory>
namespace bts::persistence {
class IDGenerator;
}
namespace bts::api::md {
class MarketDataConfig;
}
namespace bts::md {
class TaifexMarketDataProvider : public IMarketDataProvider {
public:
explicit TaifexMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~TaifexMarketDataProvider();
std::vector<
std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,40 @@
#include "TaifexSubscription.h"
#include "lib/api/md/TaifexMarketDataConfig.pb.h"
#include "lib/utils/thread_manager.h"
#include "server/sparkd/md/taifex/TaifexDataGateway.h"
namespace bts::md {
using bts::api::md::TaifexMarketDataConfig;
using bts::md::taifex::TaifexDataGateway;
using bts::persistence::IDGenerator;
using std::shared_ptr;
struct TaifexSubscription::PImpl {
TaifexMarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
PImpl(TaifexMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
TaifexSubscription::TaifexSubscription(
TaifexMarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
TaifexSubscription::~TaifexSubscription() = default;
void TaifexSubscription::start(shared_ptr<IMarketDataListener> listener) {
bts::threading::make_thread(
"taifex",
[config = me_->config, sec_id_store = me_->sec_id_store,
listener = std::move(listener)]() {
TaifexDataGateway::run(
config, std::move(listener), std::move(sec_id_store));
})
.detach();
// TODO: implement 'stop' (signal the thread above to terminate, then join it).
}
} // namespace bts::md
@@ -0,0 +1,43 @@
#pragma once
#include "lib/md/IMarketDataSource.h"
#include <memory>
namespace bts::api::md {
class TaifexMarketDataConfig;
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
/**
* @brief Subscribes to market data at Taifex
*/
class TaifexSubscription : public IMarketDataSource {
public:
TaifexSubscription(
api::md::TaifexMarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
TaifexSubscription(const TaifexSubscription &) noexcept = delete;
TaifexSubscription(TaifexSubscription &&) noexcept;
TaifexSubscription &operator=(const TaifexSubscription &) noexcept = delete;
TaifexSubscription &operator=(TaifexSubscription &&) noexcept;
~TaifexSubscription() override;
/**
* Starts market data subscription on a background thread.
*
* @note Does not block.
*/
void start(std::shared_ptr<IMarketDataListener> listener) override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md