#include "DatabentoDataHandler.h" #include "databento/datetime.hpp" #include "databento/timeseries.hpp" #include "domain/book.h" #include "domain/quantity.h" #include "domain/security.h" #include "domain/security_id.h" #include "spark/config/config.h" #include "spark/config/protos/system_config.pb.h" #include "spark/md/IMarketDataListener.h" #include "spark/persistence/IDGenerator.h" #include "spark/utils/TimeStamp.h" #include "spark/utils/bts_assert.h" #include "spark/utils/bts_chrono.h" #include #include #include #include #include namespace bts::md::DatabentoHandler { using bts::domain::protobufs::Venue; std::string remove_spaces(const std::string_view &str) { std::string result; result.reserve(str.size()); // Reserve space for better performance for (char c : str) { if (!isspace(c)) { result += c; } } return result; } void DatabentoDatahandler::run( bool use_file, std::shared_ptr listener, std::shared_ptr sec_id_store, Venue venue, const std::string &securities_path, const std::string &books_path, const std::chrono::duration sleep_duration) { auto client = databento::HistoricalBuilder{} .SetKey("db-Q4eCbTQfrAHaG6sAcuFUYB9YqkF6j") .Build(); auto process_security = [&](const databento::Record &record) { const auto &msg = record.Get(); std::string id = remove_spaces(msg.RawSymbol()); std::string symbol; switch (venue) { case Venue::VENUE_CBOE: symbol = id.substr(0, 3); break; case Venue::VENUE_GLBX: if (id.length() > 5) return databento::KeepGoing::Continue; symbol = id.substr(0, 2); break; default: bts_assert_msg(false, "Exchange not supported on databento"); return databento::KeepGoing::Stop; } bts::domain::SecurityID sec_id{sec_id_store->get_or_add( venue, std::to_string(msg.hd.instrument_id))}; bts::domain::Security sec; sec.security_common = domain::SecurityCommon(); sec.security_common.venue = venue; sec.security_common.id = sec_id; sec.security_common.exchange_sec_id = msg.hd.instrument_id; sec.security_common.desc = msg.RawSymbol(); sec.security_common.symbol = symbol; sec.security_common.asset = symbol; sec.security_common.group = symbol; sec.security_common.min_price_increment = 0.01; sec.security_common.min_quantity = 0.01; sec.security_common.min_quantity_increment = 0.01; sec.security_common.currency = {msg.currency.data(), msg.currency.size()}; const std::string_view sec_type = { msg.security_type.data(), msg.security_type.size()}; const auto exp_nanos = msg.expiration; std::string exp_string = databento::ToIso8601(exp_nanos); const auto expiration_time = bts::core::TimeStamp::from_iso_string(exp_string); if (sec_type.find("OPT") != std::string::npos) { sec.security.emplace( expiration_time, bts::domain::Price::from_double(msg.strike_price)); } else if (sec_type.find("FUT") != std::string::npos) { sec.security.emplace(expiration_time); } else { bts_assert_msg( false, "Only futures and options supported on databento"); return databento::KeepGoing::Stop; } listener->on_securities({&sec}); return databento::KeepGoing::Continue; }; auto process_bbo = [&](const databento::Record &record) { const auto &msg = record.Get(); auto sec_id_opt = sec_id_store->get_id( venue, std::to_string(msg.hd.instrument_id), false); if (!sec_id_opt.has_value()) return databento::KeepGoing::Continue; sleep_for(sleep_duration); bts::domain::SecurityID sec_id = bts::domain::SecurityID(sec_id_opt.value()); domain::Book book; book.set_security_id(sec_id); auto &bids = book.bids(); auto &asks = book.asks(); bids.begin()->price = bts::domain::Price(msg.levels[0].bid_px, -9); bids.begin()->total_quantity = bts::domain::Quantity(msg.levels[0].bid_sz); asks.begin()->price = bts::domain::Price(msg.levels[0].ask_px, -9); asks.begin()->total_quantity = bts::domain::Quantity(msg.levels[0].ask_sz); auto recv_time = bts::core::TimeStamp::from_iso_string( databento::ToIso8601(msg.ts_recv)); listener->on_book( book, utils::TimeStamp{datetime::ptime_to_timestamp(recv_time.to_ptime())}); return databento::KeepGoing::Continue; }; if (use_file) { std::string base_path = config::get().data_path(); databento::DbnFileStore dbn_file_store(securities_path); dbn_file_store.Replay(process_security); databento::DbnFileStore dbn_file_store2(books_path); dbn_file_store2.Replay(process_bbo); } } } // namespace bts::md::DatabentoHandler