#include "DatabentoDataHandler.h" #include "databento/datetime.hpp" #include "databento/dbn_encoder.hpp" #include "databento/enums.hpp" #include "databento/exceptions.hpp" #include "databento/live_blocking.hpp" #include "databento/log.hpp" #include "databento/record.hpp" #include "databento/timeseries.hpp" #include "spark/api/domain/CallPut.pb.h" #include "spark/api/domain/Venue.pb.h" #include "spark/api/md/MarketDataConfig.pb.h" #include "spark/config/config.h" #include "spark/config/protos/system_config.pb.h" #include "spark/domain/AggressorSide.h" #include "spark/domain/Book.h" #include "spark/domain/Quantity.h" #include "spark/domain/Security.h" #include "spark/domain/SecurityID.h" #include "spark/domain/Side.h" #include "spark/domain/TradeRecord.h" #include "spark/domain_model/Security.pb.h" #include "spark/logging/logging.h" #include "spark/md/IMarketDataListener.h" #include "spark/persistence/IDGenerator.h" #include "spark/core/TimeStamp.h" #include "spark/utils/TimeStamp.h" #include "spark/utils/bts_assert.h" #include "spark/utils/bts_chrono.h" #include "spark/utils/bts_assert.h" #include "spark/utils/datetime.h" #include #include #include #include #include #include #include #include #include #include #include #include #include namespace bts::md::DatabentoHandler { #define SEC_TYPE(name) bts::api::domain::SECURITY_TYPE_##name using bts::api::domain::CallPut; using bts::api::domain::SecurityType; using bts::api::domain::Venue; using bts::domain::SecurityID; std::string remove_spaces(const std::string_view &str) { std::string result; result.reserve(str.size()); // Reserve space for better performance for (char c : str) { if (!isspace(c)) { result += c; } } return result; } static SecurityType parse_sec_type(std::string_view str) { if (str.find("OPT") != std::string::npos || str.find("OOF") != std::string::npos) { return SEC_TYPE(OPTION); } if (str.find("FUT") != std::string::npos) { return SEC_TYPE(FUTURE); } return SEC_TYPE(UNSPECIFIED); } void update_file( const std::string &key, const std::string &dataset, const std::vector &parent_symbols, const uint time_to_run, std::string file_path) { auto client = databento::LiveBuilder{} .SetKey(key) .SetDataset(dataset) .SetUpgradePolicy(databento::VersionUpgradePolicy::UpgradeToV2) .BuildBlocking(); client.Subscribe( parent_symbols, databento::Schema::Definition, databento::SType::Parent, databento::UnixNanos{}); // We start streaming databento::Metadata metadata = client.Start(); { // We open a file for writing and encode the metadata databento::OutFileStream out_file{file_path}; databento::DbnEncoder encoder{metadata, &out_file}; // Wait for at least one record and 5 seconds before closing the connection const std::chrono::seconds timeout{time_to_run}; const auto start = std::chrono::steady_clock::now(); while (auto *record = client.NextRecord(timeout)) { encoder.EncodeRecord(*record); if (std::chrono::steady_clock::now() - start >= timeout) { break; } } } return; } void log_time_statistics( const std::vector>> &all_times) { std::unordered_map> subpart_times; // Collect all timing values per subpart for (const auto &entry : all_times) { for (const auto &[time, label] : entry) { subpart_times[label].push_back(time); } } // Function to compute statistics auto compute_statistics = [](std::vector &values) -> std::tuple { if (values.empty()) return std::make_tuple(0.0, 0.0, 0, 0, 0, 0, 0); std::sort(values.begin(), values.end()); double sum = std::accumulate(values.begin(), values.end(), 0.0); double avg = sum / values.size(); double sq_sum = std::inner_product(values.begin(), values.end(), values.begin(), 0.0); double stddev = std::sqrt(sq_sum / values.size() - avg * avg); auto percentile = [&](double p) -> uint64_t { size_t idx = static_cast(p * (values.size() - 1)); return values[idx]; }; return std::make_tuple( avg, stddev, percentile(0.00), percentile(0.25), percentile(0.50), percentile(0.75), percentile(1.00)); }; // Compute and log statistics for (auto &[label, times] : subpart_times) { auto [avg, stddev, p0, p25, p50, p75, p100] = compute_statistics(times); LOG(md, Sys, "Subpart: {}, Avg: {:.2f} ns, Std: {:.2f} ns, P0: {} ns, P25: {} ns, " "P50: {} ns, P75: {} ns, P100: {} ns", label, avg, stddev, p0, p25, p50, p75, p100); } } void DatabentoDatahandler::run( bool use_file, std::shared_ptr listener, std::shared_ptr sec_id_store, bts::api::domain::Venue venue, const std::string &securities_path, const std::string &books_path, const std::string &key, const std::string &dataset, const uint filter_securities_upto_n_months, const bool load_reference_data, const std::chrono::duration sleep_duration) { uint DEBUG_SEC_ID = 0; if (venue == api::domain::VENUE_CBOE) DEBUG_SEC_ID = config::get() .cboe() .at(0) .databento_settings() .debug_sec(); auto sleep_duration_updated = sleep_duration; std::vector>> all_times; std::unordered_map sec_id_map; if (!use_file) { sleep_duration_updated = std::chrono::duration< long double>::zero(); // Set to zero for live data } auto sec_filter = [&](const core::TimeStamp expiration_time, const std::string &symbol, const api::domain::Venue &venue, uint strike) { uint months = filter_securities_upto_n_months; switch (venue) { case api::domain::Venue::VENUE_CBOE: { if (strike < 3800 || strike > 8000) { return false; } if (symbol == "SPX") months = 6; else if (symbol == "SPXW") months = 1; break; } case api::domain::Venue::VENUE_GLBX: return true; default: bts_assert_msg(false, "Exchange not supported on databento"); } auto duration = months * 32 * 24 * 60 * 60; return expiration_time.seconds() < bts::core::TimeStamp::now().seconds() + duration; }; auto process_security_recorded = [&](const databento::Record &record) { const auto &msg = record.Get(); const auto exp_nanos = msg.expiration; std::string exp_string = databento::ToIso8601(exp_nanos); const auto expiration_time = bts::core::TimeStamp::from_iso_string(exp_string); if (!sec_filter( expiration_time, msg.Underlying(), venue, msg.strike_price / 1000000000)) { return databento::KeepGoing::Continue; } bts::domain::Security sec; sec.security_common = domain::SecurityCommon(); sec.security_common.venue = venue; sec.security_common.exchange_sec_id = msg.hd.instrument_id; sec.security_common.desc = msg.RawSymbol(); sec.security_common.symbol = msg.Asset(); sec.security_common.asset = msg.Asset(); sec.security_common.group = msg.Group(); sec.security_common.min_price_increment = 0.01; sec.security_common.min_quantity = 0.01; sec.security_common.min_quantity_increment = 0.01; sec.security_common.currency = {msg.currency.data(), msg.currency.size()}; const SecurityType sec_type = parse_sec_type({msg.security_type.data(), msg.security_type.size()}); switch (sec_type) { case SEC_TYPE(FUTURE): sec.security.emplace(expiration_time); break; case SEC_TYPE(OPTION): { if (sec.venue() == api::domain::VENUE_CBOE) { if (sec.security_common.symbol == "SPX" || sec.security_common.symbol == "SPXW") { sec.security_common.min_quantity_increment = 1; sec.security_common.min_quantity = 1; sec.security_common.price_ticks.push_back(3.0); sec.security_common.price_ticks.push_back(0.05); sec.security_common.price_ticks.push_back(0.1); sec.security_common.cash_per_point = 100; } } if (sec.venue() == api::domain::VENUE_GLBX) { sec.security_common.min_quantity_increment = 1; sec.security_common.min_quantity = 1; sec.security_common.price_ticks.push_back(10); sec.security_common.price_ticks.push_back(0.05); sec.security_common.price_ticks.push_back(0.25); sec.security_common.cash_per_point = 50; } if (msg.instrument_class != databento::instrument_class::Call && msg.instrument_class != databento::instrument_class::Put){ return databento::KeepGoing::Continue; } auto call_put = (msg.instrument_class == databento::instrument_class::Call)? CallPut::CALL : CallPut::PUT; sec.security.emplace( expiration_time, bts::domain::Price(msg.strike_price, -9), call_put); break; } default: LOG(md, Info, "skipping sec as only options and futures supported, {}",sec.security_common.desc); return databento::KeepGoing::Continue; // bts_assert_msg( // false, "Only futures and options supported on databento"); } *sec.security_common.id = sec_id_store->get_or_add( venue, sec_type, std::to_string(msg.hd.instrument_id)); listener->on_securities({&sec}); sec_id_map.insert({msg.hd.instrument_id, sec.security_common.id}); return databento::KeepGoing::Continue; }; auto process_security = [&](const databento::Record &record) -> bool { const auto &msg = record.Get(); const auto exp_nanos = msg.expiration; std::string exp_string = databento::ToIso8601(exp_nanos); const auto expiration_time = bts::core::TimeStamp::from_iso_string(exp_string); if (!sec_filter( expiration_time, msg.Underlying(), venue, msg.strike_price / 1000000000)) { return false; } bts::domain::Security sec; sec.security_common = domain::SecurityCommon(); sec.security_common.venue = venue; sec.security_common.exchange_sec_id = msg.hd.instrument_id; sec.security_common.desc = msg.RawSymbol(); sec.security_common.symbol = msg.Underlying(); sec.security_common.asset = msg.Asset(); sec.security_common.group = msg.Group(); sec.security_common.min_price_increment = 0.01; sec.security_common.min_quantity = 0.01; sec.security_common.min_quantity_increment = 0.01; sec.security_common.currency = {msg.currency.data(), msg.currency.size()}; const SecurityType sec_type = parse_sec_type({msg.security_type.data(), msg.security_type.size()}); switch (sec_type) { case SEC_TYPE(FUTURE): sec.security.emplace(expiration_time); break; case SEC_TYPE(OPTION): { if (sec.venue() == api::domain::VENUE_CBOE) { if (sec.security_common.symbol == "SPX" || sec.security_common.symbol == "SPXW") { sec.security_common.min_quantity_increment = 1; sec.security_common.min_quantity = 1; sec.security_common.price_ticks.push_back(3.0); sec.security_common.price_ticks.push_back(0.05); sec.security_common.price_ticks.push_back(0.1); sec.security_common.cash_per_point = 100; } } if (sec.venue() == api::domain::VENUE_GLBX) { sec.security_common.min_quantity_increment = 1; sec.security_common.min_quantity = 1; sec.security_common.price_ticks.push_back(10); sec.security_common.price_ticks.push_back(0.05); sec.security_common.price_ticks.push_back(0.25); sec.security_common.cash_per_point = 50; } auto call_put = CallPut::PUT; if (msg.instrument_class == 'C') call_put = CallPut::CALL; else bts_assert(msg.instrument_class == 'P'); sec.security.emplace( expiration_time, bts::domain::Price(msg.strike_price, -9), call_put); break; } default: bts_assert_msg( false, "Only futures and options supported on databento"); } *sec.security_common.id = sec_id_store->get_or_add( venue, sec_type, std::to_string(msg.hd.instrument_id)); listener->on_securities({&sec}); sec_id_map.insert({msg.hd.instrument_id, sec.security_common.id}); return true; }; core::TimeStamp last_time; long i = 0; auto debug_log = [&](const databento::Mbp1Msg &msg) { if (DEBUG_SEC_ID != msg.hd.instrument_id) return; const double denom = 1000000000; // auto delta = std::to_string(std::chrono::duration(msg.ts_in_delta).count()); auto seq_num = msg.sequence; auto ap = msg.levels[0].ask_px / denom; auto bp = msg.levels[0].bid_px / denom; auto as = msg.levels[0].ask_sz; auto bs = msg.levels[0].bid_sz; LOG(md, Sys, "ID: {}, seq_num: {}, B_P: {}, B_S: {}, A_P: {}, A_S: {}", msg.hd.instrument_id, seq_num, bp, bs, ap, as); }; auto process_mbp1 = [&](const databento::Record &record) { const auto &msg = record.Get(); debug_log(msg); if(i++==0){ last_time=core::TimeStamp::from_int64(msg.ts_recv.time_since_epoch().count()); }else{ auto new_time = core::TimeStamp::from_int64(msg.ts_recv.time_since_epoch().count()); auto dif = new_time-last_time; if(dif.nanos()>0){ // LOG(md,Info,"sleeping for {}",dif.nanos()); sleep_for(std::chrono::nanoseconds(dif.nanos())); } last_time = new_time; } // std::vector>& times = all_times.emplace_back(); // times.reserve(8); // return databento::KeepGoing::Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version // The security type to is needed to get the id. Since the sec type isn't known, // check for an id using both types (OPTION and FUTURE). This is safe to do since // options and futures never have common names (instrument ids). // auto current = bts::datetime::unix_nano_now(); // auto prev = current; // times.push_back({current-prev,"Start Process bbo"}); // prev = current; // current = bts::datetime::unix_nano_now(); // times.push_back({current-prev,"MISC to check log time"}); if (!sec_id_map.contains(msg.hd.instrument_id)) return databento::KeepGoing::Continue; bts::domain::SecurityID sec_id = sec_id_map.at(msg.hd.instrument_id); if(msg.action == databento::action::Trade){ domain::TradeRecord trade; trade.trade_price = bts::domain::Price(msg.price, -9); trade.trade_quantity = bts::domain::Quantity(msg.size); trade.sec_id = sec_id; switch (msg.side) { case databento::Side::Ask: trade.agg_side = domain::AggressorSide::Sell; break; case databento::Side::Bid: trade.agg_side = domain::AggressorSide::Buy; break; case databento::Side::None: trade.agg_side = domain::AggressorSide::None; break; } trade.time_stamp = core::TimeStamp::now(); //bts::core::TimeStamp::from_iso_string( // databento::ToIso8601(msg.ts_recv)); std::vector trade_ptrs; trade_ptrs.push_back(&trade); listener->on_trades(trade_ptrs); return databento::KeepGoing::Continue; } // prev = current; // current = bts::datetime::unix_nano_now(); // times.push_back({current-prev,"Sec_ID done"}); // sleep_for(sleep_duration_updated); // prev = current; // current = bts::datetime::unix_nano_now(); // times.push_back({current-prev,"Sleep done"}); domain::Book book; book.set_security_id(sec_id); // prev = current; // current = bts::datetime::unix_nano_now(); // times.push_back({current-prev,"Empty Book Made"}); auto &bids = book.bids(); auto &asks = book.asks(); bids.begin()->price = bts::domain::Price(msg.levels[0].bid_px, -9); bids.begin()->total_quantity = bts::domain::Quantity(msg.levels[0].bid_sz); asks.begin()->price = bts::domain::Price(msg.levels[0].ask_px, -9); asks.begin()->total_quantity = bts::domain::Quantity(msg.levels[0].ask_sz); // prev = current; // current = bts::datetime::unix_nano_now(); // times.push_back({current-prev,"Complete book done"}); //this is correct // auto t2 = datetime::parse_iso_date_time_string(databento::ToIso8601(msg.ts_recv)); // auto t3 = core::TimeStamp::from_int64(msg.ts_recv.time_since_epoch().count()); // auto now_ts = utils::TimeStamp::now(); // auto now = datetime::now(); // auto delta = (now - t).total_milliseconds(); // return now_ts.shift_ms(delta); //this is wrong // auto t3 = utils::TimeStamp(msg.ts_recv.time_since_epoch().count()); //this is also // auto t4 =datetime::ptime_to_timestamp(t3.to_ptime()); // LOG(md,Info,"b01 {}",databento::ToIso8601(msg.ts_recv)); // LOG(md,Info,"b02 {}",datetime::iso_date_time_string(t3.to_ptime())); // LOG(md,Info,"b03: {}",t3.to_iso_string()); // LOG(md,Info,"b04: {}",t4.to_uint64()); // LOG(md,Info,"b041: {}",t4.to_int64()); // LOG(md,Info,"b042: {}",utils::TimeStamp(msg.ts_recv.time_since_epoch().count()).to_int64()); // LOG(md,Info,"b05: {}",t3.nanos()); // LOG(md,Info,"b06: {}",msg.ts_recv.time_since_epoch().count()); // LOG(md,Info,"b07: {}",utils::TimeStamp::now().to_int64()); listener->on_book( book, //utils::TimeStamp(msg.ts_recv.time_since_epoch().count()) utils::TimeStamp::now() ); // prev = current; // current = bts::datetime::unix_nano_now(); // times.push_back({current-prev,"On book done"}); return databento::KeepGoing:: Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version }; auto process_trades = [&](const databento::Record &record) { const auto &msg = record.Get(); // The security type to is needed to get the id. Since the sec type isn't known, // check for an id using both types (OPTION and FUTURE). This is safe to do since // options and futures never have common names (instrument ids). if (!sec_id_map.contains(msg.hd.instrument_id)) return databento::KeepGoing::Continue; bts::domain::SecurityID sec_id = sec_id_map.at(msg.hd.instrument_id); // sleep_for(sleep_duration_updated); domain::TradeRecord trade; trade.trade_price = bts::domain::Price(msg.price, -9); trade.trade_quantity = bts::domain::Quantity(msg.size); trade.sec_id = sec_id; switch (msg.side) { case databento::Side::Ask: trade.agg_side = domain::AggressorSide::Sell; break; case databento::Side::Bid: trade.agg_side = domain::AggressorSide::Buy; break; case databento::Side::None: trade.agg_side = domain::AggressorSide::None; break; } trade.time_stamp = bts::core::TimeStamp::from_iso_string( databento::ToIso8601(msg.ts_recv)); std::vector trade_ptrs; trade_ptrs.push_back(&trade); listener->on_trades(trade_ptrs); return databento::KeepGoing:: Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version }; if (use_file) { auto client = databento::HistoricalBuilder{}.SetKey(key).Build(); databento::DbnFileStore dbn_file_store(securities_path); dbn_file_store.Replay(process_security_recorded); // fix this LOG(md,Info,"done secs"); databento::DbnFileStore dbn_file_store2(books_path); dbn_file_store2.Replay(process_mbp1); LOG(md,Info,"done bbo"); sleep(1000); } else { bts_assert(dataset != ""); //hardcoding for now will update later std::vector symbols; switch (venue) { case api::domain::VENUE_CBOE: symbols = {"SPX.OPT", "SPXW.OPT"}; break; case api::domain::VENUE_GLBX: symbols = {"ES.FUT"}; break; default: bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW"); } //ocassionaly run security writer to update security dbn if (!boost::filesystem::exists(securities_path) || load_reference_data) { update_file(key, dataset, symbols, 15 * 60, securities_path); } //regular flow //read from security dbn, filter contracts that we need and subscribe to those in batches of two thousand LOG(md, Info, "starting loading symbols"); databento::DbnFileStore store{securities_path}; std::vector> all_sec; all_sec.emplace_back(); all_sec.back().reserve(2000); size_t total_count = 0; while (auto *record = store.NextRecord()) { if (auto *def = record->GetIf()) { std::string s = def->RawSymbol(); if (process_security(*record)) { ++total_count; // Check if the last vector is full if (all_sec.back().size() == 2000) { all_sec.emplace_back(); all_sec.back().reserve(2000); LOG(md, Info, "{} symbols added", total_count); } all_sec.back().push_back(std::move(s)); // Move to avoid copying } } } // std::ostringstream output; // for (const auto& sec_group : all_sec) { // for (size_t i = 0; i < sec_group.size(); ++i) { // if (i > 0) output << ", "; // output << sec_group[i]; // } // output << '\n'; // Newline after each group // } // // Store the final output in a stringls // std::string final_output = output.str(); // size_t size = all_sec.size(); LOG(md, Info, "total {} secs loaded", total_count); bool first = true; auto client = databento::LiveBlocking{ databento::ILogReceiver::Default(), key, dataset, true, databento::VersionUpgradePolicy::UpgradeToV2, {}}; std::string start = config::get() .cboe() .at(0) .databento_settings() .bbo_start_time(); if(start != ""){ for (std::vector vec : all_sec) { client.Subscribe( vec, databento::Schema::Mbp1, databento::SType::RawSymbol, start); client.Subscribe( vec, databento::Schema::Trades, databento::SType::RawSymbol,start); } }else{ for (std::vector vec : all_sec) { client.Subscribe( vec, databento::Schema::Mbp1, databento::SType::RawSymbol); client.Subscribe( vec, databento::Schema::Trades, databento::SType::RawSymbol); } } bool only_securities = config::get() .cboe() .at(0) .databento_settings() .only_securities(); client.Start(); while (true) { try { auto *record = client.NextRecord(std::chrono::seconds{500}); if (first) { LOG(md, Info, "first record"); first = false; } if (record == nullptr) { LOG(md, Error, "Timed out waiting for data"); break; } if(only_securities) continue; if (record->Holds()) { process_mbp1(*record); } if (all_times.size() == 1001) { log_time_statistics(all_times); client.Stop(); } // if (record->Holds()) { // process_security(*record); // } if (record->Holds()) { process_trades(*record); } } catch (const databento::TcpError &error) { std::string err = error.what(); LOG(md, Error, "Caught TCP error: {} \nAttempting reconnect\n", err); client.Reconnect(); } catch (const databento::DbnResponseError &err) { std::string msg = err.Message(); std::string what = err.what(); LOG(md, Error, "Caught DbnResponseError what: {} ,msg:{} \nShutting down\n", msg, what); client.Stop(); // bts_assert(false); } catch (const databento::Exception &excp) { std::string err = excp.Message(); std::string err2 = excp.what(); LOG(md, Error, "Caught unknown dadtabento::exception what: {}, msg: {} " "\nShutting down\n", err, err2); client.Stop(); // bts_assert(false); } catch (const std::exception &exc) { std::string err = exc.what(); LOG(md, Error, "Caught unknown exception: {} \nShutting down\n", err); client.Stop(); // bts_assert(false); } } } } } // namespace bts::md::DatabentoHandler