686 lines
26 KiB
C++
686 lines
26 KiB
C++
#include "DatabentoDataHandler.h"
|
|
#include "databento/datetime.hpp"
|
|
#include "databento/dbn_encoder.hpp"
|
|
#include "databento/enums.hpp"
|
|
#include "databento/exceptions.hpp"
|
|
#include "databento/live_blocking.hpp"
|
|
#include "databento/log.hpp"
|
|
#include "databento/record.hpp"
|
|
#include "databento/timeseries.hpp"
|
|
#include "spark/api/domain/CallPut.pb.h"
|
|
#include "spark/api/domain/Venue.pb.h"
|
|
#include "spark/api/md/MarketDataConfig.pb.h"
|
|
#include "spark/config/config.h"
|
|
#include "spark/config/protos/system_config.pb.h"
|
|
#include "spark/domain/AggressorSide.h"
|
|
#include "spark/domain/Book.h"
|
|
#include "spark/domain/Quantity.h"
|
|
#include "spark/domain/Security.h"
|
|
#include "spark/domain/SecurityID.h"
|
|
#include "spark/domain/Side.h"
|
|
#include "spark/domain/TradeRecord.h"
|
|
#include "spark/domain_model/Security.pb.h"
|
|
#include "spark/logging/logging.h"
|
|
#include "spark/md/IMarketDataListener.h"
|
|
#include "spark/persistence/IDGenerator.h"
|
|
#include "spark/core/TimeStamp.h"
|
|
#include "spark/utils/TimeStamp.h"
|
|
#include "spark/utils/bts_assert.h"
|
|
#include "spark/utils/bts_chrono.h"
|
|
#include "spark/utils/bts_assert.h"
|
|
#include "spark/utils/datetime.h"
|
|
|
|
#include <boost/filesystem/operations.hpp>
|
|
#include <chrono>
|
|
#include <cstddef>
|
|
#include <databento/constants.hpp>
|
|
#include <databento/historical.hpp>
|
|
#include <databento/live.hpp>
|
|
#include <optional>
|
|
#include <ratio>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <tuple>
|
|
#include <unordered_map>
|
|
#include <vector>
|
|
|
|
namespace bts::md::DatabentoHandler {
|
|
|
|
#define SEC_TYPE(name) bts::api::domain::SECURITY_TYPE_##name
|
|
|
|
using bts::api::domain::CallPut;
|
|
using bts::api::domain::SecurityType;
|
|
using bts::api::domain::Venue;
|
|
using bts::domain::SecurityID;
|
|
|
|
std::string remove_spaces(const std::string_view &str) {
|
|
std::string result;
|
|
result.reserve(str.size()); // Reserve space for better performance
|
|
for (char c : str) {
|
|
if (!isspace(c)) {
|
|
result += c;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
static SecurityType parse_sec_type(std::string_view str) {
|
|
if (str.find("OPT") != std::string::npos || str.find("OOF") != std::string::npos) {
|
|
return SEC_TYPE(OPTION);
|
|
}
|
|
if (str.find("FUT") != std::string::npos) {
|
|
return SEC_TYPE(FUTURE);
|
|
}
|
|
return SEC_TYPE(UNSPECIFIED);
|
|
}
|
|
void update_file(
|
|
const std::string &key,
|
|
const std::string &dataset,
|
|
const std::vector<std::string> &parent_symbols,
|
|
const uint time_to_run,
|
|
std::string file_path) {
|
|
|
|
auto client =
|
|
databento::LiveBuilder{}
|
|
.SetKey(key)
|
|
.SetDataset(dataset)
|
|
.SetUpgradePolicy(databento::VersionUpgradePolicy::UpgradeToV2)
|
|
.BuildBlocking();
|
|
|
|
client.Subscribe(
|
|
parent_symbols, databento::Schema::Definition, databento::SType::Parent,
|
|
databento::UnixNanos{});
|
|
// We start streaming
|
|
databento::Metadata metadata = client.Start();
|
|
{
|
|
// We open a file for writing and encode the metadata
|
|
databento::OutFileStream out_file{file_path};
|
|
databento::DbnEncoder encoder{metadata, &out_file};
|
|
// Wait for at least one record and 5 seconds before closing the connection
|
|
const std::chrono::seconds timeout{time_to_run};
|
|
const auto start = std::chrono::steady_clock::now();
|
|
while (auto *record = client.NextRecord(timeout)) {
|
|
encoder.EncodeRecord(*record);
|
|
if (std::chrono::steady_clock::now() - start >= timeout) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
void log_time_statistics(
|
|
const std::vector<std::vector<std::tuple<uint64_t, std::string>>>
|
|
&all_times) {
|
|
std::unordered_map<std::string, std::vector<uint64_t>> subpart_times;
|
|
|
|
// Collect all timing values per subpart
|
|
for (const auto &entry : all_times) {
|
|
for (const auto &[time, label] : entry) {
|
|
subpart_times[label].push_back(time);
|
|
}
|
|
}
|
|
|
|
// Function to compute statistics
|
|
auto compute_statistics = [](std::vector<uint64_t> &values)
|
|
-> std::tuple<double, double, double, double, double, double, double> {
|
|
if (values.empty())
|
|
return std::make_tuple(0.0, 0.0, 0, 0, 0, 0, 0);
|
|
|
|
std::sort(values.begin(), values.end());
|
|
|
|
double sum = std::accumulate(values.begin(), values.end(), 0.0);
|
|
double avg = sum / values.size();
|
|
|
|
double sq_sum =
|
|
std::inner_product(values.begin(), values.end(), values.begin(), 0.0);
|
|
double stddev = std::sqrt(sq_sum / values.size() - avg * avg);
|
|
|
|
auto percentile = [&](double p) -> uint64_t {
|
|
size_t idx = static_cast<size_t>(p * (values.size() - 1));
|
|
return values[idx];
|
|
};
|
|
|
|
return std::make_tuple(
|
|
avg, stddev, percentile(0.00), percentile(0.25), percentile(0.50),
|
|
percentile(0.75), percentile(1.00));
|
|
};
|
|
|
|
// Compute and log statistics
|
|
for (auto &[label, times] : subpart_times) {
|
|
auto [avg, stddev, p0, p25, p50, p75, p100] = compute_statistics(times);
|
|
LOG(md, Sys,
|
|
"Subpart: {}, Avg: {:.2f} ns, Std: {:.2f} ns, P0: {} ns, P25: {} ns, "
|
|
"P50: {} ns, P75: {} ns, P100: {} ns",
|
|
label, avg, stddev, p0, p25, p50, p75, p100);
|
|
}
|
|
}
|
|
void DatabentoDatahandler::run(
|
|
bool use_file,
|
|
std::shared_ptr<IMarketDataListener> listener,
|
|
std::shared_ptr<persistence::IDGenerator> sec_id_store,
|
|
bts::api::domain::Venue venue,
|
|
const std::string &securities_path,
|
|
const std::string &books_path,
|
|
const std::string &key,
|
|
const std::string &dataset,
|
|
const uint filter_securities_upto_n_months,
|
|
const bool load_reference_data,
|
|
const std::chrono::duration<long double> sleep_duration) {
|
|
uint DEBUG_SEC_ID = 0;
|
|
if (venue == api::domain::VENUE_CBOE)
|
|
DEBUG_SEC_ID = config::get<bts::api::md::MarketDataConfig>()
|
|
.cboe()
|
|
.at(0)
|
|
.databento_settings()
|
|
.debug_sec();
|
|
auto sleep_duration_updated = sleep_duration;
|
|
std::vector<std::vector<std::tuple<uint64_t, std::string>>> all_times;
|
|
std::unordered_map<uint32_t, bts::domain::SecurityID> sec_id_map;
|
|
if (!use_file) {
|
|
sleep_duration_updated = std::chrono::duration<
|
|
long double>::zero(); // Set to zero for live data
|
|
}
|
|
auto sec_filter = [&](const core::TimeStamp expiration_time,
|
|
const std::string &symbol,
|
|
const api::domain::Venue &venue, uint strike) {
|
|
uint months = filter_securities_upto_n_months;
|
|
switch (venue) {
|
|
case api::domain::Venue::VENUE_CBOE: {
|
|
if (strike < 3800 || strike > 8000) {
|
|
return false;
|
|
}
|
|
if (symbol == "SPX")
|
|
months = 6;
|
|
else if (symbol == "SPXW")
|
|
months = 1;
|
|
break;
|
|
}
|
|
case api::domain::Venue::VENUE_GLBX:
|
|
return true;
|
|
default:
|
|
bts_assert_msg(false, "Exchange not supported on databento");
|
|
}
|
|
|
|
auto duration = months * 32 * 24 * 60 * 60;
|
|
return expiration_time.seconds() <
|
|
bts::core::TimeStamp::now().seconds() + duration;
|
|
};
|
|
auto process_security_recorded = [&](const databento::Record &record) {
|
|
const auto &msg = record.Get<databento::InstrumentDefMsg>();
|
|
const auto exp_nanos = msg.expiration;
|
|
std::string exp_string = databento::ToIso8601(exp_nanos);
|
|
const auto expiration_time =
|
|
bts::core::TimeStamp::from_iso_string(exp_string);
|
|
if (!sec_filter(
|
|
expiration_time, msg.Underlying(), venue,
|
|
msg.strike_price / 1000000000)) {
|
|
return databento::KeepGoing::Continue;
|
|
}
|
|
bts::domain::Security sec;
|
|
sec.security_common = domain::SecurityCommon();
|
|
sec.security_common.venue = venue;
|
|
sec.security_common.exchange_sec_id = msg.hd.instrument_id;
|
|
sec.security_common.desc = msg.RawSymbol();
|
|
sec.security_common.symbol = msg.Asset();
|
|
sec.security_common.asset = msg.Asset();
|
|
sec.security_common.group = msg.Group();
|
|
sec.security_common.min_price_increment = 0.01;
|
|
sec.security_common.min_quantity = 0.01;
|
|
sec.security_common.min_quantity_increment = 0.01;
|
|
sec.security_common.currency = {msg.currency.data(), msg.currency.size()};
|
|
|
|
const SecurityType sec_type =
|
|
parse_sec_type({msg.security_type.data(), msg.security_type.size()});
|
|
|
|
switch (sec_type) {
|
|
case SEC_TYPE(FUTURE):
|
|
sec.security.emplace<bts::domain::FutureSecurity>(expiration_time);
|
|
break;
|
|
case SEC_TYPE(OPTION): {
|
|
if (sec.venue() == api::domain::VENUE_CBOE) {
|
|
if (sec.security_common.symbol == "SPX" ||
|
|
sec.security_common.symbol == "SPXW") {
|
|
sec.security_common.min_quantity_increment = 1;
|
|
sec.security_common.min_quantity = 1;
|
|
sec.security_common.price_ticks.push_back(3.0);
|
|
sec.security_common.price_ticks.push_back(0.05);
|
|
sec.security_common.price_ticks.push_back(0.1);
|
|
sec.security_common.cash_per_point = 100;
|
|
}
|
|
}
|
|
if (sec.venue() == api::domain::VENUE_GLBX) {
|
|
sec.security_common.min_quantity_increment = 1;
|
|
sec.security_common.min_quantity = 1;
|
|
sec.security_common.price_ticks.push_back(10);
|
|
sec.security_common.price_ticks.push_back(0.05);
|
|
sec.security_common.price_ticks.push_back(0.25);
|
|
sec.security_common.cash_per_point = 50;
|
|
}
|
|
if (msg.instrument_class != databento::instrument_class::Call && msg.instrument_class != databento::instrument_class::Put){
|
|
return databento::KeepGoing::Continue;
|
|
}
|
|
auto call_put = (msg.instrument_class == databento::instrument_class::Call)? CallPut::CALL : CallPut::PUT;
|
|
sec.security.emplace<bts::domain::OptionSecurity>(
|
|
expiration_time, bts::domain::Price(msg.strike_price, -9),
|
|
call_put);
|
|
break;
|
|
}
|
|
default:
|
|
LOG(md, Info, "skipping sec as only options and futures supported, {}",sec.security_common.desc);
|
|
return databento::KeepGoing::Continue;
|
|
// bts_assert_msg(
|
|
// false, "Only futures and options supported on databento");
|
|
}
|
|
*sec.security_common.id = sec_id_store->get_or_add(
|
|
venue, sec_type, std::to_string(msg.hd.instrument_id));
|
|
listener->on_securities({&sec});
|
|
sec_id_map.insert({msg.hd.instrument_id, sec.security_common.id});
|
|
return databento::KeepGoing::Continue;
|
|
};
|
|
auto process_security = [&](const databento::Record &record) -> bool {
|
|
const auto &msg = record.Get<databento::InstrumentDefMsg>();
|
|
const auto exp_nanos = msg.expiration;
|
|
std::string exp_string = databento::ToIso8601(exp_nanos);
|
|
const auto expiration_time =
|
|
bts::core::TimeStamp::from_iso_string(exp_string);
|
|
if (!sec_filter(
|
|
expiration_time, msg.Underlying(), venue,
|
|
msg.strike_price / 1000000000)) {
|
|
return false;
|
|
}
|
|
bts::domain::Security sec;
|
|
sec.security_common = domain::SecurityCommon();
|
|
sec.security_common.venue = venue;
|
|
sec.security_common.exchange_sec_id = msg.hd.instrument_id;
|
|
sec.security_common.desc = msg.RawSymbol();
|
|
sec.security_common.symbol = msg.Underlying();
|
|
sec.security_common.asset = msg.Asset();
|
|
sec.security_common.group = msg.Group();
|
|
sec.security_common.min_price_increment = 0.01;
|
|
sec.security_common.min_quantity = 0.01;
|
|
sec.security_common.min_quantity_increment = 0.01;
|
|
sec.security_common.currency = {msg.currency.data(), msg.currency.size()};
|
|
|
|
const SecurityType sec_type =
|
|
parse_sec_type({msg.security_type.data(), msg.security_type.size()});
|
|
|
|
switch (sec_type) {
|
|
case SEC_TYPE(FUTURE):
|
|
sec.security.emplace<bts::domain::FutureSecurity>(expiration_time);
|
|
break;
|
|
case SEC_TYPE(OPTION): {
|
|
if (sec.venue() == api::domain::VENUE_CBOE) {
|
|
if (sec.security_common.symbol == "SPX" ||
|
|
sec.security_common.symbol == "SPXW") {
|
|
sec.security_common.min_quantity_increment = 1;
|
|
sec.security_common.min_quantity = 1;
|
|
sec.security_common.price_ticks.push_back(3.0);
|
|
sec.security_common.price_ticks.push_back(0.05);
|
|
sec.security_common.price_ticks.push_back(0.1);
|
|
sec.security_common.cash_per_point = 100;
|
|
}
|
|
}
|
|
if (sec.venue() == api::domain::VENUE_GLBX) {
|
|
sec.security_common.min_quantity_increment = 1;
|
|
sec.security_common.min_quantity = 1;
|
|
sec.security_common.price_ticks.push_back(10);
|
|
sec.security_common.price_ticks.push_back(0.05);
|
|
sec.security_common.price_ticks.push_back(0.25);
|
|
sec.security_common.cash_per_point = 50;
|
|
}
|
|
auto call_put = CallPut::PUT;
|
|
if (msg.instrument_class == 'C')
|
|
call_put = CallPut::CALL;
|
|
else
|
|
bts_assert(msg.instrument_class == 'P');
|
|
sec.security.emplace<bts::domain::OptionSecurity>(
|
|
expiration_time, bts::domain::Price(msg.strike_price, -9),
|
|
call_put);
|
|
break;
|
|
}
|
|
default:
|
|
bts_assert_msg(
|
|
false, "Only futures and options supported on databento");
|
|
}
|
|
*sec.security_common.id = sec_id_store->get_or_add(
|
|
venue, sec_type, std::to_string(msg.hd.instrument_id));
|
|
listener->on_securities({&sec});
|
|
sec_id_map.insert({msg.hd.instrument_id, sec.security_common.id});
|
|
return true;
|
|
};
|
|
core::TimeStamp last_time;
|
|
long i = 0;
|
|
auto debug_log = [&](const databento::Mbp1Msg &msg) {
|
|
|
|
if (DEBUG_SEC_ID != msg.hd.instrument_id)
|
|
return;
|
|
const double denom = 1000000000;
|
|
// auto delta = std::to_string(std::chrono::duration<double, std::micro>(msg.ts_in_delta).count());
|
|
auto seq_num = msg.sequence;
|
|
auto ap = msg.levels[0].ask_px / denom;
|
|
auto bp = msg.levels[0].bid_px / denom;
|
|
auto as = msg.levels[0].ask_sz;
|
|
auto bs = msg.levels[0].bid_sz;
|
|
LOG(md, Sys, "ID: {}, seq_num: {}, B_P: {}, B_S: {}, A_P: {}, A_S: {}",
|
|
msg.hd.instrument_id, seq_num, bp, bs, ap, as);
|
|
};
|
|
|
|
auto process_mbp1 = [&](const databento::Record &record) {
|
|
const auto &msg = record.Get<databento::Mbp1Msg>();
|
|
debug_log(msg);
|
|
if(i++==0){
|
|
last_time=core::TimeStamp::from_int64(msg.ts_recv.time_since_epoch().count());
|
|
}else{
|
|
auto new_time = core::TimeStamp::from_int64(msg.ts_recv.time_since_epoch().count());
|
|
auto dif = new_time-last_time;
|
|
if(dif.nanos()>0){
|
|
// LOG(md,Info,"sleeping for {}",dif.nanos());
|
|
sleep_for(std::chrono::nanoseconds(dif.nanos()));
|
|
}
|
|
last_time = new_time;
|
|
}
|
|
|
|
// std::vector<std::tuple<uint64_t,std::string>>& times = all_times.emplace_back();
|
|
// times.reserve(8);
|
|
// return databento::KeepGoing::Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version
|
|
// The security type to is needed to get the id. Since the sec type isn't known,
|
|
// check for an id using both types (OPTION and FUTURE). This is safe to do since
|
|
// options and futures never have common names (instrument ids).
|
|
// auto current = bts::datetime::unix_nano_now();
|
|
// auto prev = current;
|
|
// times.push_back({current-prev,"Start Process bbo"});
|
|
|
|
// prev = current;
|
|
// current = bts::datetime::unix_nano_now();
|
|
// times.push_back({current-prev,"MISC to check log time"});
|
|
if (!sec_id_map.contains(msg.hd.instrument_id))
|
|
return databento::KeepGoing::Continue;
|
|
bts::domain::SecurityID sec_id = sec_id_map.at(msg.hd.instrument_id);
|
|
if(msg.action == databento::action::Trade){
|
|
domain::TradeRecord trade;
|
|
trade.trade_price = bts::domain::Price(msg.price, -9);
|
|
trade.trade_quantity = bts::domain::Quantity(msg.size);
|
|
trade.sec_id = sec_id;
|
|
switch (msg.side) {
|
|
case databento::Side::Ask:
|
|
trade.agg_side = domain::AggressorSide::Sell;
|
|
break;
|
|
case databento::Side::Bid:
|
|
trade.agg_side = domain::AggressorSide::Buy;
|
|
break;
|
|
case databento::Side::None:
|
|
trade.agg_side = domain::AggressorSide::None;
|
|
break;
|
|
}
|
|
trade.time_stamp = core::TimeStamp::now();
|
|
//bts::core::TimeStamp::from_iso_string(
|
|
// databento::ToIso8601(msg.ts_recv));
|
|
std::vector<const domain::TradeRecord *> trade_ptrs;
|
|
trade_ptrs.push_back(&trade);
|
|
listener->on_trades(trade_ptrs);
|
|
return databento::KeepGoing::Continue;
|
|
}
|
|
// prev = current;
|
|
// current = bts::datetime::unix_nano_now();
|
|
// times.push_back({current-prev,"Sec_ID done"});
|
|
|
|
// sleep_for(sleep_duration_updated);
|
|
// prev = current;
|
|
// current = bts::datetime::unix_nano_now();
|
|
// times.push_back({current-prev,"Sleep done"});
|
|
|
|
domain::Book book;
|
|
book.set_security_id(sec_id);
|
|
// prev = current;
|
|
// current = bts::datetime::unix_nano_now();
|
|
// times.push_back({current-prev,"Empty Book Made"});
|
|
|
|
auto &bids = book.bids();
|
|
auto &asks = book.asks();
|
|
bids.begin()->price = bts::domain::Price(msg.levels[0].bid_px, -9);
|
|
bids.begin()->total_quantity =
|
|
bts::domain::Quantity(msg.levels[0].bid_sz);
|
|
asks.begin()->price = bts::domain::Price(msg.levels[0].ask_px, -9);
|
|
asks.begin()->total_quantity =
|
|
bts::domain::Quantity(msg.levels[0].ask_sz);
|
|
// prev = current;
|
|
// current = bts::datetime::unix_nano_now();
|
|
// times.push_back({current-prev,"Complete book done"});
|
|
|
|
|
|
|
|
|
|
//this is correct
|
|
// auto t2 = datetime::parse_iso_date_time_string(databento::ToIso8601(msg.ts_recv));
|
|
// auto t3 = core::TimeStamp::from_int64(msg.ts_recv.time_since_epoch().count());
|
|
// auto now_ts = utils::TimeStamp::now();
|
|
// auto now = datetime::now();
|
|
|
|
// auto delta = (now - t).total_milliseconds();
|
|
// return now_ts.shift_ms(delta);
|
|
|
|
|
|
//this is wrong
|
|
// auto t3 = utils::TimeStamp(msg.ts_recv.time_since_epoch().count());
|
|
//this is also
|
|
// auto t4 =datetime::ptime_to_timestamp(t3.to_ptime());
|
|
|
|
// LOG(md,Info,"b01 {}",databento::ToIso8601(msg.ts_recv));
|
|
// LOG(md,Info,"b02 {}",datetime::iso_date_time_string(t3.to_ptime()));
|
|
// LOG(md,Info,"b03: {}",t3.to_iso_string());
|
|
// LOG(md,Info,"b04: {}",t4.to_uint64());
|
|
// LOG(md,Info,"b041: {}",t4.to_int64());
|
|
// LOG(md,Info,"b042: {}",utils::TimeStamp(msg.ts_recv.time_since_epoch().count()).to_int64());
|
|
// LOG(md,Info,"b05: {}",t3.nanos());
|
|
// LOG(md,Info,"b06: {}",msg.ts_recv.time_since_epoch().count());
|
|
// LOG(md,Info,"b07: {}",utils::TimeStamp::now().to_int64());
|
|
|
|
listener->on_book(
|
|
book,
|
|
//utils::TimeStamp(msg.ts_recv.time_since_epoch().count())
|
|
utils::TimeStamp::now()
|
|
);
|
|
// prev = current;
|
|
// current = bts::datetime::unix_nano_now();
|
|
// times.push_back({current-prev,"On book done"});
|
|
|
|
return databento::KeepGoing::
|
|
Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version
|
|
};
|
|
|
|
auto process_trades = [&](const databento::Record &record) {
|
|
const auto &msg = record.Get<databento::TradeMsg>();
|
|
// The security type to is needed to get the id. Since the sec type isn't known,
|
|
// check for an id using both types (OPTION and FUTURE). This is safe to do since
|
|
// options and futures never have common names (instrument ids).
|
|
if (!sec_id_map.contains(msg.hd.instrument_id))
|
|
return databento::KeepGoing::Continue;
|
|
bts::domain::SecurityID sec_id = sec_id_map.at(msg.hd.instrument_id);
|
|
|
|
// sleep_for(sleep_duration_updated);
|
|
domain::TradeRecord trade;
|
|
trade.trade_price = bts::domain::Price(msg.price, -9);
|
|
trade.trade_quantity = bts::domain::Quantity(msg.size);
|
|
trade.sec_id = sec_id;
|
|
switch (msg.side) {
|
|
case databento::Side::Ask:
|
|
trade.agg_side = domain::AggressorSide::Sell;
|
|
break;
|
|
case databento::Side::Bid:
|
|
trade.agg_side = domain::AggressorSide::Buy;
|
|
break;
|
|
case databento::Side::None:
|
|
trade.agg_side = domain::AggressorSide::None;
|
|
break;
|
|
}
|
|
trade.time_stamp = bts::core::TimeStamp::from_iso_string(
|
|
databento::ToIso8601(msg.ts_recv));
|
|
std::vector<const domain::TradeRecord *> trade_ptrs;
|
|
trade_ptrs.push_back(&trade);
|
|
listener->on_trades(trade_ptrs);
|
|
|
|
return databento::KeepGoing::
|
|
Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version
|
|
};
|
|
if (use_file) {
|
|
auto client = databento::HistoricalBuilder{}.SetKey(key).Build();
|
|
databento::DbnFileStore dbn_file_store(securities_path);
|
|
dbn_file_store.Replay(process_security_recorded); // fix this
|
|
LOG(md,Info,"done secs");
|
|
databento::DbnFileStore dbn_file_store2(books_path);
|
|
dbn_file_store2.Replay(process_mbp1);
|
|
LOG(md,Info,"done bbo");
|
|
sleep(1000);
|
|
} else {
|
|
bts_assert(dataset != "");
|
|
|
|
//hardcoding for now will update later
|
|
std::vector<std::string> symbols;
|
|
switch (venue) {
|
|
case api::domain::VENUE_CBOE:
|
|
symbols = {"SPX.OPT", "SPXW.OPT"};
|
|
break;
|
|
case api::domain::VENUE_GLBX:
|
|
symbols = {"ES.FUT"};
|
|
break;
|
|
default:
|
|
bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW");
|
|
}
|
|
//ocassionaly run security writer to update security dbn
|
|
if (!boost::filesystem::exists(securities_path) || load_reference_data) {
|
|
update_file(key, dataset, symbols, 15 * 60, securities_path);
|
|
}
|
|
|
|
//regular flow
|
|
//read from security dbn, filter contracts that we need and subscribe to those in batches of two thousand
|
|
LOG(md, Info, "starting loading symbols");
|
|
databento::DbnFileStore store{securities_path};
|
|
std::vector<std::vector<std::string>> all_sec;
|
|
|
|
all_sec.emplace_back();
|
|
all_sec.back().reserve(2000);
|
|
size_t total_count = 0;
|
|
|
|
while (auto *record = store.NextRecord()) {
|
|
if (auto *def = record->GetIf<databento::InstrumentDefMsg>()) {
|
|
std::string s = def->RawSymbol();
|
|
if (process_security(*record)) {
|
|
++total_count;
|
|
// Check if the last vector is full
|
|
if (all_sec.back().size() == 2000) {
|
|
all_sec.emplace_back();
|
|
all_sec.back().reserve(2000);
|
|
LOG(md, Info, "{} symbols added", total_count);
|
|
}
|
|
all_sec.back().push_back(std::move(s)); // Move to avoid copying
|
|
}
|
|
}
|
|
}
|
|
// std::ostringstream output;
|
|
|
|
// for (const auto& sec_group : all_sec) {
|
|
// for (size_t i = 0; i < sec_group.size(); ++i) {
|
|
// if (i > 0) output << ", ";
|
|
// output << sec_group[i];
|
|
// }
|
|
// output << '\n'; // Newline after each group
|
|
// }
|
|
|
|
// // Store the final output in a stringls
|
|
|
|
// std::string final_output = output.str();
|
|
// size_t size = all_sec.size();
|
|
LOG(md, Info, "total {} secs loaded", total_count);
|
|
|
|
|
|
bool first = true;
|
|
auto client = databento::LiveBlocking{
|
|
databento::ILogReceiver::Default(), key, dataset, true,
|
|
databento::VersionUpgradePolicy::UpgradeToV2, {}};
|
|
std::string start = config::get<bts::api::md::MarketDataConfig>()
|
|
.cboe()
|
|
.at(0)
|
|
.databento_settings()
|
|
.bbo_start_time();
|
|
if(start != ""){
|
|
for (std::vector vec : all_sec) {
|
|
client.Subscribe(
|
|
vec, databento::Schema::Mbp1, databento::SType::RawSymbol, start);
|
|
client.Subscribe(
|
|
vec, databento::Schema::Trades, databento::SType::RawSymbol,start);
|
|
}
|
|
}else{
|
|
for (std::vector vec : all_sec) {
|
|
client.Subscribe(
|
|
vec, databento::Schema::Mbp1, databento::SType::RawSymbol);
|
|
client.Subscribe(
|
|
vec, databento::Schema::Trades, databento::SType::RawSymbol);
|
|
}
|
|
}
|
|
bool only_securities = config::get<bts::api::md::MarketDataConfig>()
|
|
.cboe()
|
|
.at(0)
|
|
.databento_settings()
|
|
.only_securities();
|
|
client.Start();
|
|
while (true) {
|
|
try {
|
|
auto *record = client.NextRecord(std::chrono::seconds{500});
|
|
if (first) {
|
|
LOG(md, Info, "first record");
|
|
first = false;
|
|
}
|
|
if (record == nullptr) {
|
|
LOG(md, Error, "Timed out waiting for data");
|
|
break;
|
|
}
|
|
if(only_securities)
|
|
continue;
|
|
if (record->Holds<databento::Mbp1Msg>()) {
|
|
process_mbp1(*record);
|
|
}
|
|
if (all_times.size() == 1001) {
|
|
log_time_statistics(all_times);
|
|
client.Stop();
|
|
}
|
|
// if (record->Holds<databento::InstrumentDefMsg>()) {
|
|
// process_security(*record);
|
|
// }
|
|
if (record->Holds<databento::TradeMsg>()) {
|
|
process_trades(*record);
|
|
}
|
|
} catch (const databento::TcpError &error) {
|
|
std::string err = error.what();
|
|
LOG(md, Error, "Caught TCP error: {} \nAttempting reconnect\n",
|
|
err);
|
|
client.Reconnect();
|
|
} catch (const databento::DbnResponseError &err) {
|
|
std::string msg = err.Message();
|
|
std::string what = err.what();
|
|
LOG(md, Error,
|
|
"Caught DbnResponseError what: {} ,msg:{} \nShutting down\n",
|
|
msg, what);
|
|
client.Stop();
|
|
// bts_assert(false);
|
|
} catch (const databento::Exception &excp) {
|
|
std::string err = excp.Message();
|
|
std::string err2 = excp.what();
|
|
LOG(md, Error,
|
|
"Caught unknown dadtabento::exception what: {}, msg: {} "
|
|
"\nShutting down\n",
|
|
err, err2);
|
|
client.Stop();
|
|
// bts_assert(false);
|
|
} catch (const std::exception &exc) {
|
|
std::string err = exc.what();
|
|
LOG(md, Error, "Caught unknown exception: {} \nShutting down\n",
|
|
err);
|
|
client.Stop();
|
|
// bts_assert(false);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
} // namespace bts::md::DatabentoHandler
|