Files
spark/src/app_sparkd/sparkd/md/databento/DatabentoDataHandler.cpp
T

686 lines
26 KiB
C++

#include "DatabentoDataHandler.h"
#include "databento/datetime.hpp"
#include "databento/dbn_encoder.hpp"
#include "databento/enums.hpp"
#include "databento/exceptions.hpp"
#include "databento/live_blocking.hpp"
#include "databento/log.hpp"
#include "databento/record.hpp"
#include "databento/timeseries.hpp"
#include "spark/api/domain/CallPut.pb.h"
#include "spark/api/domain/Venue.pb.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/config/config.h"
#include "spark/config/protos/system_config.pb.h"
#include "spark/domain/AggressorSide.h"
#include "spark/domain/Book.h"
#include "spark/domain/Quantity.h"
#include "spark/domain/Security.h"
#include "spark/domain/SecurityID.h"
#include "spark/domain/Side.h"
#include "spark/domain/TradeRecord.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/core/TimeStamp.h"
#include "spark/utils/TimeStamp.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/bts_chrono.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/datetime.h"
#include <boost/filesystem/operations.hpp>
#include <chrono>
#include <cstddef>
#include <databento/constants.hpp>
#include <databento/historical.hpp>
#include <databento/live.hpp>
#include <optional>
#include <ratio>
#include <string>
#include <string_view>
#include <tuple>
#include <unordered_map>
#include <vector>
namespace bts::md::DatabentoHandler {
#define SEC_TYPE(name) bts::api::domain::SECURITY_TYPE_##name
using bts::api::domain::CallPut;
using bts::api::domain::SecurityType;
using bts::api::domain::Venue;
using bts::domain::SecurityID;
std::string remove_spaces(const std::string_view &str) {
std::string result;
result.reserve(str.size()); // Reserve space for better performance
for (char c : str) {
if (!isspace(c)) {
result += c;
}
}
return result;
}
static SecurityType parse_sec_type(std::string_view str) {
if (str.find("OPT") != std::string::npos || str.find("OOF") != std::string::npos) {
return SEC_TYPE(OPTION);
}
if (str.find("FUT") != std::string::npos) {
return SEC_TYPE(FUTURE);
}
return SEC_TYPE(UNSPECIFIED);
}
void update_file(
const std::string &key,
const std::string &dataset,
const std::vector<std::string> &parent_symbols,
const uint time_to_run,
std::string file_path) {
auto client =
databento::LiveBuilder{}
.SetKey(key)
.SetDataset(dataset)
.SetUpgradePolicy(databento::VersionUpgradePolicy::UpgradeToV2)
.BuildBlocking();
client.Subscribe(
parent_symbols, databento::Schema::Definition, databento::SType::Parent,
databento::UnixNanos{});
// We start streaming
databento::Metadata metadata = client.Start();
{
// We open a file for writing and encode the metadata
databento::OutFileStream out_file{file_path};
databento::DbnEncoder encoder{metadata, &out_file};
// Wait for at least one record and 5 seconds before closing the connection
const std::chrono::seconds timeout{time_to_run};
const auto start = std::chrono::steady_clock::now();
while (auto *record = client.NextRecord(timeout)) {
encoder.EncodeRecord(*record);
if (std::chrono::steady_clock::now() - start >= timeout) {
break;
}
}
}
return;
}
void log_time_statistics(
const std::vector<std::vector<std::tuple<uint64_t, std::string>>>
&all_times) {
std::unordered_map<std::string, std::vector<uint64_t>> subpart_times;
// Collect all timing values per subpart
for (const auto &entry : all_times) {
for (const auto &[time, label] : entry) {
subpart_times[label].push_back(time);
}
}
// Function to compute statistics
auto compute_statistics = [](std::vector<uint64_t> &values)
-> std::tuple<double, double, double, double, double, double, double> {
if (values.empty())
return std::make_tuple(0.0, 0.0, 0, 0, 0, 0, 0);
std::sort(values.begin(), values.end());
double sum = std::accumulate(values.begin(), values.end(), 0.0);
double avg = sum / values.size();
double sq_sum =
std::inner_product(values.begin(), values.end(), values.begin(), 0.0);
double stddev = std::sqrt(sq_sum / values.size() - avg * avg);
auto percentile = [&](double p) -> uint64_t {
size_t idx = static_cast<size_t>(p * (values.size() - 1));
return values[idx];
};
return std::make_tuple(
avg, stddev, percentile(0.00), percentile(0.25), percentile(0.50),
percentile(0.75), percentile(1.00));
};
// Compute and log statistics
for (auto &[label, times] : subpart_times) {
auto [avg, stddev, p0, p25, p50, p75, p100] = compute_statistics(times);
LOG(md, Sys,
"Subpart: {}, Avg: {:.2f} ns, Std: {:.2f} ns, P0: {} ns, P25: {} ns, "
"P50: {} ns, P75: {} ns, P100: {} ns",
label, avg, stddev, p0, p25, p50, p75, p100);
}
}
void DatabentoDatahandler::run(
bool use_file,
std::shared_ptr<IMarketDataListener> listener,
std::shared_ptr<persistence::IDGenerator> sec_id_store,
bts::api::domain::Venue venue,
const std::string &securities_path,
const std::string &books_path,
const std::string &key,
const std::string &dataset,
const uint filter_securities_upto_n_months,
const bool load_reference_data,
const std::chrono::duration<long double> sleep_duration) {
uint DEBUG_SEC_ID = 0;
if (venue == api::domain::VENUE_CBOE)
DEBUG_SEC_ID = config::get<bts::api::md::MarketDataConfig>()
.cboe()
.at(0)
.databento_settings()
.debug_sec();
auto sleep_duration_updated = sleep_duration;
std::vector<std::vector<std::tuple<uint64_t, std::string>>> all_times;
std::unordered_map<uint32_t, bts::domain::SecurityID> sec_id_map;
if (!use_file) {
sleep_duration_updated = std::chrono::duration<
long double>::zero(); // Set to zero for live data
}
auto sec_filter = [&](const core::TimeStamp expiration_time,
const std::string &symbol,
const api::domain::Venue &venue, uint strike) {
uint months = filter_securities_upto_n_months;
switch (venue) {
case api::domain::Venue::VENUE_CBOE: {
if (strike < 3800 || strike > 8000) {
return false;
}
if (symbol == "SPX")
months = 6;
else if (symbol == "SPXW")
months = 1;
break;
}
case api::domain::Venue::VENUE_GLBX:
return true;
default:
bts_assert_msg(false, "Exchange not supported on databento");
}
auto duration = months * 32 * 24 * 60 * 60;
return expiration_time.seconds() <
bts::core::TimeStamp::now().seconds() + duration;
};
auto process_security_recorded = [&](const databento::Record &record) {
const auto &msg = record.Get<databento::InstrumentDefMsg>();
const auto exp_nanos = msg.expiration;
std::string exp_string = databento::ToIso8601(exp_nanos);
const auto expiration_time =
bts::core::TimeStamp::from_iso_string(exp_string);
if (!sec_filter(
expiration_time, msg.Underlying(), venue,
msg.strike_price / 1000000000)) {
return databento::KeepGoing::Continue;
}
bts::domain::Security sec;
sec.security_common = domain::SecurityCommon();
sec.security_common.venue = venue;
sec.security_common.exchange_sec_id = msg.hd.instrument_id;
sec.security_common.desc = msg.RawSymbol();
sec.security_common.symbol = msg.Asset();
sec.security_common.asset = msg.Asset();
sec.security_common.group = msg.Group();
sec.security_common.min_price_increment = 0.01;
sec.security_common.min_quantity = 0.01;
sec.security_common.min_quantity_increment = 0.01;
sec.security_common.currency = {msg.currency.data(), msg.currency.size()};
const SecurityType sec_type =
parse_sec_type({msg.security_type.data(), msg.security_type.size()});
switch (sec_type) {
case SEC_TYPE(FUTURE):
sec.security.emplace<bts::domain::FutureSecurity>(expiration_time);
break;
case SEC_TYPE(OPTION): {
if (sec.venue() == api::domain::VENUE_CBOE) {
if (sec.security_common.symbol == "SPX" ||
sec.security_common.symbol == "SPXW") {
sec.security_common.min_quantity_increment = 1;
sec.security_common.min_quantity = 1;
sec.security_common.price_ticks.push_back(3.0);
sec.security_common.price_ticks.push_back(0.05);
sec.security_common.price_ticks.push_back(0.1);
sec.security_common.cash_per_point = 100;
}
}
if (sec.venue() == api::domain::VENUE_GLBX) {
sec.security_common.min_quantity_increment = 1;
sec.security_common.min_quantity = 1;
sec.security_common.price_ticks.push_back(10);
sec.security_common.price_ticks.push_back(0.05);
sec.security_common.price_ticks.push_back(0.25);
sec.security_common.cash_per_point = 50;
}
if (msg.instrument_class != databento::instrument_class::Call && msg.instrument_class != databento::instrument_class::Put){
return databento::KeepGoing::Continue;
}
auto call_put = (msg.instrument_class == databento::instrument_class::Call)? CallPut::CALL : CallPut::PUT;
sec.security.emplace<bts::domain::OptionSecurity>(
expiration_time, bts::domain::Price(msg.strike_price, -9),
call_put);
break;
}
default:
LOG(md, Info, "skipping sec as only options and futures supported, {}",sec.security_common.desc);
return databento::KeepGoing::Continue;
// bts_assert_msg(
// false, "Only futures and options supported on databento");
}
*sec.security_common.id = sec_id_store->get_or_add(
venue, sec_type, std::to_string(msg.hd.instrument_id));
listener->on_securities({&sec});
sec_id_map.insert({msg.hd.instrument_id, sec.security_common.id});
return databento::KeepGoing::Continue;
};
auto process_security = [&](const databento::Record &record) -> bool {
const auto &msg = record.Get<databento::InstrumentDefMsg>();
const auto exp_nanos = msg.expiration;
std::string exp_string = databento::ToIso8601(exp_nanos);
const auto expiration_time =
bts::core::TimeStamp::from_iso_string(exp_string);
if (!sec_filter(
expiration_time, msg.Underlying(), venue,
msg.strike_price / 1000000000)) {
return false;
}
bts::domain::Security sec;
sec.security_common = domain::SecurityCommon();
sec.security_common.venue = venue;
sec.security_common.exchange_sec_id = msg.hd.instrument_id;
sec.security_common.desc = msg.RawSymbol();
sec.security_common.symbol = msg.Underlying();
sec.security_common.asset = msg.Asset();
sec.security_common.group = msg.Group();
sec.security_common.min_price_increment = 0.01;
sec.security_common.min_quantity = 0.01;
sec.security_common.min_quantity_increment = 0.01;
sec.security_common.currency = {msg.currency.data(), msg.currency.size()};
const SecurityType sec_type =
parse_sec_type({msg.security_type.data(), msg.security_type.size()});
switch (sec_type) {
case SEC_TYPE(FUTURE):
sec.security.emplace<bts::domain::FutureSecurity>(expiration_time);
break;
case SEC_TYPE(OPTION): {
if (sec.venue() == api::domain::VENUE_CBOE) {
if (sec.security_common.symbol == "SPX" ||
sec.security_common.symbol == "SPXW") {
sec.security_common.min_quantity_increment = 1;
sec.security_common.min_quantity = 1;
sec.security_common.price_ticks.push_back(3.0);
sec.security_common.price_ticks.push_back(0.05);
sec.security_common.price_ticks.push_back(0.1);
sec.security_common.cash_per_point = 100;
}
}
if (sec.venue() == api::domain::VENUE_GLBX) {
sec.security_common.min_quantity_increment = 1;
sec.security_common.min_quantity = 1;
sec.security_common.price_ticks.push_back(10);
sec.security_common.price_ticks.push_back(0.05);
sec.security_common.price_ticks.push_back(0.25);
sec.security_common.cash_per_point = 50;
}
auto call_put = CallPut::PUT;
if (msg.instrument_class == 'C')
call_put = CallPut::CALL;
else
bts_assert(msg.instrument_class == 'P');
sec.security.emplace<bts::domain::OptionSecurity>(
expiration_time, bts::domain::Price(msg.strike_price, -9),
call_put);
break;
}
default:
bts_assert_msg(
false, "Only futures and options supported on databento");
}
*sec.security_common.id = sec_id_store->get_or_add(
venue, sec_type, std::to_string(msg.hd.instrument_id));
listener->on_securities({&sec});
sec_id_map.insert({msg.hd.instrument_id, sec.security_common.id});
return true;
};
core::TimeStamp last_time;
long i = 0;
auto debug_log = [&](const databento::Mbp1Msg &msg) {
if (DEBUG_SEC_ID != msg.hd.instrument_id)
return;
const double denom = 1000000000;
// auto delta = std::to_string(std::chrono::duration<double, std::micro>(msg.ts_in_delta).count());
auto seq_num = msg.sequence;
auto ap = msg.levels[0].ask_px / denom;
auto bp = msg.levels[0].bid_px / denom;
auto as = msg.levels[0].ask_sz;
auto bs = msg.levels[0].bid_sz;
LOG(md, Sys, "ID: {}, seq_num: {}, B_P: {}, B_S: {}, A_P: {}, A_S: {}",
msg.hd.instrument_id, seq_num, bp, bs, ap, as);
};
auto process_mbp1 = [&](const databento::Record &record) {
const auto &msg = record.Get<databento::Mbp1Msg>();
debug_log(msg);
if(i++==0){
last_time=core::TimeStamp::from_int64(msg.ts_recv.time_since_epoch().count());
}else{
auto new_time = core::TimeStamp::from_int64(msg.ts_recv.time_since_epoch().count());
auto dif = new_time-last_time;
if(dif.nanos()>0){
// LOG(md,Info,"sleeping for {}",dif.nanos());
sleep_for(std::chrono::nanoseconds(dif.nanos()));
}
last_time = new_time;
}
// std::vector<std::tuple<uint64_t,std::string>>& times = all_times.emplace_back();
// times.reserve(8);
// return databento::KeepGoing::Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version
// The security type to is needed to get the id. Since the sec type isn't known,
// check for an id using both types (OPTION and FUTURE). This is safe to do since
// options and futures never have common names (instrument ids).
// auto current = bts::datetime::unix_nano_now();
// auto prev = current;
// times.push_back({current-prev,"Start Process bbo"});
// prev = current;
// current = bts::datetime::unix_nano_now();
// times.push_back({current-prev,"MISC to check log time"});
if (!sec_id_map.contains(msg.hd.instrument_id))
return databento::KeepGoing::Continue;
bts::domain::SecurityID sec_id = sec_id_map.at(msg.hd.instrument_id);
if(msg.action == databento::action::Trade){
domain::TradeRecord trade;
trade.trade_price = bts::domain::Price(msg.price, -9);
trade.trade_quantity = bts::domain::Quantity(msg.size);
trade.sec_id = sec_id;
switch (msg.side) {
case databento::Side::Ask:
trade.agg_side = domain::AggressorSide::Sell;
break;
case databento::Side::Bid:
trade.agg_side = domain::AggressorSide::Buy;
break;
case databento::Side::None:
trade.agg_side = domain::AggressorSide::None;
break;
}
trade.time_stamp = core::TimeStamp::now();
//bts::core::TimeStamp::from_iso_string(
// databento::ToIso8601(msg.ts_recv));
std::vector<const domain::TradeRecord *> trade_ptrs;
trade_ptrs.push_back(&trade);
listener->on_trades(trade_ptrs);
return databento::KeepGoing::Continue;
}
// prev = current;
// current = bts::datetime::unix_nano_now();
// times.push_back({current-prev,"Sec_ID done"});
// sleep_for(sleep_duration_updated);
// prev = current;
// current = bts::datetime::unix_nano_now();
// times.push_back({current-prev,"Sleep done"});
domain::Book book;
book.set_security_id(sec_id);
// prev = current;
// current = bts::datetime::unix_nano_now();
// times.push_back({current-prev,"Empty Book Made"});
auto &bids = book.bids();
auto &asks = book.asks();
bids.begin()->price = bts::domain::Price(msg.levels[0].bid_px, -9);
bids.begin()->total_quantity =
bts::domain::Quantity(msg.levels[0].bid_sz);
asks.begin()->price = bts::domain::Price(msg.levels[0].ask_px, -9);
asks.begin()->total_quantity =
bts::domain::Quantity(msg.levels[0].ask_sz);
// prev = current;
// current = bts::datetime::unix_nano_now();
// times.push_back({current-prev,"Complete book done"});
//this is correct
// auto t2 = datetime::parse_iso_date_time_string(databento::ToIso8601(msg.ts_recv));
// auto t3 = core::TimeStamp::from_int64(msg.ts_recv.time_since_epoch().count());
// auto now_ts = utils::TimeStamp::now();
// auto now = datetime::now();
// auto delta = (now - t).total_milliseconds();
// return now_ts.shift_ms(delta);
//this is wrong
// auto t3 = utils::TimeStamp(msg.ts_recv.time_since_epoch().count());
//this is also
// auto t4 =datetime::ptime_to_timestamp(t3.to_ptime());
// LOG(md,Info,"b01 {}",databento::ToIso8601(msg.ts_recv));
// LOG(md,Info,"b02 {}",datetime::iso_date_time_string(t3.to_ptime()));
// LOG(md,Info,"b03: {}",t3.to_iso_string());
// LOG(md,Info,"b04: {}",t4.to_uint64());
// LOG(md,Info,"b041: {}",t4.to_int64());
// LOG(md,Info,"b042: {}",utils::TimeStamp(msg.ts_recv.time_since_epoch().count()).to_int64());
// LOG(md,Info,"b05: {}",t3.nanos());
// LOG(md,Info,"b06: {}",msg.ts_recv.time_since_epoch().count());
// LOG(md,Info,"b07: {}",utils::TimeStamp::now().to_int64());
listener->on_book(
book,
//utils::TimeStamp(msg.ts_recv.time_since_epoch().count())
utils::TimeStamp::now()
);
// prev = current;
// current = bts::datetime::unix_nano_now();
// times.push_back({current-prev,"On book done"});
return databento::KeepGoing::
Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version
};
auto process_trades = [&](const databento::Record &record) {
const auto &msg = record.Get<databento::TradeMsg>();
// The security type to is needed to get the id. Since the sec type isn't known,
// check for an id using both types (OPTION and FUTURE). This is safe to do since
// options and futures never have common names (instrument ids).
if (!sec_id_map.contains(msg.hd.instrument_id))
return databento::KeepGoing::Continue;
bts::domain::SecurityID sec_id = sec_id_map.at(msg.hd.instrument_id);
// sleep_for(sleep_duration_updated);
domain::TradeRecord trade;
trade.trade_price = bts::domain::Price(msg.price, -9);
trade.trade_quantity = bts::domain::Quantity(msg.size);
trade.sec_id = sec_id;
switch (msg.side) {
case databento::Side::Ask:
trade.agg_side = domain::AggressorSide::Sell;
break;
case databento::Side::Bid:
trade.agg_side = domain::AggressorSide::Buy;
break;
case databento::Side::None:
trade.agg_side = domain::AggressorSide::None;
break;
}
trade.time_stamp = bts::core::TimeStamp::from_iso_string(
databento::ToIso8601(msg.ts_recv));
std::vector<const domain::TradeRecord *> trade_ptrs;
trade_ptrs.push_back(&trade);
listener->on_trades(trade_ptrs);
return databento::KeepGoing::
Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version
};
if (use_file) {
auto client = databento::HistoricalBuilder{}.SetKey(key).Build();
databento::DbnFileStore dbn_file_store(securities_path);
dbn_file_store.Replay(process_security_recorded); // fix this
LOG(md,Info,"done secs");
databento::DbnFileStore dbn_file_store2(books_path);
dbn_file_store2.Replay(process_mbp1);
LOG(md,Info,"done bbo");
sleep(1000);
} else {
bts_assert(dataset != "");
//hardcoding for now will update later
std::vector<std::string> symbols;
switch (venue) {
case api::domain::VENUE_CBOE:
symbols = {"SPX.OPT", "SPXW.OPT"};
break;
case api::domain::VENUE_GLBX:
symbols = {"ES.FUT"};
break;
default:
bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW");
}
//ocassionaly run security writer to update security dbn
if (!boost::filesystem::exists(securities_path) || load_reference_data) {
update_file(key, dataset, symbols, 15 * 60, securities_path);
}
//regular flow
//read from security dbn, filter contracts that we need and subscribe to those in batches of two thousand
LOG(md, Info, "starting loading symbols");
databento::DbnFileStore store{securities_path};
std::vector<std::vector<std::string>> all_sec;
all_sec.emplace_back();
all_sec.back().reserve(2000);
size_t total_count = 0;
while (auto *record = store.NextRecord()) {
if (auto *def = record->GetIf<databento::InstrumentDefMsg>()) {
std::string s = def->RawSymbol();
if (process_security(*record)) {
++total_count;
// Check if the last vector is full
if (all_sec.back().size() == 2000) {
all_sec.emplace_back();
all_sec.back().reserve(2000);
LOG(md, Info, "{} symbols added", total_count);
}
all_sec.back().push_back(std::move(s)); // Move to avoid copying
}
}
}
// std::ostringstream output;
// for (const auto& sec_group : all_sec) {
// for (size_t i = 0; i < sec_group.size(); ++i) {
// if (i > 0) output << ", ";
// output << sec_group[i];
// }
// output << '\n'; // Newline after each group
// }
// // Store the final output in a stringls
// std::string final_output = output.str();
// size_t size = all_sec.size();
LOG(md, Info, "total {} secs loaded", total_count);
bool first = true;
auto client = databento::LiveBlocking{
databento::ILogReceiver::Default(), key, dataset, true,
databento::VersionUpgradePolicy::UpgradeToV2, {}};
std::string start = config::get<bts::api::md::MarketDataConfig>()
.cboe()
.at(0)
.databento_settings()
.bbo_start_time();
if(start != ""){
for (std::vector vec : all_sec) {
client.Subscribe(
vec, databento::Schema::Mbp1, databento::SType::RawSymbol, start);
client.Subscribe(
vec, databento::Schema::Trades, databento::SType::RawSymbol,start);
}
}else{
for (std::vector vec : all_sec) {
client.Subscribe(
vec, databento::Schema::Mbp1, databento::SType::RawSymbol);
client.Subscribe(
vec, databento::Schema::Trades, databento::SType::RawSymbol);
}
}
bool only_securities = config::get<bts::api::md::MarketDataConfig>()
.cboe()
.at(0)
.databento_settings()
.only_securities();
client.Start();
while (true) {
try {
auto *record = client.NextRecord(std::chrono::seconds{500});
if (first) {
LOG(md, Info, "first record");
first = false;
}
if (record == nullptr) {
LOG(md, Error, "Timed out waiting for data");
break;
}
if(only_securities)
continue;
if (record->Holds<databento::Mbp1Msg>()) {
process_mbp1(*record);
}
if (all_times.size() == 1001) {
log_time_statistics(all_times);
client.Stop();
}
// if (record->Holds<databento::InstrumentDefMsg>()) {
// process_security(*record);
// }
if (record->Holds<databento::TradeMsg>()) {
process_trades(*record);
}
} catch (const databento::TcpError &error) {
std::string err = error.what();
LOG(md, Error, "Caught TCP error: {} \nAttempting reconnect\n",
err);
client.Reconnect();
} catch (const databento::DbnResponseError &err) {
std::string msg = err.Message();
std::string what = err.what();
LOG(md, Error,
"Caught DbnResponseError what: {} ,msg:{} \nShutting down\n",
msg, what);
client.Stop();
// bts_assert(false);
} catch (const databento::Exception &excp) {
std::string err = excp.Message();
std::string err2 = excp.what();
LOG(md, Error,
"Caught unknown dadtabento::exception what: {}, msg: {} "
"\nShutting down\n",
err, err2);
client.Stop();
// bts_assert(false);
} catch (const std::exception &exc) {
std::string err = exc.what();
LOG(md, Error, "Caught unknown exception: {} \nShutting down\n",
err);
client.Stop();
// bts_assert(false);
}
}
}
}
} // namespace bts::md::DatabentoHandler