Files
spark/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp
T
John Colvin a26918efe9 refactor: Rename a few TitleCase .h files snake_case
* EnumMacros.h -> enum_macros.h
* TimeSpan.h ---> time_span.h
* TimeStamp.h --> time_stamp.h
2026-03-26 16:20:40 -04:00

465 lines
17 KiB
C++

#include "PyMarketDataSubscription.h"
#include "python_interop/md/md_bridge_impl.h"
#include "python_interop/md/py_book.h"
#include "python_interop/md/py_trade.h"
#include "python_interop/plugin_manager.h"
#include "python_interop/py_contract_specification.h"
#include "python_interop/py_security.h"
#include "python_interop/py_spread.h"
#include "spark/domain/SpreadSecurity.h"
#include "spark/domain/LegDefinition.h"
#include "spark/domain/Quantity.h"
#include "spark/domain/Side.h"
#include "python_interop/python_object.h"
#include "spark/api/domain/CallPut.pb.h"
#include "spark/api/domain/SecurityType.h"
#include "spark/api/domain/Venue.h"
#include "spark/api/md/PyMarketDataProviderConfig.pb.h"
#include "spark/core/time_span.h"
#include "spark/core/time_stamp.h"
#include "spark/domain/AggressorSide.h"
#include "spark/domain/FutureSecurity.h"
#include "spark/domain/Security.h"
#include "spark/domain/SecurityID.h"
#include "spark/domain/SpotSecurity.h"
#include "spark/domain/TradeRecord.h"
#include "spark/domain_model/SecurityType.h"
#include "spark/io/json_formatter.h"
#include "spark/logging/logging.h"
#include "spark/logging/trace_log.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/stopwatch.h"
#include <future>
#include <memory>
#include <optional>
#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
#include <utility>
#include <vector>
namespace bts::md {
#define _LOG(level, ...) LOG(md, level, __VA_ARGS__)
#define INFO(...) _LOG(bts::Info, __VA_ARGS__)
#define ERROR(...) _LOG(bts::Error, __VA_ARGS__)
#define _LOG_TRACE(level, name) \
LOG_TRACE(md, level, std::string("py_md_sub"), std::string(name))
#define INFO_TRACE(name) _LOG_TRACE(bts::Info, name)
#define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name)
#define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name)
#define INFO_TRACE_PARAMS(name, params) \
LOG_TRACE_PARAMS( \
md, bts::Info, std::string("py_md_sub"), std::string(name), params)
#define LOG_DISPATCH_STATS(level, typ, n, dur) \
_LOG( \
level, \
R"([py_md_dispatch.json]{{ "typ": ")" typ \
R"(", "n_recs": {}, "dur": "{:.2f}ms", "ts": {} }})", \
n, dur.total_millis(), bts::core::TimeStamp::now_ns())
#define LOG_SECURITY(level, py_sec) \
LOG(md, level, R"([py_md_sec.json]{{ "sec": {}, "ts": {}}})", \
json_formatter(py_sec), bts::core::TimeStamp::now_ns())
#define LOG_BOOK(level, py_book) \
LOG(md, level, R"([py_md_book.json]{{ "bk": {}, "ts": {}}})", \
json_formatter(py_book), bts::core::TimeStamp::now_ns())
#define LOG_TRADE(level, py_trade) \
LOG(md, level, R"([py_md_trade.json]{{ "trd": {}, "ts": {}}})", \
json_formatter(py_trade), bts::core::TimeStamp::now_ns())
using bts::api::domain::CallPut;
using bts::api::domain::SecurityType;
using bts::api::domain::Venue;
using bts::api::md::PyMarketDataProviderConfig;
using bts::core::TimeSpan;
using bts::core::TimeStamp;
using bts::domain::AggressorSide;
using bts::domain::Book;
using bts::domain::FutureSecurity;
using bts::domain::SpreadSecurity;
using bts::domain::LegDefinition;
using bts::domain::Quantity;
using bts::domain::Side;
using bts::domain::OptionSecurity;
using bts::domain::Price;
using bts::domain::Security;
using bts::domain::SecurityID;
using bts::domain::SpotSecurity;
using bts::domain::TradeRecord;
using bts::io::json_formatter;
using bts::md::py_interop::PyBook;
using bts::md::py_interop::PyBookSide;
using bts::md::py_interop::PySecurity;
using bts::md::py_interop::PyTrade;
using bts::persistence::IDGenerator;
using std::optional;
using std::shared_ptr;
using std::size_t;
using std::string;
using std::thread;
using std::vector;
using Path = boost::filesystem::path;
struct TradeTuple {
PyTrade py_trade;
TradeRecord trade;
Venue venue;
};
struct PyMarketDataSubscription::PImpl {
PImpl(
PyMarketDataProviderConfig config,
IDGenerator *sec_id_store,
shared_ptr<python_interop::PluginManager> py_plugin_man)
: config(std::move(config)),
name(fmt::format("{}_md_pro", string{this->config.provider_type()})),
bridge_ptr(std::make_shared<py_interop::MdBridgeImpl>()),
bridge(*bridge_ptr), py_plugin_man(std::move(py_plugin_man)),
sec_id_store_(sec_id_store) {}
size_t dispatch_securities() {
// `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them
// again. We keep them as member variables to avoid (re)allocating memory in each callback.
securities_.clear();
security_ptrs_.clear();
std::shared_ptr<const PySecurity> py_sec_p;
while (bridge.pop_security(py_sec_p)) {
const auto &py_sec = *py_sec_p;
LOG_SECURITY(Info, py_sec);
auto sec = to_bts_security_(py_sec);
if (!sec) {
LOG(md, Error,
"Failed to convert security {} to bts::domain::Security",
py_sec.name());
LOG(md, Error,
R"([py_md_sec_parse_error.json]{{ "sec": {}, "ts": {}}})",
py_sec);
continue;
}
securities_.push_back(std::move(*sec));
}
const size_t n = securities_.size();
if (!securities_.empty()) {
// Populate vectors of security ptrs per each venue.
for (auto &sec : securities_) {
security_ptrs_.push_back(&sec);
}
// Pass securities to the sinks (per venue).
listener_->on_securities(security_ptrs_);
securities_.clear();
security_ptrs_.clear();
}
return n;
}
size_t dispatch_books() {
size_t n = 0;
std::shared_ptr<const PyBook> py_book_p;
while (bridge.pop_book(py_book_p)) {
const auto &py_book = *py_book_p;
LOG_BOOK(Diag, py_book);
bts_assert(book_.count_populated() == 0);
to_book_(py_book, book_);
listener_->on_book(book_);
book_.clear();
n++;
}
return n;
}
size_t dispatch_trades() {
// `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again.
// We keep them as member variables to avoid (re)allocating memory in each callback.
trades_.clear();
trade_ptrs_.clear();
std::shared_ptr<const PyTrade> py_trade_p;
while (bridge.pop_trade(py_trade_p)) {
const auto &py_trade = *py_trade_p;
LOG_TRADE(Diag, py_trade);
to_bts_trade_(py_trade, trades_.emplace_back());
}
const size_t n = trades_.size();
for (const auto &t : trades_) {
trade_ptrs_.push_back(&t);
}
listener_->on_trades(trade_ptrs_);
trades_.clear();
trade_ptrs_.clear();
return n;
}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener_ = std::move(listener);
LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
py_keep_going = true;
py_thread.emplace([&]() {
INFO_TRACE_PARAMS(
"run_py", json_formatter(std::map<string, string>{
{"name", name}, {"ev", "STARTING_PY_MD_PRO"}}));
utils::Stopwatch sw = utils::Stopwatch::start_new();
std::future<std::shared_ptr<
python_interop::PythonObjectT<bts::md::py_interop::MdProvider>>>
md_pro_fut = py_plugin_man->start_md_provider(
name, config.provider_type(), bridge_ptr);
md_pro_fut.wait();
INFO_TRACE_PARAMS(
"run_py",
json_formatter(std::map<string, string>{
{"name", name},
{"ev", "STARTED_PY_MD_PRO"},
{"dur",
fmt::format("{:.3f}ms", sw.elapsed().total_millis())}}));
});
cpp_keep_going = true;
cpp_thread.emplace([&]() {
static const TimeSpan sec_warn_thresh = TimeSpan::from_millis(100);
static const TimeSpan bk_warn_thresh = TimeSpan::from_millis(30);
static const TimeSpan trd_warn_thresh = TimeSpan::from_millis(30);
utils::Stopwatch sw;
TimeSpan dur;
size_t n;
while (cpp_keep_going) {
// Dispatch securities
sw.restart();
n = dispatch_securities();
dur = sw.elapsed();
if (n > 0) {
LOG_DISPATCH_STATS(
dur > sec_warn_thresh ? Warn : Info, "sec", n, dur);
}
// Dispatch books
sw.restart();
n = dispatch_books();
dur = sw.elapsed();
if (n > 0) {
LOG_DISPATCH_STATS(
dur > bk_warn_thresh ? Warn : Diag, "bk", n, dur);
}
// Dispatch trades
sw.restart();
n = dispatch_trades();
dur = sw.elapsed();
if (n > 0) {
LOG_DISPATCH_STATS(
dur > trd_warn_thresh ? Warn : Diag, "trd", n, dur);
}
}
});
}
void stop() {
py_keep_going = false;
if (py_thread) {
py_thread->join();
py_thread.reset();
}
cpp_keep_going = false;
if (cpp_thread) {
cpp_thread->join();
cpp_thread.reset();
}
}
private:
PyMarketDataProviderConfig config;
std::string name;
std::shared_ptr<py_interop::MdBridgeImpl> bridge_ptr;
py_interop::MdBridgeImpl &bridge;
optional<thread> py_thread;
std::shared_ptr<python_interop::PluginManager> py_plugin_man;
volatile bool py_keep_going = false;
optional<thread> cpp_thread;
volatile bool cpp_keep_going = false;
shared_ptr<IMarketDataListener> listener_;
shared_ptr<IDGenerator> sec_id_store_;
// The main store for securities.
vector<Security> securities_;
vector<const Security *> security_ptrs_;
Book book_;
// Since the python md provider isn't tethered to a single venue, we
// must know the venue for each individual trade.
vector<TradeRecord> trades_;
vector<const TradeRecord *> trade_ptrs_;
SecurityID get_sec_id_(
Venue venue, SecurityType sec_type, const string &sec_name) {
auto sec_id = sec_id_store_->get_id(venue, sec_type, sec_name, true);
bts_assert_msgf(
sec_id,
"Failed to get security id for py_book. venue: {}, type: {}, name: "
"{}",
venue, sec_type, sec_name);
return SecurityID{*sec_id};
}
optional<Security> to_bts_security_(const PySecurity &py_sec) {
// We can't let python provider assign the id, because it is unaware of other
// md providers, so can't possibly guarentee a unique id.
bts_assert(py_sec.id() == 0);
Security sec;
auto &sec_comm = sec.security_common;
if (!venue_parse_smart(py_sec.venue(), &sec_comm.venue)) {
LOG(md, Error, "Failed to parse venue from '{}'", py_sec.venue());
return std::nullopt;
}
sec_comm.exchange_sec_id = py_sec.exchange_id();
sec_comm.currency = py_sec.currency();
sec_comm.asset = py_sec.asset();
sec_comm.group = py_sec.group();
sec_comm.symbol = py_sec.asset();
if (sec_comm.symbol.empty()) {
sec_comm.symbol = py_sec.group();
}
const py_interop::PyContractSpecification &spec = py_sec.contract_spec();
if (spec.price_infos().size() > 0) {
bts_assert(spec.price_infos().size() == 1);
const py_interop::PyContractPriceInfo &price_info =
spec.price_infos()[0];
bts_assert(price_info.tick_unit() > 0 && price_info.tick_base() > 0);
sec_comm.min_price_increment =
price_info.tick_unit() / price_info.tick_base();
if (price_info.cash_per_point() > 0) {
sec_comm.cash_per_point = price_info.cash_per_point();
}
}
if (spec.quantity_infos().size() > 0) {
bts_assert(spec.quantity_infos().size() == 1);
const py_interop::PyContractQuantityInfo &qty_info =
spec.quantity_infos()[0];
bts_assert(qty_info.tick_unit() > 0 && qty_info.tick_base() > 0);
sec_comm.min_quantity_increment =
qty_info.tick_unit() / qty_info.tick_base();
sec_comm.min_quantity =
std::max(qty_info.min_value(), sec_comm.min_quantity_increment);
}
if (spec.notional_quantity_infos().size() > 0) {
const auto &notional_qty_info = spec.notional_quantity_infos()[0];
sec_comm.min_notional_value = notional_qty_info.min_value();
}
sec_comm.desc = py_sec.name();
if (py_sec.is_spot()) {
sec.security = SpotSecurity{};
} else if (py_sec.is_future()) {
FutureSecurity fut;
fut.expiration =
TimeStamp::from_iso_string(py_sec.future().expiration());
sec.security = std::move(fut);
} else if (py_sec.is_option()) {
OptionSecurity opt;
const auto &py_opt = py_sec.option();
opt.expiration = TimeStamp::from_iso_string(py_opt.expiration());
opt.call_put = py_opt.is_call() ? CallPut::CALL : CallPut::PUT;
opt.strike = Price::from_double(py_opt.strike());
sec.security = std::move(opt);
} else if (py_sec.is_spread()) {
SpreadSecurity spread;
const auto &py_spread = py_sec.spread();
// Convert Python spread legs to C++ LegDefinition
for (const auto &leg_name : py_spread.legs()) {
LOG(md, Info, "Processing spread leg: {}", leg_name);
// Use the existing security resolution mechanism
// Each leg should be resolved using the same venue and appropriate security type
// For now, we'll assume the leg is a future (most common for spreads)
// You may need to adjust this based on your specific leg types
auto leg_sec_id = get_sec_id_(sec_comm.venue, SecurityType::SECURITY_TYPE_FUTURE, leg_name);
auto quantity = bts::domain::Quantity(1); // Default quantity
auto side = bts::domain::Side::Value::Buy; // Default side
bts::domain::LegDefinition leg_d{
leg_sec_id,
quantity,
side
};
spread.legs.push_back(leg_d);
}
LOG(md, Info,
"Created spread with {} legs. Spread type: {}, expiration: {}",
py_spread.legs().size(),
py_spread.spread_type(),
py_spread.expiration());
sec.security = std::move(spread);
} else {
LOG(md, Error,
"PySecurity has incomplete definition (not spot, future, "
"option or spread)");
return std::nullopt;
}
sec_comm.id =
get_sec_id_(sec_comm.venue, sec.type(), sec_comm.exchange_sec_id);
return sec;
}
void to_bts_trade_(const PyTrade &source, domain::TradeRecord &dest) {
Venue venue;
bts_assert(venue_parse_smart(source.venue(), &venue));
SecurityType sec_type;
bts_assert(security_type_parse_smart(source.security_type(), &sec_type));
dest.sec_id = get_sec_id_(venue, sec_type, source.exchange_security_id());
dest.time_stamp = core::TimeStamp::from_nanos(source.timestamp());
const string &agg_str = source.aggressor_side();
dest.agg_side = agg_str.starts_with('B') || agg_str.starts_with('b')
? AggressorSide::Buy
: AggressorSide::Sell;
dest.trade_price = Price::from_double(source.price());
*dest.trade_quantity = source.quantity();
}
void to_book_side_(const PyBookSide &source, domain::BookSide &dest) {
for (int i = 0; i < source.size(); i++) {
auto py_level = source.get_level(i);
const auto price = py_level.price();
const auto quantity = py_level.quantity();
bts_assert(!std::isnan(price));
bts_assert(quantity > 0);
*dest[i].total_quantity = quantity;
*dest[i].direct_quantity = quantity;
dest[i].price = domain::Price::from_double(price);
}
}
void to_book_(const PyBook &source, domain::Book &dest) {
to_book_side_(source.bids(), dest.bids());
to_book_side_(source.asks(), dest.asks());
Venue venue;
bts_assert(venue_parse_smart(source.venue(), &venue));
SecurityType type;
bts_assert(security_type_parse_smart(source.security_type(), &type));
dest.set_security_id(
get_sec_id_(venue, type, source.exchange_security_id()));
}
};
PyMarketDataSubscription::PyMarketDataSubscription(
PyMarketDataProviderConfig config,
IDGenerator *sec_id_store,
std::shared_ptr<python_interop::PluginManager> py_plugin_man)
: me(std::make_unique<PyMarketDataSubscription::PImpl>(
std::move(config), sec_id_store, std::move(py_plugin_man))) {}
PyMarketDataSubscription::~PyMarketDataSubscription() { me->stop(); }
void PyMarketDataSubscription::start(shared_ptr<IMarketDataListener> listener) {
me->start(std::move(listener));
}
void PyMarketDataSubscription::stop() { me->stop(); }
} // namespace bts::md