#include "PyMarketDataSubscription.h" #include "python_interop/md/md_bridge_impl.h" #include "python_interop/md/py_book.h" #include "python_interop/md/py_trade.h" #include "python_interop/plugin_manager.h" #include "python_interop/py_contract_specification.h" #include "python_interop/py_security.h" #include "python_interop/py_spread.h" #include "spark/domain/SpreadSecurity.h" #include "spark/domain/LegDefinition.h" #include "spark/domain/Quantity.h" #include "spark/domain/Side.h" #include "python_interop/python_object.h" #include "spark/api/domain/CallPut.pb.h" #include "spark/api/domain/SecurityType.h" #include "spark/api/domain/Venue.h" #include "spark/api/md/PyMarketDataProviderConfig.pb.h" #include "spark/core/time_span.h" #include "spark/core/time_stamp.h" #include "spark/domain/AggressorSide.h" #include "spark/domain/FutureSecurity.h" #include "spark/domain/Security.h" #include "spark/domain/SecurityID.h" #include "spark/domain/SpotSecurity.h" #include "spark/domain/TradeRecord.h" #include "spark/domain_model/SecurityType.h" #include "spark/io/json_formatter.h" #include "spark/logging/logging.h" #include "spark/logging/trace_log.h" #include "spark/md/IMarketDataListener.h" #include "spark/os/os.h" #include "spark/persistence/IDGenerator.h" #include "spark/utils/bts_assert.h" #include "spark/utils/stopwatch.h" #include #include #include #include #include #include #include namespace bts::md { #define _LOG(level, ...) LOG(md, level, __VA_ARGS__) #define INFO(...) _LOG(bts::Info, __VA_ARGS__) #define ERROR(...) _LOG(bts::Error, __VA_ARGS__) #define _LOG_TRACE(level, name) \ LOG_TRACE(md, level, std::string("py_md_sub"), std::string(name)) #define INFO_TRACE(name) _LOG_TRACE(bts::Info, name) #define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name) #define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name) #define INFO_TRACE_PARAMS(name, params) \ LOG_TRACE_PARAMS( \ md, bts::Info, std::string("py_md_sub"), std::string(name), params) #define LOG_DISPATCH_STATS(level, typ, n, dur) \ _LOG( \ level, \ R"([py_md_dispatch.json]{{ "typ": ")" typ \ R"(", "n_recs": {}, "dur": "{:.2f}ms", "ts": {} }})", \ n, dur.total_millis(), bts::core::TimeStamp::now_ns()) #define LOG_SECURITY(level, py_sec) \ LOG(md, level, R"([py_md_sec.json]{{ "sec": {}, "ts": {}}})", \ json_formatter(py_sec), bts::core::TimeStamp::now_ns()) #define LOG_BOOK(level, py_book) \ LOG(md, level, R"([py_md_book.json]{{ "bk": {}, "ts": {}}})", \ json_formatter(py_book), bts::core::TimeStamp::now_ns()) #define LOG_TRADE(level, py_trade) \ LOG(md, level, R"([py_md_trade.json]{{ "trd": {}, "ts": {}}})", \ json_formatter(py_trade), bts::core::TimeStamp::now_ns()) using bts::api::domain::CallPut; using bts::api::domain::SecurityType; using bts::api::domain::Venue; using bts::api::md::PyMarketDataProviderConfig; using bts::core::TimeSpan; using bts::core::TimeStamp; using bts::domain::AggressorSide; using bts::domain::Book; using bts::domain::FutureSecurity; using bts::domain::SpreadSecurity; using bts::domain::LegDefinition; using bts::domain::Quantity; using bts::domain::Side; using bts::domain::OptionSecurity; using bts::domain::Price; using bts::domain::Security; using bts::domain::SecurityID; using bts::domain::SpotSecurity; using bts::domain::TradeRecord; using bts::io::json_formatter; using bts::md::py_interop::PyBook; using bts::md::py_interop::PyBookSide; using bts::md::py_interop::PySecurity; using bts::md::py_interop::PyTrade; using bts::persistence::IDGenerator; using std::optional; using std::shared_ptr; using std::size_t; using std::string; using std::thread; using std::vector; using Path = boost::filesystem::path; struct TradeTuple { PyTrade py_trade; TradeRecord trade; Venue venue; }; struct PyMarketDataSubscription::PImpl { PImpl( PyMarketDataProviderConfig config, IDGenerator *sec_id_store, shared_ptr py_plugin_man) : config(std::move(config)), name(fmt::format("{}_md_pro", string{this->config.provider_type()})), bridge_ptr(std::make_shared()), bridge(*bridge_ptr), py_plugin_man(std::move(py_plugin_man)), sec_id_store_(sec_id_store) {} size_t dispatch_securities() { // `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them // again. We keep them as member variables to avoid (re)allocating memory in each callback. securities_.clear(); security_ptrs_.clear(); std::shared_ptr py_sec_p; while (bridge.pop_security(py_sec_p)) { const auto &py_sec = *py_sec_p; LOG_SECURITY(Info, py_sec); auto sec = to_bts_security_(py_sec); if (!sec) { LOG(md, Error, "Failed to convert security {} to bts::domain::Security", py_sec.name()); LOG(md, Error, R"([py_md_sec_parse_error.json]{{ "sec": {}, "ts": {}}})", py_sec); continue; } securities_.push_back(std::move(*sec)); } const size_t n = securities_.size(); if (!securities_.empty()) { // Populate vectors of security ptrs per each venue. for (auto &sec : securities_) { security_ptrs_.push_back(&sec); } // Pass securities to the sinks (per venue). listener_->on_securities(security_ptrs_); securities_.clear(); security_ptrs_.clear(); } return n; } size_t dispatch_books() { size_t n = 0; std::shared_ptr py_book_p; while (bridge.pop_book(py_book_p)) { const auto &py_book = *py_book_p; LOG_BOOK(Diag, py_book); bts_assert(book_.count_populated() == 0); to_book_(py_book, book_); listener_->on_book(book_); book_.clear(); n++; } return n; } size_t dispatch_trades() { // `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again. // We keep them as member variables to avoid (re)allocating memory in each callback. trades_.clear(); trade_ptrs_.clear(); std::shared_ptr py_trade_p; while (bridge.pop_trade(py_trade_p)) { const auto &py_trade = *py_trade_p; LOG_TRADE(Diag, py_trade); to_bts_trade_(py_trade, trades_.emplace_back()); } const size_t n = trades_.size(); for (const auto &t : trades_) { trade_ptrs_.push_back(&t); } listener_->on_trades(trade_ptrs_); trades_.clear(); trade_ptrs_.clear(); return n; } void start(shared_ptr listener) { bts_assert(listener != nullptr); this->listener_ = std::move(listener); LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}", os::get_thread_id(), os::get_thread_name()); py_keep_going = true; py_thread.emplace([&]() { INFO_TRACE_PARAMS( "run_py", json_formatter(std::map{ {"name", name}, {"ev", "STARTING_PY_MD_PRO"}})); utils::Stopwatch sw = utils::Stopwatch::start_new(); std::future>> md_pro_fut = py_plugin_man->start_md_provider( name, config.provider_type(), bridge_ptr); md_pro_fut.wait(); INFO_TRACE_PARAMS( "run_py", json_formatter(std::map{ {"name", name}, {"ev", "STARTED_PY_MD_PRO"}, {"dur", fmt::format("{:.3f}ms", sw.elapsed().total_millis())}})); }); cpp_keep_going = true; cpp_thread.emplace([&]() { static const TimeSpan sec_warn_thresh = TimeSpan::from_millis(100); static const TimeSpan bk_warn_thresh = TimeSpan::from_millis(30); static const TimeSpan trd_warn_thresh = TimeSpan::from_millis(30); utils::Stopwatch sw; TimeSpan dur; size_t n; while (cpp_keep_going) { // Dispatch securities sw.restart(); n = dispatch_securities(); dur = sw.elapsed(); if (n > 0) { LOG_DISPATCH_STATS( dur > sec_warn_thresh ? Warn : Info, "sec", n, dur); } // Dispatch books sw.restart(); n = dispatch_books(); dur = sw.elapsed(); if (n > 0) { LOG_DISPATCH_STATS( dur > bk_warn_thresh ? Warn : Diag, "bk", n, dur); } // Dispatch trades sw.restart(); n = dispatch_trades(); dur = sw.elapsed(); if (n > 0) { LOG_DISPATCH_STATS( dur > trd_warn_thresh ? Warn : Diag, "trd", n, dur); } } }); } void stop() { py_keep_going = false; if (py_thread) { py_thread->join(); py_thread.reset(); } cpp_keep_going = false; if (cpp_thread) { cpp_thread->join(); cpp_thread.reset(); } } private: PyMarketDataProviderConfig config; std::string name; std::shared_ptr bridge_ptr; py_interop::MdBridgeImpl &bridge; optional py_thread; std::shared_ptr py_plugin_man; volatile bool py_keep_going = false; optional cpp_thread; volatile bool cpp_keep_going = false; shared_ptr listener_; shared_ptr sec_id_store_; // The main store for securities. vector securities_; vector security_ptrs_; Book book_; // Since the python md provider isn't tethered to a single venue, we // must know the venue for each individual trade. vector trades_; vector trade_ptrs_; SecurityID get_sec_id_( Venue venue, SecurityType sec_type, const string &sec_name) { auto sec_id = sec_id_store_->get_id(venue, sec_type, sec_name, true); bts_assert_msgf( sec_id, "Failed to get security id for py_book. venue: {}, type: {}, name: " "{}", venue, sec_type, sec_name); return SecurityID{*sec_id}; } optional to_bts_security_(const PySecurity &py_sec) { // We can't let python provider assign the id, because it is unaware of other // md providers, so can't possibly guarentee a unique id. bts_assert(py_sec.id() == 0); Security sec; auto &sec_comm = sec.security_common; if (!venue_parse_smart(py_sec.venue(), &sec_comm.venue)) { LOG(md, Error, "Failed to parse venue from '{}'", py_sec.venue()); return std::nullopt; } sec_comm.exchange_sec_id = py_sec.exchange_id(); sec_comm.currency = py_sec.currency(); sec_comm.asset = py_sec.asset(); sec_comm.group = py_sec.group(); sec_comm.symbol = py_sec.asset(); if (sec_comm.symbol.empty()) { sec_comm.symbol = py_sec.group(); } const py_interop::PyContractSpecification &spec = py_sec.contract_spec(); if (spec.price_infos().size() > 0) { bts_assert(spec.price_infos().size() == 1); const py_interop::PyContractPriceInfo &price_info = spec.price_infos()[0]; bts_assert(price_info.tick_unit() > 0 && price_info.tick_base() > 0); sec_comm.min_price_increment = price_info.tick_unit() / price_info.tick_base(); if (price_info.cash_per_point() > 0) { sec_comm.cash_per_point = price_info.cash_per_point(); } } if (spec.quantity_infos().size() > 0) { bts_assert(spec.quantity_infos().size() == 1); const py_interop::PyContractQuantityInfo &qty_info = spec.quantity_infos()[0]; bts_assert(qty_info.tick_unit() > 0 && qty_info.tick_base() > 0); sec_comm.min_quantity_increment = qty_info.tick_unit() / qty_info.tick_base(); sec_comm.min_quantity = std::max(qty_info.min_value(), sec_comm.min_quantity_increment); } if (spec.notional_quantity_infos().size() > 0) { const auto ¬ional_qty_info = spec.notional_quantity_infos()[0]; sec_comm.min_notional_value = notional_qty_info.min_value(); } sec_comm.desc = py_sec.name(); if (py_sec.is_spot()) { sec.security = SpotSecurity{}; } else if (py_sec.is_future()) { FutureSecurity fut; fut.expiration = TimeStamp::from_iso_string(py_sec.future().expiration()); sec.security = std::move(fut); } else if (py_sec.is_option()) { OptionSecurity opt; const auto &py_opt = py_sec.option(); opt.expiration = TimeStamp::from_iso_string(py_opt.expiration()); opt.call_put = py_opt.is_call() ? CallPut::CALL : CallPut::PUT; opt.strike = Price::from_double(py_opt.strike()); sec.security = std::move(opt); } else if (py_sec.is_spread()) { SpreadSecurity spread; const auto &py_spread = py_sec.spread(); // Convert Python spread legs to C++ LegDefinition for (const auto &leg_name : py_spread.legs()) { LOG(md, Info, "Processing spread leg: {}", leg_name); // Use the existing security resolution mechanism // Each leg should be resolved using the same venue and appropriate security type // For now, we'll assume the leg is a future (most common for spreads) // You may need to adjust this based on your specific leg types auto leg_sec_id = get_sec_id_(sec_comm.venue, SecurityType::SECURITY_TYPE_FUTURE, leg_name); auto quantity = bts::domain::Quantity(1); // Default quantity auto side = bts::domain::Side::Value::Buy; // Default side bts::domain::LegDefinition leg_d{ leg_sec_id, quantity, side }; spread.legs.push_back(leg_d); } LOG(md, Info, "Created spread with {} legs. Spread type: {}, expiration: {}", py_spread.legs().size(), py_spread.spread_type(), py_spread.expiration()); sec.security = std::move(spread); } else { LOG(md, Error, "PySecurity has incomplete definition (not spot, future, " "option or spread)"); return std::nullopt; } sec_comm.id = get_sec_id_(sec_comm.venue, sec.type(), sec_comm.exchange_sec_id); return sec; } void to_bts_trade_(const PyTrade &source, domain::TradeRecord &dest) { Venue venue; bts_assert(venue_parse_smart(source.venue(), &venue)); SecurityType sec_type; bts_assert(security_type_parse_smart(source.security_type(), &sec_type)); dest.sec_id = get_sec_id_(venue, sec_type, source.exchange_security_id()); dest.time_stamp = core::TimeStamp::from_nanos(source.timestamp()); const string &agg_str = source.aggressor_side(); dest.agg_side = agg_str.starts_with('B') || agg_str.starts_with('b') ? AggressorSide::Buy : AggressorSide::Sell; dest.trade_price = Price::from_double(source.price()); *dest.trade_quantity = source.quantity(); } void to_book_side_(const PyBookSide &source, domain::BookSide &dest) { for (int i = 0; i < source.size(); i++) { auto py_level = source.get_level(i); const auto price = py_level.price(); const auto quantity = py_level.quantity(); bts_assert(!std::isnan(price)); bts_assert(quantity > 0); *dest[i].total_quantity = quantity; *dest[i].direct_quantity = quantity; dest[i].price = domain::Price::from_double(price); } } void to_book_(const PyBook &source, domain::Book &dest) { to_book_side_(source.bids(), dest.bids()); to_book_side_(source.asks(), dest.asks()); Venue venue; bts_assert(venue_parse_smart(source.venue(), &venue)); SecurityType type; bts_assert(security_type_parse_smart(source.security_type(), &type)); dest.set_security_id( get_sec_id_(venue, type, source.exchange_security_id())); } }; PyMarketDataSubscription::PyMarketDataSubscription( PyMarketDataProviderConfig config, IDGenerator *sec_id_store, std::shared_ptr py_plugin_man) : me(std::make_unique( std::move(config), sec_id_store, std::move(py_plugin_man))) {} PyMarketDataSubscription::~PyMarketDataSubscription() { me->stop(); } void PyMarketDataSubscription::start(shared_ptr listener) { me->start(std::move(listener)); } void PyMarketDataSubscription::stop() { me->stop(); } } // namespace bts::md