#include "PyMarketDataSubscription.h" #include "domain/future_security.h" #include "domain/protobufs/aggressor_side.pb.h" #include "domain/protobufs/call_put.pb.h" #include "domain/protobufs/venue.h" #include "domain/security.h" #include "domain/security_id.h" #include "domain/spot_security.h" #include "domain/trade_record.h" #include "logging/logging.h" #include "python_md/md_sink.h" #include "python_md/py_book.h" #include "python_md/py_md_executor.h" #include "python_md/py_security.h" #include "python_md/py_security_list.h" #include "python_md/py_trade.h" #include "spark/md/IMarketDataListener.h" #include "spark/os/os.h" #include "spark/persistence/IDGenerator.h" #include "spark/utils/bts_assert.h" #include "system/time_stamp.h" #include #include #include #include #include namespace bts::md { #define AGGRESSOR_SIDE(x) bts::domain::protobufs::AGGRESSOR_SIDE_##x using bts::domain::SecurityID; using bts::md::py_interop::PyBook; using bts::md::py_interop::PyBookList; using bts::md::py_interop::PyBookSide; using bts::md::py_interop::PyMdExecutor; using bts::md::py_interop::PySecurityList; using bts::md::py_interop::PyTrade; using bts::md::py_interop::PyTradeList; using bts::persistence::IDGenerator; using std::shared_ptr; using std::string; using std::vector; using Path = boost::filesystem::path; struct TradeTuple { py_interop::PyTrade py_trade; domain::TradeRecord trade; domain::protobufs::Venue venue; }; struct PyMarketDataSubscription::PImpl : private py_interop::MdSink { PImpl( string py_plugin, vector module_paths, IDGenerator *sec_id_store) : py_plugin_(std::move(py_plugin)), module_paths_(std::move(module_paths)), py_md_exec_(py_plugin_, module_paths_, this), sec_id_store_(sec_id_store) {} void start(shared_ptr listener) { bts_assert(listener != nullptr); this->listener_ = std::move(listener); LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}", os::get_thread_id(), os::get_thread_name()); py_md_exec_.start(); } void stop() { py_md_exec_.stop(); } private: string py_plugin_; vector module_paths_; PyMdExecutor py_md_exec_; shared_ptr listener_; IDGenerator *sec_id_store_; // The main store for securities. vector securities_; vector security_ptrs_; domain::Book book_; // Since the python md provider isn't tethered to a single venue, we // must know the venue for each individual trade. vector trades_; vector trade_ptrs_; void on_securities(const PySecurityList &py_securities) override { LOG(md, Info, "Received securities from python plugin. count: {}", py_securities.size()); // `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them // again. We keep them as member variables to avoid (re)allocating memory in each callback. securities_.clear(); security_ptrs_.clear(); for (const auto &py_sec : py_securities.items()) { LOG(md, Info, "PyMarketDataSubscription adding security {}", py_sec.name()); auto sec = to_bts_security_(py_sec); if (!sec) { LOG(md, Error, "Failed to convert security {} to bts::domain::Security", py_sec.name()); continue; } securities_.push_back(std::move(*sec)); } // Populate vectors of security ptrs per each venue. for (auto &sec : securities_) { security_ptrs_.push_back(&sec); } // Pass securities to the sinks (per venue). listener_->on_securities(security_ptrs_); securities_.clear(); security_ptrs_.clear(); } void on_books(const PyBookList &py_books) override { for (auto &py_book : py_books.items()) { bts_assert(book_.count_populated() == 0); to_book_(py_book, book_); listener_->on_book(book_); book_.clear(); } } void on_trades(const PyTradeList &py_trades) override { LOG(md, Info, "Received trades from python plugin. count: {}", py_trades.size()); // `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again. // We keep them as member variables to avoid (re)allocating memory in each callback. trades_.clear(); trade_ptrs_.clear(); for (const auto &py_trade : py_trades.items()) { LOG(md, Info, "PyMarketDataSubscription received trade {}", py_trade.to_string()); to_bts_trade_(py_trade, trades_.emplace_back()); } for (const auto &t : trades_) { trade_ptrs_.push_back(&t); } listener_->on_trades(trade_ptrs_); trades_.clear(); trade_ptrs_.clear(); } SecurityID get_sec_id_(const string &venue_name, const string &sec_name) { domain::protobufs::Venue venue; // Although an overload of 'get_id' accepts a string, we parse `venue_name` to the enum to // normalize different spellings of the same venue. For example, "deribt", "DERIBIT", and // "VENUE_deribit" all represent the VENUE_DERIBIT enum (see `venue_parse_smart` for details). bts_assert(domain::protobufs::venue_parse_smart(venue_name, &venue)); auto sec_id = sec_id_store_->get_id(venue, sec_name, true); bts_assert_msgf( sec_id, "Failed to get security id for py_book. venue: {}; name: {}", venue, sec_name); return SecurityID{*sec_id}; } std::optional to_bts_security_( const py_interop::PySecurity &py_sec) { // We can't let python provider assign the id, because it is unaware of other // md providers, so can't possibly guarentee a unique id. bts_assert(py_sec.id() == 0); domain::Security sec; auto &sec_comm = sec.security_common; if (!domain::protobufs::venue_parse_smart( py_sec.venue(), &sec_comm.venue)) { LOG(md, Error, "Failed to parse venue from '{}'", py_sec.venue()); return std::nullopt; } sec_comm.id = get_sec_id_(py_sec.venue(), py_sec.exchange_id()); sec_comm.exchange_sec_id = py_sec.exchange_id(); sec_comm.currency = py_sec.currency(); sec_comm.asset = py_sec.asset(); sec_comm.group = py_sec.group(); sec_comm.symbol = py_sec.asset(); if (sec_comm.symbol.empty()) { sec_comm.symbol = py_sec.group(); } sec_comm.desc = py_sec.name(); if (py_sec.is_spot()) { sec.security = domain::SpotSecurity{}; } else if (py_sec.is_future()) { domain::FutureSecurity bts_fut; bts_fut.expiration = core::TimeStamp::from_iso_string(py_sec.future().expiration()); sec.security = std::move(bts_fut); } else if (py_sec.is_option()) { domain::OptionSecurity opt; const auto &py_opt = py_sec.option(); opt.expiration = core::TimeStamp::from_iso_string(py_opt.expiration()); opt.call_put = py_opt.is_call() ? domain::protobufs::CallPut::CALL : domain::protobufs::CallPut::PUT; opt.strike = domain::Price::from_double(py_opt.strike()); sec.security = std::move(opt); } else { LOG(md, Error, "PySecurity has incomplete definition (not spot, future, or " "option)"); return std::nullopt; } return sec; } void to_bts_trade_(const PyTrade &source, domain::TradeRecord &dest) { dest.sec_id = get_sec_id_(source.venue(), source.exchange_security_id()); dest.time_stamp = core::TimeStamp::from_nanos(source.timestamp()); dest.agg_side = source.aggressor_side().starts_with('B') || source.aggressor_side().starts_with('b') ? AGGRESSOR_SIDE(BUY) : AGGRESSOR_SIDE(SELL); dest.trade_price = domain::Price::from_double(source.price()); *dest.trade_quantity = source.quantity(); } void to_book_side_(const PyBookSide &source, domain::BookSide &dest) { for (int i = 0; i < source.size(); i++) { auto py_level = source.get_level(i); const auto price = py_level.price(); const auto quantity = py_level.quantity(); bts_assert(!std::isnan(price)); bts_assert(quantity > 0); *dest[i].total_quantity = quantity; *dest[i].direct_quantity = quantity; dest[i].price = domain::Price::from_double(price); } } void to_book_(const PyBook &source, domain::Book &dest) { to_book_side_(source.bids(), dest.bids()); to_book_side_(source.asks(), dest.asks()); dest.set_security_id( get_sec_id_(source.venue(), source.exchange_security_id())); } }; PyMarketDataSubscription::PyMarketDataSubscription( string py_plugin, vector module_paths, IDGenerator *sec_id_store) : me_(std::make_unique( std::move(py_plugin), std::move(module_paths), sec_id_store)) {} PyMarketDataSubscription::~PyMarketDataSubscription() { me_->stop(); } void PyMarketDataSubscription::start(shared_ptr listener) { me_->start(std::move(listener)); } void PyMarketDataSubscription::stop() { me_->stop(); } } // namespace bts::md