Files
spark/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp
T
John Colvin 1b9c4fba2f refactor: Move logging into its own lib
Also moved a few logging deps from lib_spark -> system:
* bts_chrono.h
* bts_no_copy.h
* defer.h
* dispatcher.h/cpp
* file_utils.h/cpp
* type_name.h
2024-09-18 14:02:40 +00:00

250 lines
9.3 KiB
C++

#include "PyMarketDataSubscription.h"
#include "domain/future_security.h"
#include "domain/protobufs/aggressor_side.pb.h"
#include "domain/protobufs/call_put.pb.h"
#include "domain/protobufs/venue.h"
#include "domain/security.h"
#include "domain/security_id.h"
#include "domain/spot_security.h"
#include "domain/trade_record.h"
#include "logging/logging.h"
#include "python_md/md_sink.h"
#include "python_md/py_book.h"
#include "python_md/py_md_executor.h"
#include "python_md/py_security.h"
#include "python_md/py_security_list.h"
#include "python_md/py_trade.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/os/os.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/utils/bts_assert.h"
#include "system/time_stamp.h"
#include <memory>
#include <optional>
#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
#include <vector>
namespace bts::md {
#define AGGRESSOR_SIDE(x) bts::domain::protobufs::AGGRESSOR_SIDE_##x
using bts::domain::SecurityID;
using bts::md::py_interop::PyBook;
using bts::md::py_interop::PyBookList;
using bts::md::py_interop::PyBookSide;
using bts::md::py_interop::PyMdExecutor;
using bts::md::py_interop::PySecurityList;
using bts::md::py_interop::PyTrade;
using bts::md::py_interop::PyTradeList;
using bts::persistence::IDGenerator;
using std::shared_ptr;
using std::string;
using std::vector;
using Path = boost::filesystem::path;
struct TradeTuple {
py_interop::PyTrade py_trade;
domain::TradeRecord trade;
domain::protobufs::Venue venue;
};
struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
PImpl(
string py_plugin, vector<string> module_paths, IDGenerator *sec_id_store)
: py_plugin_(std::move(py_plugin)),
module_paths_(std::move(module_paths)),
py_md_exec_(py_plugin_, module_paths_, this),
sec_id_store_(sec_id_store) {}
void start(shared_ptr<IMarketDataListener> listener) {
bts_assert(listener != nullptr);
this->listener_ = std::move(listener);
LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}",
os::get_thread_id(), os::get_thread_name());
py_md_exec_.start();
}
void stop() { py_md_exec_.stop(); }
private:
string py_plugin_;
vector<string> module_paths_;
PyMdExecutor py_md_exec_;
shared_ptr<IMarketDataListener> listener_;
IDGenerator *sec_id_store_;
// The main store for securities.
vector<domain::Security> securities_;
vector<const domain::Security *> security_ptrs_;
domain::Book book_;
// Since the python md provider isn't tethered to a single venue, we
// must know the venue for each individual trade.
vector<domain::TradeRecord> trades_;
vector<const domain::TradeRecord *> trade_ptrs_;
void on_securities(const PySecurityList &py_securities) override {
LOG(md, Info, "Received securities from python plugin. count: {}",
py_securities.size());
// `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them
// again. We keep them as member variables to avoid (re)allocating memory in each callback.
securities_.clear();
security_ptrs_.clear();
for (const auto &py_sec : py_securities.items()) {
LOG(md, Info, "PyMarketDataSubscription adding security {}",
py_sec.name());
auto sec = to_bts_security_(py_sec);
if (!sec) {
LOG(md, Error,
"Failed to convert security {} to bts::domain::Security",
py_sec.name());
continue;
}
securities_.push_back(std::move(*sec));
}
// Populate vectors of security ptrs per each venue.
for (auto &sec : securities_) {
security_ptrs_.push_back(&sec);
}
// Pass securities to the sinks (per venue).
listener_->on_securities(security_ptrs_);
securities_.clear();
security_ptrs_.clear();
}
void on_books(const PyBookList &py_books) override {
for (auto &py_book : py_books.items()) {
bts_assert(book_.count_populated() == 0);
to_book_(py_book, book_);
listener_->on_book(book_);
book_.clear();
}
}
void on_trades(const PyTradeList &py_trades) override {
LOG(md, Info, "Received trades from python plugin. count: {}",
py_trades.size());
// `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again.
// We keep them as member variables to avoid (re)allocating memory in each callback.
trades_.clear();
trade_ptrs_.clear();
for (const auto &py_trade : py_trades.items()) {
LOG(md, Info, "PyMarketDataSubscription received trade {}",
py_trade.to_string());
to_bts_trade_(py_trade, trades_.emplace_back());
}
for (const auto &t : trades_) {
trade_ptrs_.push_back(&t);
}
listener_->on_trades(trade_ptrs_);
trades_.clear();
trade_ptrs_.clear();
}
SecurityID get_sec_id_(const string &venue_name, const string &sec_name) {
domain::protobufs::Venue venue;
// Although an overload of 'get_id' accepts a string, we parse `venue_name` to the enum to
// normalize different spellings of the same venue. For example, "deribt", "DERIBIT", and
// "VENUE_deribit" all represent the VENUE_DERIBIT enum (see `venue_parse_smart` for details).
bts_assert(domain::protobufs::venue_parse_smart(venue_name, &venue));
auto sec_id = sec_id_store_->get_id(venue, sec_name, true);
bts_assert_msgf(
sec_id, "Failed to get security id for py_book. venue: {}; name: {}",
venue, sec_name);
return SecurityID{*sec_id};
}
std::optional<domain::Security> to_bts_security_(
const py_interop::PySecurity &py_sec) {
// We can't let python provider assign the id, because it is unaware of other
// md providers, so can't possibly guarentee a unique id.
bts_assert(py_sec.id() == 0);
domain::Security sec;
auto &sec_comm = sec.security_common;
if (!domain::protobufs::venue_parse_smart(
py_sec.venue(), &sec_comm.venue)) {
LOG(md, Error, "Failed to parse venue from '{}'", py_sec.venue());
return std::nullopt;
}
sec_comm.id = get_sec_id_(py_sec.venue(), py_sec.exchange_id());
sec_comm.exchange_sec_id = py_sec.exchange_id();
sec_comm.currency = py_sec.currency();
sec_comm.asset = py_sec.asset();
sec_comm.group = py_sec.group();
sec_comm.symbol = py_sec.asset();
if (sec_comm.symbol.empty()) {
sec_comm.symbol = py_sec.group();
}
sec_comm.desc = py_sec.name();
if (py_sec.is_spot()) {
sec.security = domain::SpotSecurity{};
} else if (py_sec.is_future()) {
domain::FutureSecurity bts_fut;
bts_fut.expiration =
core::TimeStamp::from_iso_string(py_sec.future().expiration());
sec.security = std::move(bts_fut);
} else if (py_sec.is_option()) {
domain::OptionSecurity opt;
const auto &py_opt = py_sec.option();
opt.expiration = core::TimeStamp::from_iso_string(py_opt.expiration());
opt.call_put = py_opt.is_call() ? domain::protobufs::CallPut::CALL
: domain::protobufs::CallPut::PUT;
opt.strike = domain::Price::from_double(py_opt.strike());
sec.security = std::move(opt);
} else {
LOG(md, Error,
"PySecurity has incomplete definition (not spot, future, or "
"option)");
return std::nullopt;
}
return sec;
}
void to_bts_trade_(const PyTrade &source, domain::TradeRecord &dest) {
dest.sec_id = get_sec_id_(source.venue(), source.exchange_security_id());
dest.time_stamp = core::TimeStamp::from_nanos(source.timestamp());
dest.agg_side = source.aggressor_side().starts_with('B') ||
source.aggressor_side().starts_with('b')
? AGGRESSOR_SIDE(BUY)
: AGGRESSOR_SIDE(SELL);
dest.trade_price = domain::Price::from_double(source.price());
*dest.trade_quantity = source.quantity();
}
void to_book_side_(const PyBookSide &source, domain::BookSide &dest) {
for (int i = 0; i < source.size(); i++) {
auto py_level = source.get_level(i);
const auto price = py_level.price();
const auto quantity = py_level.quantity();
bts_assert(!std::isnan(price));
bts_assert(quantity > 0);
*dest[i].total_quantity = quantity;
*dest[i].direct_quantity = quantity;
dest[i].price = domain::Price::from_double(price);
}
}
void to_book_(const PyBook &source, domain::Book &dest) {
to_book_side_(source.bids(), dest.bids());
to_book_side_(source.asks(), dest.asks());
dest.set_security_id(
get_sec_id_(source.venue(), source.exchange_security_id()));
}
};
PyMarketDataSubscription::PyMarketDataSubscription(
string py_plugin, vector<string> module_paths, IDGenerator *sec_id_store)
: me_(std::make_unique<PyMarketDataSubscription::PImpl>(
std::move(py_plugin), std::move(module_paths), sec_id_store)) {}
PyMarketDataSubscription::~PyMarketDataSubscription() { me_->stop(); }
void PyMarketDataSubscription::start(shared_ptr<IMarketDataListener> listener) {
me_->start(std::move(listener));
}
void PyMarketDataSubscription::stop() { me_->stop(); }
} // namespace bts::md