Also moved a few logging deps from lib_spark -> system: * bts_chrono.h * bts_no_copy.h * defer.h * dispatcher.h/cpp * file_utils.h/cpp * type_name.h
250 lines
9.3 KiB
C++
250 lines
9.3 KiB
C++
#include "PyMarketDataSubscription.h"
|
|
#include "domain/future_security.h"
|
|
#include "domain/protobufs/aggressor_side.pb.h"
|
|
#include "domain/protobufs/call_put.pb.h"
|
|
#include "domain/protobufs/venue.h"
|
|
#include "domain/security.h"
|
|
#include "domain/security_id.h"
|
|
#include "domain/spot_security.h"
|
|
#include "domain/trade_record.h"
|
|
#include "logging/logging.h"
|
|
#include "python_md/md_sink.h"
|
|
#include "python_md/py_book.h"
|
|
#include "python_md/py_md_executor.h"
|
|
#include "python_md/py_security.h"
|
|
#include "python_md/py_security_list.h"
|
|
#include "python_md/py_trade.h"
|
|
#include "spark/md/IMarketDataListener.h"
|
|
#include "spark/os/os.h"
|
|
#include "spark/persistence/IDGenerator.h"
|
|
#include "spark/utils/bts_assert.h"
|
|
#include "system/time_stamp.h"
|
|
|
|
#include <memory>
|
|
#include <optional>
|
|
#include <pybind11/embed.h>
|
|
#include <pybind11/pybind11.h>
|
|
#include <vector>
|
|
|
|
namespace bts::md {
|
|
|
|
#define AGGRESSOR_SIDE(x) bts::domain::protobufs::AGGRESSOR_SIDE_##x
|
|
|
|
using bts::domain::SecurityID;
|
|
using bts::md::py_interop::PyBook;
|
|
using bts::md::py_interop::PyBookList;
|
|
using bts::md::py_interop::PyBookSide;
|
|
using bts::md::py_interop::PyMdExecutor;
|
|
using bts::md::py_interop::PySecurityList;
|
|
using bts::md::py_interop::PyTrade;
|
|
using bts::md::py_interop::PyTradeList;
|
|
using bts::persistence::IDGenerator;
|
|
using std::shared_ptr;
|
|
using std::string;
|
|
using std::vector;
|
|
|
|
using Path = boost::filesystem::path;
|
|
|
|
struct TradeTuple {
|
|
py_interop::PyTrade py_trade;
|
|
domain::TradeRecord trade;
|
|
domain::protobufs::Venue venue;
|
|
};
|
|
|
|
struct PyMarketDataSubscription::PImpl : private py_interop::MdSink {
|
|
PImpl(
|
|
string py_plugin, vector<string> module_paths, IDGenerator *sec_id_store)
|
|
: py_plugin_(std::move(py_plugin)),
|
|
module_paths_(std::move(module_paths)),
|
|
py_md_exec_(py_plugin_, module_paths_, this),
|
|
sec_id_store_(sec_id_store) {}
|
|
|
|
void start(shared_ptr<IMarketDataListener> listener) {
|
|
bts_assert(listener != nullptr);
|
|
this->listener_ = std::move(listener);
|
|
LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}",
|
|
os::get_thread_id(), os::get_thread_name());
|
|
py_md_exec_.start();
|
|
}
|
|
|
|
void stop() { py_md_exec_.stop(); }
|
|
|
|
private:
|
|
string py_plugin_;
|
|
vector<string> module_paths_;
|
|
PyMdExecutor py_md_exec_;
|
|
shared_ptr<IMarketDataListener> listener_;
|
|
|
|
IDGenerator *sec_id_store_;
|
|
// The main store for securities.
|
|
vector<domain::Security> securities_;
|
|
vector<const domain::Security *> security_ptrs_;
|
|
domain::Book book_;
|
|
// Since the python md provider isn't tethered to a single venue, we
|
|
// must know the venue for each individual trade.
|
|
vector<domain::TradeRecord> trades_;
|
|
vector<const domain::TradeRecord *> trade_ptrs_;
|
|
|
|
void on_securities(const PySecurityList &py_securities) override {
|
|
LOG(md, Info, "Received securities from python plugin. count: {}",
|
|
py_securities.size());
|
|
// `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them
|
|
// again. We keep them as member variables to avoid (re)allocating memory in each callback.
|
|
securities_.clear();
|
|
security_ptrs_.clear();
|
|
for (const auto &py_sec : py_securities.items()) {
|
|
LOG(md, Info, "PyMarketDataSubscription adding security {}",
|
|
py_sec.name());
|
|
auto sec = to_bts_security_(py_sec);
|
|
if (!sec) {
|
|
LOG(md, Error,
|
|
"Failed to convert security {} to bts::domain::Security",
|
|
py_sec.name());
|
|
continue;
|
|
}
|
|
securities_.push_back(std::move(*sec));
|
|
}
|
|
// Populate vectors of security ptrs per each venue.
|
|
for (auto &sec : securities_) {
|
|
security_ptrs_.push_back(&sec);
|
|
}
|
|
// Pass securities to the sinks (per venue).
|
|
listener_->on_securities(security_ptrs_);
|
|
securities_.clear();
|
|
security_ptrs_.clear();
|
|
}
|
|
|
|
void on_books(const PyBookList &py_books) override {
|
|
for (auto &py_book : py_books.items()) {
|
|
bts_assert(book_.count_populated() == 0);
|
|
to_book_(py_book, book_);
|
|
listener_->on_book(book_);
|
|
book_.clear();
|
|
}
|
|
}
|
|
|
|
void on_trades(const PyTradeList &py_trades) override {
|
|
LOG(md, Info, "Received trades from python plugin. count: {}",
|
|
py_trades.size());
|
|
// `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again.
|
|
// We keep them as member variables to avoid (re)allocating memory in each callback.
|
|
trades_.clear();
|
|
trade_ptrs_.clear();
|
|
for (const auto &py_trade : py_trades.items()) {
|
|
LOG(md, Info, "PyMarketDataSubscription received trade {}",
|
|
py_trade.to_string());
|
|
to_bts_trade_(py_trade, trades_.emplace_back());
|
|
}
|
|
for (const auto &t : trades_) {
|
|
trade_ptrs_.push_back(&t);
|
|
}
|
|
listener_->on_trades(trade_ptrs_);
|
|
trades_.clear();
|
|
trade_ptrs_.clear();
|
|
}
|
|
|
|
SecurityID get_sec_id_(const string &venue_name, const string &sec_name) {
|
|
domain::protobufs::Venue venue;
|
|
// Although an overload of 'get_id' accepts a string, we parse `venue_name` to the enum to
|
|
// normalize different spellings of the same venue. For example, "deribt", "DERIBIT", and
|
|
// "VENUE_deribit" all represent the VENUE_DERIBIT enum (see `venue_parse_smart` for details).
|
|
bts_assert(domain::protobufs::venue_parse_smart(venue_name, &venue));
|
|
auto sec_id = sec_id_store_->get_id(venue, sec_name, true);
|
|
bts_assert_msgf(
|
|
sec_id, "Failed to get security id for py_book. venue: {}; name: {}",
|
|
venue, sec_name);
|
|
return SecurityID{*sec_id};
|
|
}
|
|
|
|
std::optional<domain::Security> to_bts_security_(
|
|
const py_interop::PySecurity &py_sec) {
|
|
// We can't let python provider assign the id, because it is unaware of other
|
|
// md providers, so can't possibly guarentee a unique id.
|
|
bts_assert(py_sec.id() == 0);
|
|
domain::Security sec;
|
|
auto &sec_comm = sec.security_common;
|
|
if (!domain::protobufs::venue_parse_smart(
|
|
py_sec.venue(), &sec_comm.venue)) {
|
|
LOG(md, Error, "Failed to parse venue from '{}'", py_sec.venue());
|
|
return std::nullopt;
|
|
}
|
|
sec_comm.id = get_sec_id_(py_sec.venue(), py_sec.exchange_id());
|
|
sec_comm.exchange_sec_id = py_sec.exchange_id();
|
|
sec_comm.currency = py_sec.currency();
|
|
sec_comm.asset = py_sec.asset();
|
|
sec_comm.group = py_sec.group();
|
|
sec_comm.symbol = py_sec.asset();
|
|
if (sec_comm.symbol.empty()) {
|
|
sec_comm.symbol = py_sec.group();
|
|
}
|
|
sec_comm.desc = py_sec.name();
|
|
if (py_sec.is_spot()) {
|
|
sec.security = domain::SpotSecurity{};
|
|
} else if (py_sec.is_future()) {
|
|
domain::FutureSecurity bts_fut;
|
|
bts_fut.expiration =
|
|
core::TimeStamp::from_iso_string(py_sec.future().expiration());
|
|
sec.security = std::move(bts_fut);
|
|
} else if (py_sec.is_option()) {
|
|
domain::OptionSecurity opt;
|
|
const auto &py_opt = py_sec.option();
|
|
opt.expiration = core::TimeStamp::from_iso_string(py_opt.expiration());
|
|
opt.call_put = py_opt.is_call() ? domain::protobufs::CallPut::CALL
|
|
: domain::protobufs::CallPut::PUT;
|
|
opt.strike = domain::Price::from_double(py_opt.strike());
|
|
sec.security = std::move(opt);
|
|
} else {
|
|
LOG(md, Error,
|
|
"PySecurity has incomplete definition (not spot, future, or "
|
|
"option)");
|
|
return std::nullopt;
|
|
}
|
|
return sec;
|
|
}
|
|
|
|
void to_bts_trade_(const PyTrade &source, domain::TradeRecord &dest) {
|
|
dest.sec_id = get_sec_id_(source.venue(), source.exchange_security_id());
|
|
dest.time_stamp = core::TimeStamp::from_nanos(source.timestamp());
|
|
dest.agg_side = source.aggressor_side().starts_with('B') ||
|
|
source.aggressor_side().starts_with('b')
|
|
? AGGRESSOR_SIDE(BUY)
|
|
: AGGRESSOR_SIDE(SELL);
|
|
dest.trade_price = domain::Price::from_double(source.price());
|
|
*dest.trade_quantity = source.quantity();
|
|
}
|
|
|
|
void to_book_side_(const PyBookSide &source, domain::BookSide &dest) {
|
|
for (int i = 0; i < source.size(); i++) {
|
|
auto py_level = source.get_level(i);
|
|
const auto price = py_level.price();
|
|
const auto quantity = py_level.quantity();
|
|
bts_assert(!std::isnan(price));
|
|
bts_assert(quantity > 0);
|
|
*dest[i].total_quantity = quantity;
|
|
*dest[i].direct_quantity = quantity;
|
|
dest[i].price = domain::Price::from_double(price);
|
|
}
|
|
}
|
|
|
|
void to_book_(const PyBook &source, domain::Book &dest) {
|
|
to_book_side_(source.bids(), dest.bids());
|
|
to_book_side_(source.asks(), dest.asks());
|
|
dest.set_security_id(
|
|
get_sec_id_(source.venue(), source.exchange_security_id()));
|
|
}
|
|
};
|
|
|
|
PyMarketDataSubscription::PyMarketDataSubscription(
|
|
string py_plugin, vector<string> module_paths, IDGenerator *sec_id_store)
|
|
: me_(std::make_unique<PyMarketDataSubscription::PImpl>(
|
|
std::move(py_plugin), std::move(module_paths), sec_id_store)) {}
|
|
|
|
PyMarketDataSubscription::~PyMarketDataSubscription() { me_->stop(); }
|
|
|
|
void PyMarketDataSubscription::start(shared_ptr<IMarketDataListener> listener) {
|
|
me_->start(std::move(listener));
|
|
}
|
|
void PyMarketDataSubscription::stop() { me_->stop(); }
|
|
|
|
} // namespace bts::md
|