* EnumMacros.h -> enum_macros.h * TimeSpan.h ---> time_span.h * TimeStamp.h --> time_stamp.h
465 lines
17 KiB
C++
465 lines
17 KiB
C++
#include "PyMarketDataSubscription.h"
|
|
#include "python_interop/md/md_bridge_impl.h"
|
|
#include "python_interop/md/py_book.h"
|
|
#include "python_interop/md/py_trade.h"
|
|
#include "python_interop/plugin_manager.h"
|
|
#include "python_interop/py_contract_specification.h"
|
|
#include "python_interop/py_security.h"
|
|
#include "python_interop/py_spread.h"
|
|
#include "spark/domain/SpreadSecurity.h"
|
|
#include "spark/domain/LegDefinition.h"
|
|
#include "spark/domain/Quantity.h"
|
|
#include "spark/domain/Side.h"
|
|
#include "python_interop/python_object.h"
|
|
#include "spark/api/domain/CallPut.pb.h"
|
|
#include "spark/api/domain/SecurityType.h"
|
|
#include "spark/api/domain/Venue.h"
|
|
#include "spark/api/md/PyMarketDataProviderConfig.pb.h"
|
|
#include "spark/core/time_span.h"
|
|
#include "spark/core/time_stamp.h"
|
|
#include "spark/domain/AggressorSide.h"
|
|
#include "spark/domain/FutureSecurity.h"
|
|
#include "spark/domain/Security.h"
|
|
#include "spark/domain/SecurityID.h"
|
|
#include "spark/domain/SpotSecurity.h"
|
|
#include "spark/domain/TradeRecord.h"
|
|
#include "spark/domain_model/SecurityType.h"
|
|
#include "spark/io/json_formatter.h"
|
|
#include "spark/logging/logging.h"
|
|
#include "spark/logging/trace_log.h"
|
|
#include "spark/md/IMarketDataListener.h"
|
|
#include "spark/os/os.h"
|
|
#include "spark/persistence/IDGenerator.h"
|
|
#include "spark/utils/bts_assert.h"
|
|
#include "spark/utils/stopwatch.h"
|
|
|
|
#include <future>
|
|
#include <memory>
|
|
#include <optional>
|
|
#include <pybind11/embed.h>
|
|
#include <pybind11/pybind11.h>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
namespace bts::md {
|
|
|
|
#define _LOG(level, ...) LOG(md, level, __VA_ARGS__)
|
|
#define INFO(...) _LOG(bts::Info, __VA_ARGS__)
|
|
#define ERROR(...) _LOG(bts::Error, __VA_ARGS__)
|
|
|
|
#define _LOG_TRACE(level, name) \
|
|
LOG_TRACE(md, level, std::string("py_md_sub"), std::string(name))
|
|
#define INFO_TRACE(name) _LOG_TRACE(bts::Info, name)
|
|
#define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name)
|
|
#define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name)
|
|
|
|
#define INFO_TRACE_PARAMS(name, params) \
|
|
LOG_TRACE_PARAMS( \
|
|
md, bts::Info, std::string("py_md_sub"), std::string(name), params)
|
|
|
|
#define LOG_DISPATCH_STATS(level, typ, n, dur) \
|
|
_LOG( \
|
|
level, \
|
|
R"([py_md_dispatch.json]{{ "typ": ")" typ \
|
|
R"(", "n_recs": {}, "dur": "{:.2f}ms", "ts": {} }})", \
|
|
n, dur.total_millis(), bts::core::TimeStamp::now_ns())
|
|
|
|
#define LOG_SECURITY(level, py_sec) \
|
|
LOG(md, level, R"([py_md_sec.json]{{ "sec": {}, "ts": {}}})", \
|
|
json_formatter(py_sec), bts::core::TimeStamp::now_ns())
|
|
|
|
#define LOG_BOOK(level, py_book) \
|
|
LOG(md, level, R"([py_md_book.json]{{ "bk": {}, "ts": {}}})", \
|
|
json_formatter(py_book), bts::core::TimeStamp::now_ns())
|
|
|
|
#define LOG_TRADE(level, py_trade) \
|
|
LOG(md, level, R"([py_md_trade.json]{{ "trd": {}, "ts": {}}})", \
|
|
json_formatter(py_trade), bts::core::TimeStamp::now_ns())
|
|
|
|
using bts::api::domain::CallPut;
|
|
using bts::api::domain::SecurityType;
|
|
using bts::api::domain::Venue;
|
|
using bts::api::md::PyMarketDataProviderConfig;
|
|
using bts::core::TimeSpan;
|
|
using bts::core::TimeStamp;
|
|
using bts::domain::AggressorSide;
|
|
using bts::domain::Book;
|
|
using bts::domain::FutureSecurity;
|
|
using bts::domain::SpreadSecurity;
|
|
using bts::domain::LegDefinition;
|
|
using bts::domain::Quantity;
|
|
using bts::domain::Side;
|
|
using bts::domain::OptionSecurity;
|
|
using bts::domain::Price;
|
|
using bts::domain::Security;
|
|
using bts::domain::SecurityID;
|
|
using bts::domain::SpotSecurity;
|
|
using bts::domain::TradeRecord;
|
|
using bts::io::json_formatter;
|
|
using bts::md::py_interop::PyBook;
|
|
using bts::md::py_interop::PyBookSide;
|
|
using bts::md::py_interop::PySecurity;
|
|
using bts::md::py_interop::PyTrade;
|
|
using bts::persistence::IDGenerator;
|
|
using std::optional;
|
|
using std::shared_ptr;
|
|
using std::size_t;
|
|
using std::string;
|
|
using std::thread;
|
|
using std::vector;
|
|
|
|
using Path = boost::filesystem::path;
|
|
|
|
struct TradeTuple {
|
|
PyTrade py_trade;
|
|
TradeRecord trade;
|
|
Venue venue;
|
|
};
|
|
|
|
struct PyMarketDataSubscription::PImpl {
|
|
PImpl(
|
|
PyMarketDataProviderConfig config,
|
|
IDGenerator *sec_id_store,
|
|
shared_ptr<python_interop::PluginManager> py_plugin_man)
|
|
: config(std::move(config)),
|
|
name(fmt::format("{}_md_pro", string{this->config.provider_type()})),
|
|
bridge_ptr(std::make_shared<py_interop::MdBridgeImpl>()),
|
|
bridge(*bridge_ptr), py_plugin_man(std::move(py_plugin_man)),
|
|
sec_id_store_(sec_id_store) {}
|
|
|
|
size_t dispatch_securities() {
|
|
// `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them
|
|
// again. We keep them as member variables to avoid (re)allocating memory in each callback.
|
|
securities_.clear();
|
|
security_ptrs_.clear();
|
|
std::shared_ptr<const PySecurity> py_sec_p;
|
|
while (bridge.pop_security(py_sec_p)) {
|
|
const auto &py_sec = *py_sec_p;
|
|
LOG_SECURITY(Info, py_sec);
|
|
auto sec = to_bts_security_(py_sec);
|
|
if (!sec) {
|
|
LOG(md, Error,
|
|
"Failed to convert security {} to bts::domain::Security",
|
|
py_sec.name());
|
|
LOG(md, Error,
|
|
R"([py_md_sec_parse_error.json]{{ "sec": {}, "ts": {}}})",
|
|
py_sec);
|
|
continue;
|
|
}
|
|
securities_.push_back(std::move(*sec));
|
|
}
|
|
const size_t n = securities_.size();
|
|
if (!securities_.empty()) {
|
|
// Populate vectors of security ptrs per each venue.
|
|
for (auto &sec : securities_) {
|
|
security_ptrs_.push_back(&sec);
|
|
}
|
|
// Pass securities to the sinks (per venue).
|
|
listener_->on_securities(security_ptrs_);
|
|
securities_.clear();
|
|
security_ptrs_.clear();
|
|
}
|
|
return n;
|
|
}
|
|
|
|
size_t dispatch_books() {
|
|
size_t n = 0;
|
|
std::shared_ptr<const PyBook> py_book_p;
|
|
while (bridge.pop_book(py_book_p)) {
|
|
const auto &py_book = *py_book_p;
|
|
LOG_BOOK(Diag, py_book);
|
|
bts_assert(book_.count_populated() == 0);
|
|
to_book_(py_book, book_);
|
|
listener_->on_book(book_);
|
|
book_.clear();
|
|
n++;
|
|
}
|
|
return n;
|
|
}
|
|
|
|
size_t dispatch_trades() {
|
|
// `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again.
|
|
// We keep them as member variables to avoid (re)allocating memory in each callback.
|
|
trades_.clear();
|
|
trade_ptrs_.clear();
|
|
std::shared_ptr<const PyTrade> py_trade_p;
|
|
while (bridge.pop_trade(py_trade_p)) {
|
|
const auto &py_trade = *py_trade_p;
|
|
LOG_TRADE(Diag, py_trade);
|
|
to_bts_trade_(py_trade, trades_.emplace_back());
|
|
}
|
|
const size_t n = trades_.size();
|
|
for (const auto &t : trades_) {
|
|
trade_ptrs_.push_back(&t);
|
|
}
|
|
listener_->on_trades(trade_ptrs_);
|
|
trades_.clear();
|
|
trade_ptrs_.clear();
|
|
return n;
|
|
}
|
|
|
|
void start(shared_ptr<IMarketDataListener> listener) {
|
|
bts_assert(listener != nullptr);
|
|
this->listener_ = std::move(listener);
|
|
LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}",
|
|
os::get_thread_id(), os::get_thread_name());
|
|
py_keep_going = true;
|
|
py_thread.emplace([&]() {
|
|
INFO_TRACE_PARAMS(
|
|
"run_py", json_formatter(std::map<string, string>{
|
|
{"name", name}, {"ev", "STARTING_PY_MD_PRO"}}));
|
|
utils::Stopwatch sw = utils::Stopwatch::start_new();
|
|
std::future<std::shared_ptr<
|
|
python_interop::PythonObjectT<bts::md::py_interop::MdProvider>>>
|
|
md_pro_fut = py_plugin_man->start_md_provider(
|
|
name, config.provider_type(), bridge_ptr);
|
|
md_pro_fut.wait();
|
|
INFO_TRACE_PARAMS(
|
|
"run_py",
|
|
json_formatter(std::map<string, string>{
|
|
{"name", name},
|
|
{"ev", "STARTED_PY_MD_PRO"},
|
|
{"dur",
|
|
fmt::format("{:.3f}ms", sw.elapsed().total_millis())}}));
|
|
});
|
|
cpp_keep_going = true;
|
|
cpp_thread.emplace([&]() {
|
|
static const TimeSpan sec_warn_thresh = TimeSpan::from_millis(100);
|
|
static const TimeSpan bk_warn_thresh = TimeSpan::from_millis(30);
|
|
static const TimeSpan trd_warn_thresh = TimeSpan::from_millis(30);
|
|
utils::Stopwatch sw;
|
|
TimeSpan dur;
|
|
size_t n;
|
|
while (cpp_keep_going) {
|
|
// Dispatch securities
|
|
sw.restart();
|
|
n = dispatch_securities();
|
|
dur = sw.elapsed();
|
|
if (n > 0) {
|
|
LOG_DISPATCH_STATS(
|
|
dur > sec_warn_thresh ? Warn : Info, "sec", n, dur);
|
|
}
|
|
// Dispatch books
|
|
sw.restart();
|
|
n = dispatch_books();
|
|
dur = sw.elapsed();
|
|
if (n > 0) {
|
|
LOG_DISPATCH_STATS(
|
|
dur > bk_warn_thresh ? Warn : Diag, "bk", n, dur);
|
|
}
|
|
// Dispatch trades
|
|
sw.restart();
|
|
n = dispatch_trades();
|
|
dur = sw.elapsed();
|
|
if (n > 0) {
|
|
LOG_DISPATCH_STATS(
|
|
dur > trd_warn_thresh ? Warn : Diag, "trd", n, dur);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
void stop() {
|
|
py_keep_going = false;
|
|
if (py_thread) {
|
|
py_thread->join();
|
|
py_thread.reset();
|
|
}
|
|
cpp_keep_going = false;
|
|
if (cpp_thread) {
|
|
cpp_thread->join();
|
|
cpp_thread.reset();
|
|
}
|
|
}
|
|
|
|
private:
|
|
PyMarketDataProviderConfig config;
|
|
std::string name;
|
|
std::shared_ptr<py_interop::MdBridgeImpl> bridge_ptr;
|
|
py_interop::MdBridgeImpl &bridge;
|
|
optional<thread> py_thread;
|
|
std::shared_ptr<python_interop::PluginManager> py_plugin_man;
|
|
volatile bool py_keep_going = false;
|
|
optional<thread> cpp_thread;
|
|
volatile bool cpp_keep_going = false;
|
|
shared_ptr<IMarketDataListener> listener_;
|
|
|
|
shared_ptr<IDGenerator> sec_id_store_;
|
|
// The main store for securities.
|
|
vector<Security> securities_;
|
|
vector<const Security *> security_ptrs_;
|
|
Book book_;
|
|
// Since the python md provider isn't tethered to a single venue, we
|
|
// must know the venue for each individual trade.
|
|
vector<TradeRecord> trades_;
|
|
vector<const TradeRecord *> trade_ptrs_;
|
|
|
|
SecurityID get_sec_id_(
|
|
Venue venue, SecurityType sec_type, const string &sec_name) {
|
|
auto sec_id = sec_id_store_->get_id(venue, sec_type, sec_name, true);
|
|
bts_assert_msgf(
|
|
sec_id,
|
|
"Failed to get security id for py_book. venue: {}, type: {}, name: "
|
|
"{}",
|
|
venue, sec_type, sec_name);
|
|
return SecurityID{*sec_id};
|
|
}
|
|
|
|
optional<Security> to_bts_security_(const PySecurity &py_sec) {
|
|
// We can't let python provider assign the id, because it is unaware of other
|
|
// md providers, so can't possibly guarentee a unique id.
|
|
bts_assert(py_sec.id() == 0);
|
|
Security sec;
|
|
auto &sec_comm = sec.security_common;
|
|
if (!venue_parse_smart(py_sec.venue(), &sec_comm.venue)) {
|
|
LOG(md, Error, "Failed to parse venue from '{}'", py_sec.venue());
|
|
return std::nullopt;
|
|
}
|
|
sec_comm.exchange_sec_id = py_sec.exchange_id();
|
|
sec_comm.currency = py_sec.currency();
|
|
sec_comm.asset = py_sec.asset();
|
|
sec_comm.group = py_sec.group();
|
|
sec_comm.symbol = py_sec.asset();
|
|
if (sec_comm.symbol.empty()) {
|
|
sec_comm.symbol = py_sec.group();
|
|
}
|
|
const py_interop::PyContractSpecification &spec = py_sec.contract_spec();
|
|
if (spec.price_infos().size() > 0) {
|
|
bts_assert(spec.price_infos().size() == 1);
|
|
const py_interop::PyContractPriceInfo &price_info =
|
|
spec.price_infos()[0];
|
|
bts_assert(price_info.tick_unit() > 0 && price_info.tick_base() > 0);
|
|
sec_comm.min_price_increment =
|
|
price_info.tick_unit() / price_info.tick_base();
|
|
if (price_info.cash_per_point() > 0) {
|
|
sec_comm.cash_per_point = price_info.cash_per_point();
|
|
}
|
|
}
|
|
if (spec.quantity_infos().size() > 0) {
|
|
bts_assert(spec.quantity_infos().size() == 1);
|
|
const py_interop::PyContractQuantityInfo &qty_info =
|
|
spec.quantity_infos()[0];
|
|
bts_assert(qty_info.tick_unit() > 0 && qty_info.tick_base() > 0);
|
|
sec_comm.min_quantity_increment =
|
|
qty_info.tick_unit() / qty_info.tick_base();
|
|
sec_comm.min_quantity =
|
|
std::max(qty_info.min_value(), sec_comm.min_quantity_increment);
|
|
}
|
|
if (spec.notional_quantity_infos().size() > 0) {
|
|
const auto ¬ional_qty_info = spec.notional_quantity_infos()[0];
|
|
sec_comm.min_notional_value = notional_qty_info.min_value();
|
|
}
|
|
sec_comm.desc = py_sec.name();
|
|
if (py_sec.is_spot()) {
|
|
sec.security = SpotSecurity{};
|
|
} else if (py_sec.is_future()) {
|
|
FutureSecurity fut;
|
|
fut.expiration =
|
|
TimeStamp::from_iso_string(py_sec.future().expiration());
|
|
sec.security = std::move(fut);
|
|
} else if (py_sec.is_option()) {
|
|
OptionSecurity opt;
|
|
const auto &py_opt = py_sec.option();
|
|
opt.expiration = TimeStamp::from_iso_string(py_opt.expiration());
|
|
opt.call_put = py_opt.is_call() ? CallPut::CALL : CallPut::PUT;
|
|
opt.strike = Price::from_double(py_opt.strike());
|
|
sec.security = std::move(opt);
|
|
} else if (py_sec.is_spread()) {
|
|
SpreadSecurity spread;
|
|
const auto &py_spread = py_sec.spread();
|
|
|
|
// Convert Python spread legs to C++ LegDefinition
|
|
for (const auto &leg_name : py_spread.legs()) {
|
|
LOG(md, Info, "Processing spread leg: {}", leg_name);
|
|
|
|
// Use the existing security resolution mechanism
|
|
// Each leg should be resolved using the same venue and appropriate security type
|
|
// For now, we'll assume the leg is a future (most common for spreads)
|
|
// You may need to adjust this based on your specific leg types
|
|
auto leg_sec_id = get_sec_id_(sec_comm.venue, SecurityType::SECURITY_TYPE_FUTURE, leg_name);
|
|
auto quantity = bts::domain::Quantity(1); // Default quantity
|
|
auto side = bts::domain::Side::Value::Buy; // Default side
|
|
|
|
bts::domain::LegDefinition leg_d{
|
|
leg_sec_id,
|
|
quantity,
|
|
side
|
|
};
|
|
spread.legs.push_back(leg_d);
|
|
}
|
|
|
|
LOG(md, Info,
|
|
"Created spread with {} legs. Spread type: {}, expiration: {}",
|
|
py_spread.legs().size(),
|
|
py_spread.spread_type(),
|
|
py_spread.expiration());
|
|
|
|
sec.security = std::move(spread);
|
|
} else {
|
|
LOG(md, Error,
|
|
"PySecurity has incomplete definition (not spot, future, "
|
|
"option or spread)");
|
|
return std::nullopt;
|
|
}
|
|
sec_comm.id =
|
|
get_sec_id_(sec_comm.venue, sec.type(), sec_comm.exchange_sec_id);
|
|
return sec;
|
|
}
|
|
|
|
void to_bts_trade_(const PyTrade &source, domain::TradeRecord &dest) {
|
|
Venue venue;
|
|
bts_assert(venue_parse_smart(source.venue(), &venue));
|
|
SecurityType sec_type;
|
|
bts_assert(security_type_parse_smart(source.security_type(), &sec_type));
|
|
dest.sec_id = get_sec_id_(venue, sec_type, source.exchange_security_id());
|
|
dest.time_stamp = core::TimeStamp::from_nanos(source.timestamp());
|
|
const string &agg_str = source.aggressor_side();
|
|
dest.agg_side = agg_str.starts_with('B') || agg_str.starts_with('b')
|
|
? AggressorSide::Buy
|
|
: AggressorSide::Sell;
|
|
dest.trade_price = Price::from_double(source.price());
|
|
*dest.trade_quantity = source.quantity();
|
|
}
|
|
|
|
void to_book_side_(const PyBookSide &source, domain::BookSide &dest) {
|
|
for (int i = 0; i < source.size(); i++) {
|
|
auto py_level = source.get_level(i);
|
|
const auto price = py_level.price();
|
|
const auto quantity = py_level.quantity();
|
|
bts_assert(!std::isnan(price));
|
|
bts_assert(quantity > 0);
|
|
*dest[i].total_quantity = quantity;
|
|
*dest[i].direct_quantity = quantity;
|
|
dest[i].price = domain::Price::from_double(price);
|
|
}
|
|
}
|
|
|
|
void to_book_(const PyBook &source, domain::Book &dest) {
|
|
to_book_side_(source.bids(), dest.bids());
|
|
to_book_side_(source.asks(), dest.asks());
|
|
Venue venue;
|
|
bts_assert(venue_parse_smart(source.venue(), &venue));
|
|
SecurityType type;
|
|
bts_assert(security_type_parse_smart(source.security_type(), &type));
|
|
dest.set_security_id(
|
|
get_sec_id_(venue, type, source.exchange_security_id()));
|
|
}
|
|
};
|
|
|
|
PyMarketDataSubscription::PyMarketDataSubscription(
|
|
PyMarketDataProviderConfig config,
|
|
IDGenerator *sec_id_store,
|
|
std::shared_ptr<python_interop::PluginManager> py_plugin_man)
|
|
: me(std::make_unique<PyMarketDataSubscription::PImpl>(
|
|
std::move(config), sec_id_store, std::move(py_plugin_man))) {}
|
|
|
|
PyMarketDataSubscription::~PyMarketDataSubscription() { me->stop(); }
|
|
|
|
void PyMarketDataSubscription::start(shared_ptr<IMarketDataListener> listener) {
|
|
me->start(std::move(listener));
|
|
}
|
|
|
|
void PyMarketDataSubscription::stop() { me->stop(); }
|
|
|
|
} // namespace bts::md
|