refactor: Manage all py plugins (md and exe) in one class
All calls to/from python are performed on one thread via a simple message queue (necessary because of the global interpreter lock).
This commit is contained in:
1 parent
178ea7066f
commit
968d33adb6
29 files changed
+811
-602
No files matched your search
@@ -1,4 +1,5 @@
|
||||
#include "Run.h"
|
||||
#include "python_interop/plugin_manager.h"
|
||||
#include "spark/api/public/ServiceContext.h"
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/config/protos/ports.pb.h"
|
||||
@@ -53,6 +54,7 @@ using bts::orderman::ExecutionGatewayProvider;
|
||||
using bts::orderman::IExecutionGatewayProvider;
|
||||
using bts::orderman::OrderServiceServer;
|
||||
using bts::trace::PerfLog;
|
||||
using std::shared_ptr;
|
||||
using std::unique_ptr;
|
||||
using std::vector;
|
||||
|
||||
@@ -62,14 +64,16 @@ static OrderServiceServer create_order_service_server(
|
||||
DynamicOrderRegistry &dyn_order_registry,
|
||||
MetricsContainer &stats,
|
||||
PerfLog &perf_log,
|
||||
TradeSummarySignalDispatcher &tss_dispatcher) {
|
||||
TradeSummarySignalDispatcher &tss_dispatcher,
|
||||
shared_ptr<python_interop::PluginManager> py_plugin_man) {
|
||||
vector<unique_ptr<IExecutionGatewayProvider>> gw_pros;
|
||||
gw_pros.push_back(std::make_unique<ExecutionGatewayProvider>(
|
||||
&perf_log, mdclient.oracle(), config::get<config::JetFIX>(),
|
||||
[](const std::string &name) {
|
||||
const spark::SparkDataDirectory data_dir;
|
||||
return data_dir.load_config_json(name);
|
||||
}));
|
||||
},
|
||||
std::move(py_plugin_man)));
|
||||
return OrderServiceServer(
|
||||
sc, mdclient, dyn_order_registry, std::move(gw_pros), stats, perf_log,
|
||||
tss_dispatcher);
|
||||
@@ -99,7 +103,11 @@ static void execute() {
|
||||
|
||||
bts::md::MBOProcessor mbo_processor;
|
||||
bts::md::TradeSummarySignalDispatcher tss_dispatcher;
|
||||
md::cmds::Run::run();
|
||||
// The py plugin manager is shared by md and execution, so it lives in this scope.
|
||||
auto py_plugin_man = std::make_shared<python_interop::PluginManager>();
|
||||
// start work loop in background
|
||||
py_plugin_man->start_work_thread();
|
||||
md::cmds::Run::run(py_plugin_man);
|
||||
|
||||
api::synthetic::SyntheticSecurityService synthetic_service(sc);
|
||||
bts::md::MDClient mdclient(
|
||||
@@ -120,7 +128,8 @@ static void execute() {
|
||||
// finished initializing order plugins
|
||||
|
||||
auto order_service_server = create_order_service_server(
|
||||
sc, mdclient, dyn_order_registry, stats, perf_log, tss_dispatcher);
|
||||
sc, mdclient, dyn_order_registry, stats, perf_log, tss_dispatcher,
|
||||
py_plugin_man);
|
||||
support::ErrorReportServer error_report_service(sc);
|
||||
admin::AdminServiceServer admin_service(sc);
|
||||
bts::md::auxiliary::MarketDataAuxiliaryServiceServer md_aux_server(sc);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "LiveLadderDemo.h"
|
||||
#include "python_interop/plugin_manager.h"
|
||||
#include "spark/api/md/MarketDataConfig.pb.h"
|
||||
#include "spark/persistence/IDGenerator.h"
|
||||
#include "sparkd/md/MarketDataGateway.h"
|
||||
@@ -15,7 +16,9 @@ LiveLadderDemo::LiveLadderDemo(const MarketDataConfig &config) {
|
||||
char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt";
|
||||
putenv(ssl_cert_file);
|
||||
|
||||
md::MarketDataGateway gw{config, IDGenerator::instance_ptr()};
|
||||
auto py_plugin_man = std::make_shared<python_interop::PluginManager>();
|
||||
py_plugin_man->start_work_thread();
|
||||
md::MarketDataGateway gw{config, IDGenerator::instance_ptr(), py_plugin_man};
|
||||
md::listeners::LiveLadderPrinter ladder_printer;
|
||||
gw.start();
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "Run.h"
|
||||
#include "Cleanup.h"
|
||||
#include "python_interop/plugin_manager.h"
|
||||
#include "spark/api/md/MarketDataConfig.pb.h"
|
||||
#include "spark/config/protos/market_data.pb.h"
|
||||
#include "spark/domain/SecurityCriteria.h"
|
||||
@@ -92,12 +93,13 @@ static void run_onixs(
|
||||
disp.run();
|
||||
}
|
||||
|
||||
static void run_md_gwy() {
|
||||
static void run_md_gwy(
|
||||
std::shared_ptr<python_interop::PluginManager> py_worker) {
|
||||
LOG(md, Info, "Run::run_md_gwy t_id:{} t_name:{}", os::get_thread_id(),
|
||||
os::get_thread_name());
|
||||
bts::md::MarketDataGateway gwy{
|
||||
config::get<api::md::MarketDataConfig>(),
|
||||
persistence::IDGenerator::instance_ptr()};
|
||||
persistence::IDGenerator::instance_ptr(), std::move(py_worker)};
|
||||
gwy.start();
|
||||
while (true) {
|
||||
using namespace std::chrono_literals;
|
||||
@@ -105,12 +107,15 @@ static void run_md_gwy() {
|
||||
}
|
||||
}
|
||||
|
||||
void Run::run() {
|
||||
void Run::run(std::shared_ptr<python_interop::PluginManager> py_worker) {
|
||||
md::cmds::Cleanup::execute();
|
||||
LOG(md, Info, "Run::run t_id:{} t_name:{}", os::get_thread_id(),
|
||||
os::get_thread_name());
|
||||
// start a thread for the md
|
||||
threading::make_thread("MarketDataGateway", run_md_gwy).detach();
|
||||
threading::make_thread(
|
||||
"MarketDataGateway",
|
||||
[py_worker = std::move(py_worker)]() { run_md_gwy(py_worker); })
|
||||
.detach();
|
||||
boot_sleep();
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
|
||||
#include <CLI11/CLI11.hpp>
|
||||
|
||||
namespace python_interop {
|
||||
class PluginManager;
|
||||
}
|
||||
|
||||
namespace bts::cmds::md::cmds {
|
||||
|
||||
/**
|
||||
@@ -9,18 +13,15 @@ namespace bts::cmds::md::cmds {
|
||||
* shared memory.
|
||||
*/
|
||||
struct Run : public CLI::App {
|
||||
enum class MDType {
|
||||
Configured,
|
||||
Replay,
|
||||
Onixs
|
||||
};
|
||||
enum class MDType { Configured, Replay, Onixs };
|
||||
|
||||
Run();
|
||||
|
||||
/**
|
||||
* Run market data
|
||||
*/
|
||||
static void run();
|
||||
static void run(
|
||||
std::shared_ptr<python_interop::PluginManager> py_worker = nullptr);
|
||||
};
|
||||
|
||||
} // namespace bts::cmds::md::cmds
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "binance_execution/usds_futures/usds_futures_gateway.h"
|
||||
#include "okx_execution/okx_execution_gateway.h"
|
||||
#include "onixs_fix/OnixFIXAdapter.h"
|
||||
#include "python_interop/plugin_manager.h"
|
||||
#include "spark/api/domain/Venue.h"
|
||||
#include "spark/domain/DomainSecurityUtils.h"
|
||||
#include "spark/domain_model/Security.pb.h"
|
||||
@@ -37,6 +38,7 @@ using bts::io::JsonReader;
|
||||
using nlohmann::json;
|
||||
using std::function;
|
||||
using std::optional;
|
||||
using std::shared_ptr;
|
||||
using std::string;
|
||||
using std::vector;
|
||||
|
||||
@@ -56,14 +58,17 @@ struct ExecutionGatewayProvider::PImpl {
|
||||
trace::PerfLog *l,
|
||||
md::SecurityOracle o,
|
||||
config::JetFIX config,
|
||||
function<optional<json>(const string &)> load_config_json)
|
||||
function<optional<json>(const string &)> load_config_json,
|
||||
shared_ptr<python_interop::PluginManager> py_plugin_man)
|
||||
: l(l), o(std::move(o)), config(std::move(config)),
|
||||
load_config_json(std::move(load_config_json)) {}
|
||||
load_config_json(std::move(load_config_json)),
|
||||
py_plugin_man(std::move(py_plugin_man)) {}
|
||||
|
||||
trace::PerfLog *l;
|
||||
md::SecurityOracle o;
|
||||
config::JetFIX config;
|
||||
function<optional<json>(const string &)> load_config_json;
|
||||
shared_ptr<python_interop::PluginManager> py_plugin_man;
|
||||
|
||||
template <class T>
|
||||
T parse_config_json(const string &name) {
|
||||
@@ -87,6 +92,9 @@ struct ExecutionGatewayProvider::PImpl {
|
||||
// but with a "py_" prefix, assume that file defines python gateway
|
||||
// config for that connection.
|
||||
// *************************************
|
||||
if (SimExecutionGatewayFactory::is_sim_execution_on(conn.venue())) {
|
||||
return make_gw<SimExecutionGatewayUniqueWrapper>(conn, conn.venue());
|
||||
}
|
||||
auto py_cfg_j = load_config_json(fmt::format("py_{}", conn.name()));
|
||||
if (py_cfg_j) {
|
||||
auto py_cfg = py_cfg_j->get<PyExecutionGatewayConfig>();
|
||||
@@ -94,10 +102,7 @@ struct ExecutionGatewayProvider::PImpl {
|
||||
domain::DomainSecurityUtils::convert(conn.venue()) ==
|
||||
py_cfg.venue);
|
||||
return make_gw<PyExecutionGatewayAdapter>(
|
||||
conn, o, std::move(py_cfg), conn);
|
||||
}
|
||||
if (SimExecutionGatewayFactory::is_sim_execution_on(conn.venue())) {
|
||||
return make_gw<SimExecutionGatewayUniqueWrapper>(conn, conn.venue());
|
||||
conn, o, std::move(py_cfg), conn, py_plugin_man);
|
||||
}
|
||||
switch (conn.venue()) {
|
||||
case api::VENUE_ICE:
|
||||
@@ -161,9 +166,14 @@ ExecutionGatewayProvider::ExecutionGatewayProvider(
|
||||
md::SecurityOracle o,
|
||||
config::JetFIX config,
|
||||
std::function<std::optional<nlohmann::json>(const std::string &)>
|
||||
load_config_json)
|
||||
load_config_json,
|
||||
shared_ptr<python_interop::PluginManager> py_plugin_man)
|
||||
: me_(std::make_unique<PImpl>(
|
||||
l, std::move(o), std::move(config), std::move(load_config_json))) {}
|
||||
l,
|
||||
std::move(o),
|
||||
std::move(config),
|
||||
std::move(load_config_json),
|
||||
std::move(py_plugin_man))) {}
|
||||
|
||||
ExecutionGatewayProvider::~ExecutionGatewayProvider() = default;
|
||||
|
||||
|
||||
@@ -9,6 +9,10 @@ namespace bts::spark {
|
||||
class SparkDataDirectory;
|
||||
}
|
||||
|
||||
namespace python_interop {
|
||||
class PluginManager;
|
||||
}
|
||||
|
||||
namespace bts::orderman {
|
||||
|
||||
class ExecutionGatewayProvider : public IExecutionGatewayProvider {
|
||||
@@ -18,7 +22,8 @@ class ExecutionGatewayProvider : public IExecutionGatewayProvider {
|
||||
md::SecurityOracle o,
|
||||
config::JetFIX config,
|
||||
std::function<std::optional<nlohmann::json>(const std::string &)>
|
||||
load_config_json);
|
||||
load_config_json,
|
||||
std::shared_ptr<python_interop::PluginManager> py_plugin_man);
|
||||
~ExecutionGatewayProvider() override;
|
||||
|
||||
std::vector<OrderGatewayDefinition> create() override;
|
||||
|
||||
@@ -7,11 +7,12 @@
|
||||
#include "python_interop/execution/py_create_order_request.h"
|
||||
#include "python_interop/execution/py_execution_gateway.h"
|
||||
#include "python_interop/execution/py_execution_report.h"
|
||||
#include "python_interop/plugin_manager.h"
|
||||
#include "python_interop/py_contract_price_info.h"
|
||||
#include "python_interop/py_contract_quantity_info.h"
|
||||
#include "python_interop/py_contract_specification.h"
|
||||
#include "python_interop/py_security.h"
|
||||
#include "python_interop/python_activator.h"
|
||||
#include "python_interop/python_object.h"
|
||||
#include "spark/api/domain/Venue.h"
|
||||
#include "spark/api/service_helpers/AdminServiceUtils.h"
|
||||
#include "spark/api/services/AdminService.pb.h"
|
||||
@@ -59,6 +60,7 @@
|
||||
#include "sparkd/execution/py_execution_gateway_config.h"
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <future>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <string>
|
||||
#include <variant>
|
||||
@@ -126,12 +128,13 @@ using bts::orderman::OrderMsg;
|
||||
using bts::orderman::RFQMsg;
|
||||
using bts::system::string_utils::parse_uint;
|
||||
using nlohmann::json;
|
||||
using python_execution::ExecutionGateway;
|
||||
using python_execution::ExecutionGatewayBridgeImpl;
|
||||
using python_execution::PyCancelOrderRequest;
|
||||
using python_execution::PyCancelReplaceOrderRequest;
|
||||
using python_execution::PyCreateOrderRequest;
|
||||
using python_execution::PyExecutionGateway;
|
||||
using python_interop::PythonActivatorT;
|
||||
using python_interop::PluginManager;
|
||||
using python_interop::PythonObjectT;
|
||||
using std::int64_t;
|
||||
using std::optional;
|
||||
using std::shared_ptr;
|
||||
@@ -154,21 +157,20 @@ struct PyExecutionGatewayAdapter::PImpl {
|
||||
explicit PImpl(
|
||||
bts::md::SecurityOracle sec_oracle,
|
||||
PyExecutionGatewayConfig config,
|
||||
bts::config::JetFIX::Connection jetfix_connection)
|
||||
bts::config::JetFIX::Connection jetfix_connection,
|
||||
shared_ptr<PluginManager> plugin_man)
|
||||
: sec_oracle(std::move(sec_oracle)), config(std::move(config)),
|
||||
bridge(
|
||||
jetfix_connection.name(),
|
||||
this->config.gateway_type,
|
||||
this->config.module_path),
|
||||
name(fmt::format("{}_py_gw", string{jetfix_connection.name()})),
|
||||
bridge_ptr(std::make_shared<ExecutionGatewayBridgeImpl>(
|
||||
jetfix_connection.name())),
|
||||
bridge(*bridge_ptr), plugin_man(std::move(plugin_man)),
|
||||
jetfix_connection(std::make_shared<bts::config::JetFIX_Connection>(
|
||||
std::move(jetfix_connection))),
|
||||
orders(this->config.account, this->jetfix_connection) {
|
||||
bts_assert(this->jetfix_connection->venue() != api::VENUE_UNSET);
|
||||
bts_assert(!this->jetfix_connection->name().empty());
|
||||
pos.exchange = this->jetfix_connection->venue();
|
||||
sm.set_name(fmt::format(
|
||||
"{}_py_gw",
|
||||
string{bts::api::Venue_Name(this->jetfix_connection->venue())}));
|
||||
sm.set_name(name);
|
||||
sm.set_subsystem(bts::log::OrderMan);
|
||||
sm.start_core_impl.bind<&PImpl::start_impl>(*this);
|
||||
sm.stop_core_impl.bind<&PImpl::stop_impl>(*this);
|
||||
@@ -183,8 +185,10 @@ struct PyExecutionGatewayAdapter::PImpl {
|
||||
|
||||
SecurityOracle sec_oracle;
|
||||
PyExecutionGatewayConfig config;
|
||||
ExecutionGatewayBridgeImpl bridge;
|
||||
optional<PythonActivatorT<PyExecutionGateway>> py_gw_activator;
|
||||
std::string name;
|
||||
std::shared_ptr<ExecutionGatewayBridgeImpl> bridge_ptr;
|
||||
ExecutionGatewayBridgeImpl &bridge;
|
||||
shared_ptr<PluginManager> plugin_man;
|
||||
shared_ptr<bts::config::JetFIX_Connection> jetfix_connection;
|
||||
optional<thread> pos_snapshot_thread;
|
||||
optional<thread> py_thread;
|
||||
@@ -247,7 +251,17 @@ struct PyExecutionGatewayAdapter::PImpl {
|
||||
INFO_TRACE("start_impl");
|
||||
bts_assert(!py_thread);
|
||||
py_keep_going = true;
|
||||
py_thread.emplace([&]() { bridge.run_py(&py_keep_going); });
|
||||
py_thread.emplace([&]() {
|
||||
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
|
||||
std::future<shared_ptr<PythonObjectT<ExecutionGateway>>> gw_fut =
|
||||
plugin_man->start_execution_gateway(
|
||||
name, config.gateway_type, config.module_path, bridge_ptr);
|
||||
gw_fut.wait();
|
||||
std::signal(SIGINT, orig_signal_handler);
|
||||
bridge.run_py(&py_keep_going, gw_fut.get(), [&](auto callback) {
|
||||
plugin_man->invoke(callback);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void stop() {
|
||||
@@ -519,11 +533,13 @@ struct PyExecutionGatewayAdapter::PImpl {
|
||||
PyExecutionGatewayAdapter::PyExecutionGatewayAdapter(
|
||||
SecurityOracle sec_oracle,
|
||||
PyExecutionGatewayConfig session_cfg,
|
||||
bts::config::JetFIX::Connection jetfix_connection)
|
||||
bts::config::JetFIX::Connection jetfix_connection,
|
||||
shared_ptr<PluginManager> plugin_man)
|
||||
: me(std::make_unique<PImpl>(
|
||||
std::move(sec_oracle),
|
||||
std::move(session_cfg),
|
||||
std::move(jetfix_connection))) {
|
||||
std::move(jetfix_connection),
|
||||
std::move(plugin_man))) {
|
||||
bts_assert(
|
||||
me->config.account.size() <=
|
||||
ExecutionReport{}.account_name.value().max_size());
|
||||
|
||||
@@ -6,6 +6,10 @@
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace python_interop {
|
||||
class PluginManager;
|
||||
}
|
||||
|
||||
namespace bts::orderman {
|
||||
|
||||
struct PyExecutionGatewayConfig;
|
||||
@@ -15,7 +19,8 @@ class PyExecutionGatewayAdapter : public bts::orderman::IExecutionGateway {
|
||||
explicit PyExecutionGatewayAdapter(
|
||||
bts::md::SecurityOracle sec_oracle,
|
||||
PyExecutionGatewayConfig session_cfg,
|
||||
bts::config::JetFIX::Connection jetfix_connection);
|
||||
bts::config::JetFIX::Connection jetfix_connection,
|
||||
std::shared_ptr<python_interop::PluginManager> plugin_man);
|
||||
~PyExecutionGatewayAdapter() override;
|
||||
|
||||
void start() override;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "MarketDataGateway.h"
|
||||
#include "python_interop/plugin_manager.h"
|
||||
#include "spark/api/domain/Venue.h"
|
||||
#include "spark/api/md/MarketDataConfig.pb.h"
|
||||
#include "spark/api/md/MarketDataConfigUtils.h"
|
||||
@@ -67,7 +68,10 @@ struct MarketDataGateway::PImpl {
|
||||
vector<unique_ptr<MdSink>> sinks;
|
||||
vector<shared_ptr<IMarketDataSource>> sources;
|
||||
|
||||
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
|
||||
PImpl(
|
||||
MarketDataConfig config,
|
||||
shared_ptr<IDGenerator> sec_id_store,
|
||||
shared_ptr<python_interop::PluginManager> py_plugin_man)
|
||||
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {
|
||||
providers.push_back(std::make_unique<CCApiMarketDataProvider>(
|
||||
this->config, this->sec_id_store));
|
||||
@@ -80,7 +84,7 @@ struct MarketDataGateway::PImpl {
|
||||
providers.push_back(std::make_unique<TaifexMarketDataProvider>(
|
||||
this->config, this->sec_id_store));
|
||||
providers.push_back(std::make_unique<PyMarketDataProvider>(
|
||||
this->config, this->sec_id_store));
|
||||
this->config, this->sec_id_store, std::move(py_plugin_man)));
|
||||
providers.push_back(std::make_unique<ReplayMarketDataProvider>(
|
||||
this->config, this->sec_id_store));
|
||||
}
|
||||
@@ -126,9 +130,13 @@ struct MarketDataGateway::PImpl {
|
||||
};
|
||||
|
||||
MarketDataGateway::MarketDataGateway(
|
||||
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
|
||||
MarketDataConfig config,
|
||||
shared_ptr<IDGenerator> sec_id_store,
|
||||
std::shared_ptr<python_interop::PluginManager> py_plugin_man)
|
||||
: me_(std::make_unique<MarketDataGateway::PImpl>(
|
||||
std::move(config), std::move(sec_id_store))) {}
|
||||
std::move(config),
|
||||
std::move(sec_id_store),
|
||||
std::move(py_plugin_man))) {}
|
||||
|
||||
MarketDataGateway::~MarketDataGateway() = default;
|
||||
|
||||
|
||||
@@ -10,6 +10,10 @@ namespace bts::persistence {
|
||||
class IDGenerator;
|
||||
}
|
||||
|
||||
namespace python_interop {
|
||||
class PluginManager;
|
||||
}
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
/**
|
||||
@@ -20,7 +24,8 @@ class MarketDataGateway {
|
||||
public:
|
||||
explicit MarketDataGateway(
|
||||
api::md::MarketDataConfig config,
|
||||
std::shared_ptr<persistence::IDGenerator> sec_id_store);
|
||||
std::shared_ptr<persistence::IDGenerator> sec_id_store,
|
||||
std::shared_ptr<python_interop::PluginManager> py_plugin_man);
|
||||
~MarketDataGateway();
|
||||
|
||||
void start();
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "PyMarketDataProvider.h"
|
||||
#include "python_interop/plugin_manager.h"
|
||||
#include "spark/api/md/MarketDataConfig.pb.h"
|
||||
#include "spark/md/IMarketDataSource.h"
|
||||
#include "spark/persistence/IDGenerator.h"
|
||||
@@ -19,16 +20,23 @@ using Path = boost::filesystem::path;
|
||||
struct PyMarketDataProvider::PImpl {
|
||||
MarketDataConfig config;
|
||||
shared_ptr<IDGenerator> sec_id_store;
|
||||
shared_ptr<python_interop::PluginManager> py_worker;
|
||||
vector<shared_ptr<PyMarketDataSubscription>> py_md_subs;
|
||||
|
||||
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
|
||||
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
|
||||
PImpl(
|
||||
MarketDataConfig config,
|
||||
shared_ptr<IDGenerator> sec_id_store,
|
||||
shared_ptr<python_interop::PluginManager> py_worker)
|
||||
: config(std::move(config)), sec_id_store(std::move(sec_id_store)),
|
||||
py_worker(std::move(py_worker)) {}
|
||||
};
|
||||
|
||||
PyMarketDataProvider::PyMarketDataProvider(
|
||||
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
|
||||
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
|
||||
}
|
||||
MarketDataConfig config,
|
||||
shared_ptr<IDGenerator> sec_id_store,
|
||||
shared_ptr<python_interop::PluginManager> py_worker)
|
||||
: me_(std::make_unique<PImpl>(
|
||||
std::move(config), std::move(sec_id_store), std::move(py_worker))) {}
|
||||
|
||||
PyMarketDataProvider::~PyMarketDataProvider() = default;
|
||||
|
||||
@@ -38,7 +46,7 @@ PyMarketDataProvider::get_sources() {
|
||||
for (const auto &py_plugin : me_->config.python_md_providers()) {
|
||||
try {
|
||||
auto source = std::make_shared<PyMarketDataSubscription>(
|
||||
py_plugin, me_->sec_id_store.get());
|
||||
py_plugin, me_->sec_id_store.get(), me_->py_worker);
|
||||
me_->py_md_subs.push_back(source);
|
||||
rv.emplace_back(py_plugin.venue(), std::move(source));
|
||||
} catch (const std::exception &ex) {
|
||||
|
||||
@@ -7,13 +7,18 @@
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace python_interop {
|
||||
class PluginManager;
|
||||
}
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
class PyMarketDataProvider : public IMarketDataProvider {
|
||||
public:
|
||||
explicit PyMarketDataProvider(
|
||||
api::md::MarketDataConfig config,
|
||||
std::shared_ptr<persistence::IDGenerator> sec_id_store);
|
||||
std::shared_ptr<persistence::IDGenerator> sec_id_store,
|
||||
std::shared_ptr<python_interop::PluginManager> py_worker);
|
||||
~PyMarketDataProvider();
|
||||
|
||||
std::vector<
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
#include "python_interop/md/md_bridge_impl.h"
|
||||
#include "python_interop/md/py_book.h"
|
||||
#include "python_interop/md/py_trade.h"
|
||||
#include "python_interop/plugin_manager.h"
|
||||
#include "python_interop/py_contract_specification.h"
|
||||
#include "python_interop/py_security.h"
|
||||
#include "python_interop/python_object.h"
|
||||
#include "spark/api/domain/CallPut.pb.h"
|
||||
#include "spark/api/domain/SecurityType.h"
|
||||
#include "spark/api/domain/Venue.h"
|
||||
@@ -26,6 +28,7 @@
|
||||
#include "spark/utils/bts_assert.h"
|
||||
#include "spark/utils/stopwatch.h"
|
||||
|
||||
#include <future>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <pybind11/embed.h>
|
||||
@@ -45,6 +48,10 @@ namespace bts::md {
|
||||
#define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name)
|
||||
#define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name)
|
||||
|
||||
#define INFO_TRACE_PARAMS(name, params) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
md, bts::Info, std::string("py_md_sub"), std::string(name), params)
|
||||
|
||||
#define LOG_DISPATCH_STATS(level, typ, n, dur) \
|
||||
_LOG( \
|
||||
level, \
|
||||
@@ -101,11 +108,14 @@ struct TradeTuple {
|
||||
};
|
||||
|
||||
struct PyMarketDataSubscription::PImpl {
|
||||
PImpl(PyMarketDataProviderConfig config, IDGenerator *sec_id_store)
|
||||
: config(std::move(config)), bridge(
|
||||
this->config.provider_type(),
|
||||
{this->config.module_path().begin(),
|
||||
this->config.module_path().end()}),
|
||||
PImpl(
|
||||
PyMarketDataProviderConfig config,
|
||||
IDGenerator *sec_id_store,
|
||||
shared_ptr<python_interop::PluginManager> py_plugin_man)
|
||||
: config(std::move(config)),
|
||||
name(fmt::format("{}_md_pro", string{this->config.provider_type()})),
|
||||
bridge_ptr(std::make_shared<py_interop::MdBridgeImpl>()),
|
||||
bridge(*bridge_ptr), py_plugin_man(std::move(py_plugin_man)),
|
||||
sec_id_store_(sec_id_store) {}
|
||||
|
||||
size_t dispatch_securities() {
|
||||
@@ -185,7 +195,26 @@ struct PyMarketDataSubscription::PImpl {
|
||||
LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}",
|
||||
os::get_thread_id(), os::get_thread_name());
|
||||
py_keep_going = true;
|
||||
py_thread.emplace([&]() { bridge.run_py(&py_keep_going); });
|
||||
py_thread.emplace([&]() {
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py", json_formatter(std::map<string, string>{
|
||||
{"name", name}, {"ev", "STARTING_PY_MD_PRO"}}));
|
||||
utils::Stopwatch sw = utils::Stopwatch::start_new();
|
||||
std::future<std::shared_ptr<
|
||||
python_interop::PythonObjectT<bts::md::py_interop::MdProvider>>>
|
||||
md_pro_fut = py_plugin_man->start_md_provider(
|
||||
name, config.provider_type(),
|
||||
{config.module_path().begin(), config.module_path().end()},
|
||||
bridge_ptr);
|
||||
md_pro_fut.wait();
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py",
|
||||
json_formatter(std::map<string, string>{
|
||||
{"name", name},
|
||||
{"ev", "STARTED_PY_MD_PRO"},
|
||||
{"dur",
|
||||
fmt::format("{:.3f}ms", sw.elapsed().total_millis())}}));
|
||||
});
|
||||
cpp_keep_going = true;
|
||||
cpp_thread.emplace([&]() {
|
||||
static const TimeSpan sec_warn_thresh = TimeSpan::from_millis(100);
|
||||
@@ -238,8 +267,11 @@ struct PyMarketDataSubscription::PImpl {
|
||||
|
||||
private:
|
||||
PyMarketDataProviderConfig config;
|
||||
py_interop::MdBridgeImpl bridge;
|
||||
std::string name;
|
||||
std::shared_ptr<py_interop::MdBridgeImpl> bridge_ptr;
|
||||
py_interop::MdBridgeImpl &bridge;
|
||||
optional<thread> py_thread;
|
||||
std::shared_ptr<python_interop::PluginManager> py_plugin_man;
|
||||
volatile bool py_keep_going = false;
|
||||
optional<thread> cpp_thread;
|
||||
volatile bool cpp_keep_going = false;
|
||||
@@ -373,9 +405,11 @@ struct PyMarketDataSubscription::PImpl {
|
||||
};
|
||||
|
||||
PyMarketDataSubscription::PyMarketDataSubscription(
|
||||
PyMarketDataProviderConfig config, IDGenerator *sec_id_store)
|
||||
PyMarketDataProviderConfig config,
|
||||
IDGenerator *sec_id_store,
|
||||
std::shared_ptr<python_interop::PluginManager> py_plugin_man)
|
||||
: me(std::make_unique<PyMarketDataSubscription::PImpl>(
|
||||
std::move(config), sec_id_store)) {}
|
||||
std::move(config), sec_id_store, std::move(py_plugin_man))) {}
|
||||
|
||||
PyMarketDataSubscription::~PyMarketDataSubscription() { me->stop(); }
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "python_interop/plugin_manager.h"
|
||||
#include "spark/md/IMarketDataSource.h"
|
||||
|
||||
#include <memory>
|
||||
@@ -18,7 +19,8 @@ class PyMarketDataSubscription : public IMarketDataSource {
|
||||
public:
|
||||
explicit PyMarketDataSubscription(
|
||||
api::md::PyMarketDataProviderConfig config,
|
||||
persistence::IDGenerator *sec_id_store);
|
||||
persistence::IDGenerator *sec_id_store,
|
||||
std::shared_ptr<python_interop::PluginManager> py_worker);
|
||||
~PyMarketDataSubscription();
|
||||
|
||||
void start(std::shared_ptr<IMarketDataListener> listener) override;
|
||||
|
||||
@@ -3,31 +3,25 @@
|
||||
#include "python_interop/execution/py_cancel_order_request.h"
|
||||
#include "python_interop/execution/py_create_order_request.h"
|
||||
#include "python_interop/execution/py_position_info.h"
|
||||
#include "python_interop/python_activator.h"
|
||||
#include "python_interop/python_interpreter.h"
|
||||
#include "python_interop/python_object.h"
|
||||
#include "spark/core/TimeSpan.h"
|
||||
#include "spark/domain_model/Venue.h"
|
||||
#include "spark/io/json_formatter.h"
|
||||
#include "spark/logging/logging.h"
|
||||
#include "spark/logging/trace_log.h"
|
||||
#include "spark/logging/trace_log_var.h"
|
||||
#include "spark/orderman/OrderMsg.h"
|
||||
#include "spark/orderman/log_messages/execution_report_log.h"
|
||||
#include "spark/orderman/orderman_utils/ExecutionReport.h"
|
||||
#include "spark/utils/overloaded.h"
|
||||
#include "spark/utils/stopwatch.h"
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <future>
|
||||
#include <memory>
|
||||
|
||||
using std::shared_ptr;
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace python_execution {
|
||||
|
||||
using bts::core::Delegate;
|
||||
using bts::core::TimeSpan;
|
||||
using bts::io::json_formatter;
|
||||
using bts::orderman::RFQMsg;
|
||||
using python_interop::PythonActivatorT;
|
||||
using std::string;
|
||||
using std::vector;
|
||||
|
||||
@@ -60,10 +54,8 @@ using std::vector;
|
||||
OrderMan, bts::Warn, std::string("exe_gw_bridge"), std::string(name), \
|
||||
params)
|
||||
|
||||
ExecutionGatewayBridgeImpl::ExecutionGatewayBridgeImpl(
|
||||
string name, string target, vector<string> module_paths)
|
||||
: name(std::move(name)), target(std::move(target)),
|
||||
module_paths(std::move(module_paths)) {}
|
||||
ExecutionGatewayBridgeImpl::ExecutionGatewayBridgeImpl(string name)
|
||||
: name(std::move(name)) {}
|
||||
|
||||
void ExecutionGatewayBridgeImpl::on_order_msg(PyOrderRequest msg) {
|
||||
INFO_TRACE("on_order_msg");
|
||||
@@ -106,11 +98,11 @@ bool ExecutionGatewayBridgeImpl::pop_exe_report(
|
||||
void ExecutionGatewayBridgeImpl::on_securities(
|
||||
std::vector<std::shared_ptr<bts::md::py_interop::PySecurity>> securities) {
|
||||
INFO_TRACE_PARAMS(
|
||||
"on_securities",
|
||||
json_formatter(
|
||||
fmt::format("Received {} securities", this->securities.size())));
|
||||
"on_securities", json_formatter(fmt::format(
|
||||
"Received {} securities", securities.size())));
|
||||
std::lock_guard<std::mutex> lock{securities_mx};
|
||||
this->securities = std::move(securities);
|
||||
this->is_securities_dirty = true;
|
||||
is_securities_dirty = true;
|
||||
}
|
||||
|
||||
shared_ptr<const PyExecutionReport>
|
||||
@@ -119,12 +111,10 @@ ExecutionGatewayBridgeImpl::pop_exe_report() {
|
||||
return res ? *res : nullptr;
|
||||
}
|
||||
|
||||
int ExecutionGatewayBridgeImpl::run_py(volatile bool *keep_going) {
|
||||
INFO_TRACE_PARAMS("run_py", json_formatter("Begin"));
|
||||
bts_assert(keep_going != nullptr);
|
||||
PythonActivatorT<ExecutionGateway> gw_activator{target, module_paths};
|
||||
gw = gw_activator.get_target(
|
||||
[&](ExecutionGateway &local_gw) { local_gw.start(this); }, name);
|
||||
int ExecutionGatewayBridgeImpl::run_py(
|
||||
volatile bool *keep_going,
|
||||
std::shared_ptr<python_interop::PythonObjectT<ExecutionGateway>> gw_ptr,
|
||||
std::function<void(std::function<void()>)> py_invoke) {
|
||||
INFO_TRACE_PARAMS("run_py", json_formatter("Waiting for ready..."));
|
||||
auto sw = bts::utils::Stopwatch::start_new();
|
||||
while (!wait_for_ready(TimeSpan::from_seconds(1)) && *keep_going) {
|
||||
@@ -133,57 +123,100 @@ int ExecutionGatewayBridgeImpl::run_py(volatile bool *keep_going) {
|
||||
"PyExecutionGateway still not ready after {:.2f}s",
|
||||
sw.total_elapsed().total_seconds())));
|
||||
}
|
||||
INFO_TRACE_PARAMS(
|
||||
"run", json_formatter(fmt::format(
|
||||
"PyExecutionGateway ready after {:.2f}s",
|
||||
sw.total_elapsed().total_seconds())));
|
||||
ExecutionGateway &gw = **gw_ptr;
|
||||
std::vector<std::shared_ptr<bts::md::py_interop::PySecurity>> new_securities;
|
||||
while (is_ready.value() && *keep_going) {
|
||||
INFO_TRACE_PARAMS("run_py", json_formatter("Waiting for request..."));
|
||||
if (is_securities_dirty) {
|
||||
bool was_securities_dirty = false;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock{securities_mx};
|
||||
if (is_securities_dirty) {
|
||||
was_securities_dirty = true;
|
||||
new_securities = std::move(securities);
|
||||
is_securities_dirty = false;
|
||||
}
|
||||
}
|
||||
if (was_securities_dirty) {
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py", json_formatter(std::map<string, string>{
|
||||
{"name", name}, {"ev", "SENDING_SECS_TO_PY"}}));
|
||||
std::promise<void> prom;
|
||||
sw.restart();
|
||||
py_invoke([&]() {
|
||||
try {
|
||||
gw.on_securities(new_securities);
|
||||
} catch (const std::exception &ex) {
|
||||
bts_assert_msgf(
|
||||
false, "py exe gw threw on secs. {}",
|
||||
std::string(ex.what()));
|
||||
} catch (...) {
|
||||
bts_assert_msg(false, "py exe gw threw on secs. UNKNOWN");
|
||||
}
|
||||
prom.set_value();
|
||||
});
|
||||
prom.get_future().wait();
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py",
|
||||
json_formatter(fmt::format(
|
||||
"Sending {} securities to py gw", securities.size())));
|
||||
python_interop::PythonInterpreter::exec(
|
||||
[&]() { gw->on_securities(securities); });
|
||||
is_securities_dirty = false;
|
||||
json_formatter(std::map<string, string>{
|
||||
{"name", name},
|
||||
{"ev", "SENT_SECS_TO_PY"},
|
||||
{"dur",
|
||||
fmt::format("{:.3f}ms", sw.elapsed().total_millis())}}));
|
||||
}
|
||||
auto req = outgoing_reqs.wait_for_until_pop(
|
||||
bts::core::TimeSpan::from_seconds(1));
|
||||
if (!req) {
|
||||
continue;
|
||||
}
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py", json_formatter("Forwarding request to py gateway"));
|
||||
enum ReqType { CREATE = 0, CANCEL_REPLACE = 1, CANCEL = 2 };
|
||||
static const std::string req_type_names[]{"CREATE", "CXL_REP", "CXL"};
|
||||
ReqType req_type;
|
||||
sw.restart();
|
||||
python_interop::PythonInterpreter::exec([&]() {
|
||||
std::visit(
|
||||
bts::overloaded{
|
||||
[&](std::shared_ptr<PyCreateOrderRequest> req) {
|
||||
req_type = ReqType::CREATE;
|
||||
gw->create_order(std::move(req));
|
||||
},
|
||||
[&](std::shared_ptr<PyCancelReplaceOrderRequest> req) {
|
||||
req_type = ReqType::CANCEL_REPLACE;
|
||||
gw->cancel_replace_order(std::move(req));
|
||||
},
|
||||
[&](std::shared_ptr<PyCancelOrderRequest> req) {
|
||||
req_type = ReqType::CANCEL;
|
||||
gw->cancel_order(std::move(req));
|
||||
},
|
||||
[&](const RFQMsg &) {
|
||||
bts_assert_msg(false, "RFQ not implemented");
|
||||
},
|
||||
},
|
||||
*req);
|
||||
});
|
||||
sw.stop();
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py",
|
||||
fmt::format(
|
||||
R"({{ "typ": "{}", "hndl_dur": "{:.3f}ms" }})",
|
||||
req_type_names[req_type], sw.total_elapsed().total_millis()));
|
||||
{
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py", json_formatter(std::map<string, string>{
|
||||
{"name", name}, {"ev", "SENDING_REQ_TO_PY"}}));
|
||||
std::promise<void> prom;
|
||||
sw.restart();
|
||||
py_invoke([&, req = std::move(req)]() {
|
||||
try {
|
||||
std::visit(
|
||||
bts::overloaded{
|
||||
[&](std::shared_ptr<PyCreateOrderRequest> req) {
|
||||
req_type = ReqType::CREATE;
|
||||
gw.create_order(std::move(req));
|
||||
},
|
||||
[&](std::shared_ptr<PyCancelReplaceOrderRequest> req) {
|
||||
req_type = ReqType::CANCEL_REPLACE;
|
||||
gw.cancel_replace_order(std::move(req));
|
||||
},
|
||||
[&](std::shared_ptr<PyCancelOrderRequest> req) {
|
||||
req_type = ReqType::CANCEL;
|
||||
gw.cancel_order(std::move(req));
|
||||
}},
|
||||
*req);
|
||||
} catch (const std::exception &ex) {
|
||||
bts_assert_msgf(
|
||||
false, "py exe gw threw on req. {}", std::string(ex.what()));
|
||||
} catch (...) {
|
||||
bts_assert_msg(false, "py exe gw threw on req. UNKNOWN");
|
||||
}
|
||||
prom.set_value();
|
||||
});
|
||||
prom.get_future().wait();
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py",
|
||||
json_formatter(std::map<string, string>{
|
||||
{"name", name},
|
||||
{"ev", "SENT_REQ_TO_PY"},
|
||||
{"typ", req_type_names[req_type]},
|
||||
{"dur",
|
||||
fmt::format("{:.3f}ms", sw.elapsed().total_millis())}}));
|
||||
}
|
||||
}
|
||||
python_interop::PythonInterpreter::exec([&]() { gw->stop(); });
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#include "python_interop/execution/py_order_request.h"
|
||||
#include "python_interop/execution/py_position_info.h"
|
||||
#include "python_interop/py_security.h"
|
||||
#include "python_interop/python_object.h"
|
||||
#include "spark/collections/concurrent_queue.h"
|
||||
#include "spark/utils/Delegate.h"
|
||||
#include "spark/utils/concurrent_value_store.h"
|
||||
@@ -14,10 +15,7 @@ namespace python_execution {
|
||||
|
||||
class ExecutionGatewayBridgeImpl : public ExecutionGatewayBridge {
|
||||
public:
|
||||
ExecutionGatewayBridgeImpl(
|
||||
std::string name,
|
||||
std::string target,
|
||||
std::vector<std::string> module_paths);
|
||||
ExecutionGatewayBridgeImpl(std::string name);
|
||||
~ExecutionGatewayBridgeImpl() override;
|
||||
|
||||
// Python interface
|
||||
@@ -35,21 +33,22 @@ class ExecutionGatewayBridgeImpl : public ExecutionGatewayBridge {
|
||||
bool wait_for_ready(bts::core::TimeSpan timeout);
|
||||
bool pop_exe_report(std::shared_ptr<const PyExecutionReport> &result);
|
||||
std::shared_ptr<const PyExecutionReport> pop_exe_report();
|
||||
int run_py(volatile bool *keep_going);
|
||||
int run_py(
|
||||
volatile bool *keep_going,
|
||||
std::shared_ptr<python_interop::PythonObjectT<ExecutionGateway>> gw,
|
||||
std::function<void(std::function<void()>)> py_invoke);
|
||||
|
||||
private:
|
||||
std::string name;
|
||||
std::string target;
|
||||
std::vector<std::string> module_paths;
|
||||
bts::utils::ConcurrentValueStore<bool> is_ready;
|
||||
bts::collections::ConcurrentQueue<std::shared_ptr<PyExecutionReport>>
|
||||
incoming_exe_reports;
|
||||
bts::collections::ConcurrentQueue<std::shared_ptr<PyPositionInfo>>
|
||||
incoming_pos_infos;
|
||||
bts::collections::ConcurrentQueue<PyOrderRequest> outgoing_reqs;
|
||||
std::mutex securities_mx;
|
||||
std::vector<std::shared_ptr<bts::md::py_interop::PySecurity>> securities;
|
||||
volatile bool is_securities_dirty = false;
|
||||
ExecutionGateway *gw;
|
||||
bool is_securities_dirty = false;
|
||||
};
|
||||
|
||||
} // namespace python_execution
|
||||
@@ -1,46 +1,18 @@
|
||||
#include "python_interop/md/md_bridge_impl.h"
|
||||
#include "python_interop/md/md_provider.h"
|
||||
#include "python_interop/python_activator.h"
|
||||
#include "spark/io/json_formatter.h"
|
||||
#include "spark/logging/logging.h"
|
||||
#include "spark/logging/trace_log.h"
|
||||
#include "spark/utils/Delegate.h"
|
||||
#include "spark/utils/stopwatch.h"
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <thread>
|
||||
|
||||
using std::shared_ptr;
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace bts::md::py_interop {
|
||||
|
||||
using bts::core::Delegate;
|
||||
using bts::io::json_formatter;
|
||||
using bts::utils::Stopwatch;
|
||||
using python_interop::PythonActivatorT;
|
||||
using std::string;
|
||||
using std::vector;
|
||||
|
||||
#define ASSERT_FN(test, fn, ...) \
|
||||
bts_assert_msgf(test, "md_bridge::" fn ". " __VA_ARGS__)
|
||||
#define LOG_FN(level, fn, ...) LOG(md, level, "md_bridge::" fn ". " __VA_ARGS__)
|
||||
#define DIAG_FN(fn, ...) LOG_FN(bts::Diag, fn, __VA_ARGS__)
|
||||
#define INFO_FN(fn, ...) LOG_FN(bts::Info, fn, __VA_ARGS__)
|
||||
#define WARN_FN(fn, ...) LOG_FN(bts::Warn, fn, __VA_ARGS__)
|
||||
#define ERROR_FN(fn, ...) LOG_FN(bts::Error, fn, __VA_ARGS__)
|
||||
|
||||
#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__)
|
||||
#define INFO(...) _LOG(bts::Info, __VA_ARGS__)
|
||||
#define ERROR(...) _LOG(bts::Error, __VA_ARGS__)
|
||||
|
||||
#define _LOG_TRACE(level, name) \
|
||||
LOG_TRACE(md, level, std::string("md_bridge"), std::string(name))
|
||||
#define INFO_TRACE(name) _LOG_TRACE(bts::Info, name)
|
||||
#define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name)
|
||||
#define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name)
|
||||
|
||||
#define DIAG_TRACE_PARAMS(name, params) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
md, bts::Diag, std::string("md_bridge"), std::string(name), params)
|
||||
@@ -49,13 +21,6 @@ using std::vector;
|
||||
LOG_TRACE_PARAMS( \
|
||||
md, bts::Info, std::string("md_bridge"), std::string(name), params)
|
||||
|
||||
#define WARN_TRACE_PARAMS(name, params) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
md, bts::Warn, std::string("md_bridge"), std::string(name), params)
|
||||
|
||||
MdBridgeImpl::MdBridgeImpl(string target, vector<string> module_paths)
|
||||
: target(std::move(target)), module_paths(std::move(module_paths)) {}
|
||||
|
||||
void MdBridgeImpl::on_securities(
|
||||
const std::vector<std::shared_ptr<PySecurity>> &securities) {
|
||||
INFO_TRACE_PARAMS("on_securities", securities.size());
|
||||
@@ -63,7 +28,7 @@ void MdBridgeImpl::on_securities(
|
||||
}
|
||||
|
||||
void MdBridgeImpl::on_books(const std::vector<std::shared_ptr<PyBook>> &books) {
|
||||
DIAG_TRACE_PARAMS("on_books", json_formatter(books.size()));
|
||||
INFO_TRACE_PARAMS("on_books", json_formatter(books.size()));
|
||||
incoming_books.push_many(books.begin(), books.end());
|
||||
}
|
||||
|
||||
@@ -103,26 +68,6 @@ bool MdBridgeImpl::pop_trade(shared_ptr<const PyTrade> &result) {
|
||||
return false;
|
||||
}
|
||||
|
||||
int MdBridgeImpl::run_py(volatile bool *keep_going) {
|
||||
bts_assert(keep_going != nullptr);
|
||||
PythonActivatorT<MdProvider> provider_activator{target, module_paths};
|
||||
provider = provider_activator.get_target([&](MdProvider &pro) {
|
||||
// This will start another thread in python, and call back into c++ from there.
|
||||
auto sw = Stopwatch::start_new();
|
||||
pro.start(this);
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py",
|
||||
json_formatter(fmt::format(
|
||||
"MdProvider.start took {:.2f}ms", sw.elapsed().total_millis())));
|
||||
});
|
||||
// Release gil to let python run
|
||||
py::gil_scoped_release gil_rel;
|
||||
while (*keep_going) {
|
||||
std::this_thread::sleep_for(std::chrono::seconds(1));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
MdBridgeImpl::~MdBridgeImpl() = default;
|
||||
|
||||
} // namespace bts::md::py_interop
|
||||
@@ -1,15 +1,13 @@
|
||||
#pragma once
|
||||
|
||||
#include "python_interop/md/md_bridge.h"
|
||||
#include "python_interop/md/md_provider.h"
|
||||
#include "spark/collections/concurrent_queue.h"
|
||||
|
||||
namespace bts::md::py_interop {
|
||||
|
||||
class MdBridgeImpl : public MdBridge {
|
||||
public:
|
||||
MdBridgeImpl(std::string target, std::vector<std::string> module_paths);
|
||||
~MdBridgeImpl();
|
||||
~MdBridgeImpl() override;
|
||||
|
||||
// Python interface
|
||||
void on_securities(
|
||||
@@ -21,16 +19,12 @@ class MdBridgeImpl : public MdBridge {
|
||||
bool pop_security(std::shared_ptr<const PySecurity> &result);
|
||||
bool pop_book(std::shared_ptr<const PyBook> &result);
|
||||
bool pop_trade(std::shared_ptr<const PyTrade> &result);
|
||||
int run_py(volatile bool *keep_going);
|
||||
|
||||
private:
|
||||
std::string target;
|
||||
std::vector<std::string> module_paths;
|
||||
collections::ConcurrentQueue<std::shared_ptr<const PySecurity>>
|
||||
incoming_securities;
|
||||
collections::ConcurrentQueue<std::shared_ptr<const PyBook>> incoming_books;
|
||||
collections::ConcurrentQueue<std::shared_ptr<const PyTrade>> incoming_trades;
|
||||
MdProvider *provider;
|
||||
};
|
||||
|
||||
} // namespace bts::md::py_interop
|
||||
@@ -0,0 +1,167 @@
|
||||
#include "plugin_manager.h"
|
||||
#include "python_interop/execution/execution_gateway.h"
|
||||
#include "python_interop/execution/execution_gateway_bridge.h"
|
||||
#include "python_interop/md/md_bridge.h"
|
||||
#include "python_interop/md/md_provider.h"
|
||||
#include "python_interop/plugin_manager_logs.h"
|
||||
#include "python_interop/python_object.h"
|
||||
#include "spark/utils/stopwatch.h"
|
||||
|
||||
#include <csignal>
|
||||
#include <fmt/core.h>
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <mutex>
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/gil.h>
|
||||
#include <string>
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
using bts::md::py_interop::MdBridge;
|
||||
using bts::utils::Stopwatch;
|
||||
using python_execution::ExecutionGateway;
|
||||
using python_execution::ExecutionGatewayBridge;
|
||||
using std::shared_ptr;
|
||||
using std::string;
|
||||
using std::vector;
|
||||
|
||||
PluginManager::~PluginManager() { stop_work_thread(); }
|
||||
|
||||
PythonWorkRequest::~PythonWorkRequest() = default;
|
||||
|
||||
void PluginManager::start_work_thread() {
|
||||
mx.lock();
|
||||
if (work_thread) {
|
||||
mx.unlock();
|
||||
return;
|
||||
}
|
||||
keep_going = true;
|
||||
mx.unlock();
|
||||
work_thread.emplace([&]() {
|
||||
pybind11::scoped_interpreter interpreter{false};
|
||||
try {
|
||||
pybind11::gil_scoped_release gil_rel;
|
||||
Stopwatch sw;
|
||||
int n = 0;
|
||||
while (keep_going) {
|
||||
auto work_req = work_queue.wait_until_pop();
|
||||
if (!work_req) {
|
||||
continue;
|
||||
}
|
||||
LOG_INVOKING(n);
|
||||
sw.restart();
|
||||
(**work_req).invoke();
|
||||
LOG_INVOKED(n, sw.elapsed());
|
||||
n++;
|
||||
}
|
||||
} catch (const std::exception &ex) {
|
||||
bts_assert_msgf(
|
||||
false, "PythonWorker work_thread threw. {}",
|
||||
std::string(ex.what()));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void PluginManager::stop_work_thread() {
|
||||
mx.lock();
|
||||
if (work_thread && work_thread->joinable()) {
|
||||
keep_going = false;
|
||||
mx.unlock();
|
||||
work_thread->join();
|
||||
work_thread.reset();
|
||||
} else {
|
||||
mx.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
struct PythonWorkRequestDelegate : PythonWorkRequest {
|
||||
std::function<void()> callback;
|
||||
|
||||
PythonWorkRequestDelegate(std::function<void()> callback)
|
||||
: callback(callback) {}
|
||||
|
||||
void invoke() override { callback(); }
|
||||
};
|
||||
|
||||
void PluginManager::invoke(std::function<void()> callback) {
|
||||
work_queue.push(std::make_shared<PythonWorkRequestDelegate>(callback));
|
||||
}
|
||||
|
||||
std::future<shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>
|
||||
PluginManager::start_md_provider(
|
||||
string provider_name,
|
||||
string target,
|
||||
vector<string> module_paths,
|
||||
shared_ptr<MdBridge> bridge) {
|
||||
auto prom = std::make_shared<std::promise<
|
||||
shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>>();
|
||||
std::function<void(py::object)> callback{
|
||||
[&, provider_name = std::move(provider_name), bridge = std::move(bridge),
|
||||
prom = prom](py::object obj) {
|
||||
auto [md_pro_it, md_pro_is_new] = md_providers.try_emplace(
|
||||
provider_name,
|
||||
std::make_shared<PythonObjectT<bts::md::py_interop::MdProvider>>(
|
||||
std::move(obj)));
|
||||
bts_assert(md_pro_is_new);
|
||||
LOG_STARTING("md_provider", provider_name);
|
||||
Stopwatch sw = Stopwatch::start_new();
|
||||
shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>> md_pro =
|
||||
md_pro_it->second;
|
||||
(*md_pro)->start(bridge.get());
|
||||
LOG_STARTED("md_provider", provider_name, sw.elapsed());
|
||||
prom->set_value(md_pro);
|
||||
// Don't let python handle the signals (otherwise it'll hang on ctrl+c)
|
||||
std::signal(SIGINT, SIG_DFL);
|
||||
}};
|
||||
invoke_on_obj(std::move(target), std::move(module_paths), callback);
|
||||
return prom->get_future();
|
||||
}
|
||||
|
||||
void PluginManager::stop_md_provider(const std::string &provider_name) {
|
||||
auto it = md_providers.find(provider_name);
|
||||
if (it != md_providers.end()) {
|
||||
(*it->second)->stop();
|
||||
md_providers.erase(it);
|
||||
}
|
||||
}
|
||||
|
||||
std::future<shared_ptr<PythonObjectT<ExecutionGateway>>>
|
||||
PluginManager::start_execution_gateway(
|
||||
string gateway_name,
|
||||
string target,
|
||||
vector<string> module_paths,
|
||||
shared_ptr<ExecutionGatewayBridge> bridge) {
|
||||
auto prom = std::make_shared<
|
||||
std::promise<shared_ptr<PythonObjectT<ExecutionGateway>>>>();
|
||||
std::function<void(py::object)> callback{[&, gateway_name = gateway_name,
|
||||
bridge = std::move(bridge),
|
||||
prom = prom](py::object obj) {
|
||||
auto [exe_gw_it, exe_gw_is_new] = execution_gateways.try_emplace(
|
||||
gateway_name,
|
||||
std::make_shared<PythonObjectT<ExecutionGateway>>(std::move(obj)));
|
||||
bts_assert(exe_gw_is_new);
|
||||
LOG_STARTING("execution_gateway", gateway_name);
|
||||
Stopwatch sw = Stopwatch::start_new();
|
||||
shared_ptr<PythonObjectT<ExecutionGateway>> exe_gw = exe_gw_it->second;
|
||||
(*exe_gw)->start(bridge.get());
|
||||
LOG_STARTED("execution_gateway", gateway_name, sw.elapsed());
|
||||
prom->set_value(exe_gw);
|
||||
// Don't let python handle the signals (otherwise it'll hang on ctrl+c)
|
||||
std::signal(SIGINT, SIG_DFL);
|
||||
}};
|
||||
invoke_on_obj(std::move(target), std::move(module_paths), callback);
|
||||
return prom->get_future();
|
||||
}
|
||||
|
||||
void PluginManager::stop_execution_gateway(const std::string &provider_name) {
|
||||
auto it = execution_gateways.find(provider_name);
|
||||
if (it != execution_gateways.end()) {
|
||||
(*it->second)->stop();
|
||||
execution_gateways.erase(it);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -0,0 +1,85 @@
|
||||
#pragma once
|
||||
|
||||
#include "python_interop/execution/execution_gateway.h"
|
||||
#include "python_interop/execution/execution_gateway_bridge.h"
|
||||
#include "python_interop/md/md_bridge.h"
|
||||
#include "python_interop/md/md_provider.h"
|
||||
#include "python_interop/python_object.h"
|
||||
#include "python_interop/python_work_request.h"
|
||||
#include "spark/collections/concurrent_queue.h"
|
||||
#include "spark/collections/concurrent_unordered_map.h"
|
||||
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <memory>
|
||||
#include <pybind11/pytypes.h>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
class PluginManager {
|
||||
volatile bool keep_going;
|
||||
std::mutex mx;
|
||||
std::optional<std::thread> work_thread;
|
||||
bts::collections::ConcurrentQueue<std::shared_ptr<PythonWorkRequest>>
|
||||
work_queue;
|
||||
bts::collections::ConcurrentUnorderedMap<
|
||||
std::string,
|
||||
std::shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>
|
||||
md_providers;
|
||||
bts::collections::ConcurrentUnorderedMap<
|
||||
std::string,
|
||||
std::shared_ptr<PythonObjectT<python_execution::ExecutionGateway>>>
|
||||
execution_gateways;
|
||||
|
||||
public:
|
||||
~PluginManager();
|
||||
|
||||
void start_work_thread();
|
||||
void stop_work_thread();
|
||||
|
||||
std::future<std::shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>
|
||||
start_md_provider(
|
||||
std::string provider_name,
|
||||
std::string target,
|
||||
std::vector<std::string> module_paths,
|
||||
std::shared_ptr<bts::md::py_interop::MdBridge> bridge);
|
||||
void stop_md_provider(const std::string &provider_name);
|
||||
|
||||
std::future<
|
||||
std::shared_ptr<PythonObjectT<python_execution::ExecutionGateway>>>
|
||||
start_execution_gateway(
|
||||
std::string gateway_name,
|
||||
std::string target,
|
||||
std::vector<std::string> module_paths,
|
||||
std::shared_ptr<python_execution::ExecutionGatewayBridge> bridge);
|
||||
void stop_execution_gateway(const std::string &gateway_name);
|
||||
|
||||
void invoke(std::function<void()> callback);
|
||||
|
||||
template <typename... Args>
|
||||
void invoke_on_obj(
|
||||
std::string target,
|
||||
std::vector<std::string> module_path,
|
||||
std::function<void(pybind11::object)> callback,
|
||||
Args &&...args);
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Definition
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
template <typename... Args>
|
||||
inline void PluginManager::invoke_on_obj(
|
||||
std::string target,
|
||||
std::vector<std::string> module_path,
|
||||
std::function<void(pybind11::object)> callback,
|
||||
Args &&...args) {
|
||||
work_queue.push(std::make_shared<PythonWorkRequestT<Args...>>(
|
||||
std::move(target), std::move(module_path), callback,
|
||||
std::forward<Args>(args)...));
|
||||
}
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -0,0 +1,115 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/core/FMT_IMPL.h"
|
||||
#include "spark/core/TimeSpan.h"
|
||||
#include "spark/logging/trace_log.h"
|
||||
#include "spark/os/os.h"
|
||||
#include "spark/utils/thread_id.h"
|
||||
|
||||
#include <fmt/core.h>
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/gil.h>
|
||||
#include <string>
|
||||
|
||||
namespace python_interop::logs {
|
||||
|
||||
struct StartingLog {
|
||||
std::string instance_name;
|
||||
std::string thread_name;
|
||||
std::int64_t thread_id;
|
||||
|
||||
StartingLog(std::string instance_name)
|
||||
: instance_name(std::move(instance_name)),
|
||||
thread_name(bts::os::get_thread_name()),
|
||||
thread_id(BTS_THIS_THREAD_ID) {}
|
||||
};
|
||||
|
||||
struct StartedLog {
|
||||
std::string instance_name;
|
||||
std::string thread_name;
|
||||
std::int64_t thread_id;
|
||||
bts::core::TimeSpan duration;
|
||||
|
||||
StartedLog(std::string instance_name, bts::core::TimeSpan duration)
|
||||
: instance_name(std::move(instance_name)),
|
||||
thread_name(bts::os::get_thread_name()), thread_id(BTS_THIS_THREAD_ID),
|
||||
duration(duration) {}
|
||||
};
|
||||
|
||||
struct InvokingLog {
|
||||
int count;
|
||||
std::string thread_name;
|
||||
std::int64_t thread_id;
|
||||
|
||||
InvokingLog(int count)
|
||||
: count(count), thread_name(bts::os::get_thread_name()),
|
||||
thread_id(BTS_THIS_THREAD_ID) {}
|
||||
};
|
||||
|
||||
struct InvokedLog {
|
||||
int count;
|
||||
bts::core::TimeSpan duration;
|
||||
std::string thread_name;
|
||||
std::int64_t thread_id;
|
||||
|
||||
InvokedLog(int count, bts::core::TimeSpan dequeue_duration)
|
||||
: count(count), duration(dequeue_duration),
|
||||
thread_name(bts::os::get_thread_name()),
|
||||
thread_id(BTS_THIS_THREAD_ID) {}
|
||||
};
|
||||
|
||||
} // namespace python_interop::logs
|
||||
|
||||
FMT_IMPL(
|
||||
python_interop::logs::StartingLog,
|
||||
l,
|
||||
R"({{ "name": "{}", "ev": "STARTING", "tn": "{}", "tid": "{}" }})",
|
||||
l.instance_name,
|
||||
l.thread_name,
|
||||
l.thread_id);
|
||||
|
||||
FMT_IMPL(
|
||||
python_interop::logs::StartedLog,
|
||||
l,
|
||||
R"({{ "name": "{}", "ev": "STARTED", "dur": "{:.3f}ms", "tn": "{}", "tid": "{}" }})",
|
||||
l.instance_name,
|
||||
l.duration.total_millis(),
|
||||
l.thread_name,
|
||||
l.thread_id);
|
||||
|
||||
FMT_IMPL(
|
||||
python_interop::logs::InvokingLog,
|
||||
l,
|
||||
R"({{ "ev": "INVOKING", "count": {}, "tn": "{}", "tid": "{}" }})",
|
||||
l.count,
|
||||
l.thread_name,
|
||||
l.thread_id);
|
||||
|
||||
FMT_IMPL(
|
||||
python_interop::logs::InvokedLog,
|
||||
l,
|
||||
R"({{ "ev": "INVOKED", "count": {}, "dur": "{:.3f}ms", "tn": "{}", "tid": "{}" }})",
|
||||
l.count,
|
||||
l.duration.total_millis(),
|
||||
l.thread_name,
|
||||
l.thread_id);
|
||||
|
||||
#define LOG_STARTING(type_name, instance_name) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
Gen, bts::Info, string("py_plugin"), string("start_" type_name), \
|
||||
python_interop::logs::StartingLog{instance_name})
|
||||
|
||||
#define LOG_STARTED(type_name, instance_name, dur) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
Gen, bts::Info, string("py_plugin"), string("start_" type_name), \
|
||||
python_interop::logs::StartedLog(instance_name, dur))
|
||||
|
||||
#define LOG_INVOKING(count) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
Gen, bts::Info, string("py_plugin"), string("work"), \
|
||||
python_interop::logs::InvokingLog(count))
|
||||
|
||||
#define LOG_INVOKED(count, duration) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
Gen, bts::Info, string("py_plugin"), string("work"), \
|
||||
python_interop::logs::InvokedLog(count, duration))
|
||||
@@ -1,93 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "python_interop/python_activator_impl.h"
|
||||
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
class PythonActivator {
|
||||
public:
|
||||
PythonActivator(std::string target, std::vector<std::string> module_paths)
|
||||
: me_(new PythonActivator_Impl(
|
||||
std::move(target), std::move(module_paths))) {}
|
||||
|
||||
PythonActivator() {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
delete me;
|
||||
}
|
||||
|
||||
pybind11::object &get_target_obj() {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
return me->get_target_obj();
|
||||
}
|
||||
|
||||
pybind11::object &get_target_obj(
|
||||
std::function<void(pybind11::object &)> init) {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
return me->get_target_obj(init);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
pybind11::object &get_target_obj(
|
||||
std::function<void(pybind11::object &)> init, Args &&...args) {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
return me->get_target_obj<Args...>(init, std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
void reset() {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
me->reset();
|
||||
}
|
||||
|
||||
private:
|
||||
void *me_;
|
||||
};
|
||||
|
||||
template <typename Target>
|
||||
class PythonActivatorT {
|
||||
public:
|
||||
PythonActivatorT(std::string target, std::vector<std::string> module_paths)
|
||||
: activator_(std::move(target), std::move(module_paths)) {}
|
||||
|
||||
pybind11::object &get_target_obj() { return activator_.get_target_obj(); }
|
||||
|
||||
Target *get_target() {
|
||||
if (target_ == nullptr) {
|
||||
target_ = activator_.get_target_obj().cast<Target *>();
|
||||
}
|
||||
return target_;
|
||||
}
|
||||
|
||||
Target *get_target(std::function<void(Target &)> init) {
|
||||
if (target_ == nullptr) {
|
||||
activator_.get_target_obj([&](pybind11::object &obj) {
|
||||
target_ = obj.cast<Target *>();
|
||||
init(*target_);
|
||||
});
|
||||
}
|
||||
return target_;
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
Target *get_target(std::function<void(Target &)> init, Args &&...args) {
|
||||
if (target_ == nullptr) {
|
||||
activator_.get_target_obj<Args...>(
|
||||
[&](pybind11::object &obj) {
|
||||
target_ = obj.cast<Target *>();
|
||||
init(*target_);
|
||||
},
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
return target_;
|
||||
}
|
||||
|
||||
private:
|
||||
PythonActivator activator_;
|
||||
Target *target_ = nullptr;
|
||||
};
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -1,118 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "python_interop/python_interpreter.h"
|
||||
#include "spark/core/TimeStamp.h"
|
||||
#include "spark/io/json_formatter.h"
|
||||
#include "spark/logging/logging.h"
|
||||
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
struct PythonActivator_Impl {
|
||||
std::string target;
|
||||
std::vector<std::string> module_paths;
|
||||
std::optional<pybind11::object> target_obj;
|
||||
|
||||
PythonActivator_Impl(
|
||||
std::string target, std::vector<std::string> module_paths)
|
||||
: target(std::move(target)), module_paths(std::move(module_paths)) {}
|
||||
|
||||
~PythonActivator_Impl() { reset(); }
|
||||
|
||||
pybind11::object &get_target_obj() {
|
||||
return get_target_obj([](pybind11::object &) {});
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
pybind11::object &get_target_obj(
|
||||
std::function<void(pybind11::object &)> init, Args &&...args) {
|
||||
if (!target_obj) {
|
||||
// Prevent python from overwriting our signal handler (so ctrl+c
|
||||
// actually terminates the process).
|
||||
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
|
||||
target_obj = create_target_with_args<Args...>(
|
||||
init, std::forward<Args>(args)...);
|
||||
std::signal(SIGINT, orig_signal_handler);
|
||||
}
|
||||
return *target_obj;
|
||||
}
|
||||
|
||||
// Undef after use (this is a header)
|
||||
#define LOG_EVENT(ev) \
|
||||
LOG(OrderMan, bts::Info, \
|
||||
R"([py_activator.json]{{ "tgt": "{}", "path": {}, "ev": )" ev \
|
||||
R"(, "ts": {} }})", \
|
||||
target, bts::io::json_formatter(module_paths), \
|
||||
bts::core::TimeStamp::now_ns())
|
||||
#define LOG_ERROR(err) \
|
||||
LOG(OrderMan, bts::Info, \
|
||||
R"([py_activator.json]{{ "tgt": "{}", "path": {}, "ev": "ERROR", "err": "{}", "ts": {} }})", \
|
||||
target, bts::io::json_formatter(module_paths), err, \
|
||||
bts::core::TimeStamp::now_ns())
|
||||
#define LOG_CREATE LOG_EVENT("CREATE")
|
||||
#define LOG_RESET LOG_EVENT("RESET")
|
||||
|
||||
void reset() {
|
||||
LOG_RESET;
|
||||
target_obj.reset();
|
||||
}
|
||||
|
||||
private:
|
||||
template <typename... Args>
|
||||
pybind11::object create_target_with_args(
|
||||
std::function<void(pybind11::object &)> init, Args &&...args) {
|
||||
LOG_CREATE;
|
||||
return PythonInterpreter::exec_t<pybind11::object>(
|
||||
[this, &init,
|
||||
args_tuple =
|
||||
std::make_tuple(std::forward<Args>(args)...)]() mutable {
|
||||
std::string target_module_name;
|
||||
std::string target_type_name;
|
||||
PythonInterpreter::parse_target(
|
||||
target, &target_module_name, &target_type_name);
|
||||
pybind11::object obj;
|
||||
try {
|
||||
// Prepare module path
|
||||
pybind11::module sys = pybind11::module::import("sys");
|
||||
for (const auto &module_path : module_paths) {
|
||||
sys.attr("path").attr("append")(module_path.c_str());
|
||||
}
|
||||
// Import target module
|
||||
pybind11::module target_module =
|
||||
pybind11::module::import(target_module_name.c_str());
|
||||
// Instantiate target using tuple expansion
|
||||
obj = std::apply(
|
||||
[&](auto &&...unpacked_args) {
|
||||
return target_module.attr(target_type_name.c_str())(
|
||||
std::forward<decltype(unpacked_args)>(
|
||||
unpacked_args)...);
|
||||
},
|
||||
std::move(args_tuple));
|
||||
init(obj);
|
||||
} catch (const std::exception &ex) {
|
||||
LOG_ERROR(std::string{ex.what()});
|
||||
}
|
||||
return obj;
|
||||
});
|
||||
}
|
||||
|
||||
pybind11::object create_target() {
|
||||
return create_target_with_args([](pybind11::object &) {});
|
||||
}
|
||||
|
||||
pybind11::object create_target(
|
||||
std::function<void(pybind11::object &)> init) {
|
||||
return create_target_with_args(init);
|
||||
}
|
||||
|
||||
#undef LOG_EVENT
|
||||
#undef LOG_ERROR
|
||||
#undef LOG_CREATE
|
||||
#undef LOG_RESET
|
||||
};
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -1,50 +0,0 @@
|
||||
#include "python_interpreter.h"
|
||||
#include "spark/logging/logging.h"
|
||||
#include "spark/utils/string_utils.h"
|
||||
|
||||
#include <csignal>
|
||||
#include <mutex>
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/gil.h>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
using bts::system::string_utils::snake_to_pascal_case;
|
||||
using bts::system::string_utils::split;
|
||||
|
||||
std::mutex PythonInterpreter::mx;
|
||||
std::optional<pybind11::scoped_interpreter> PythonInterpreter::interpreter;
|
||||
|
||||
void PythonInterpreter::init_interpreter() {
|
||||
if (interpreter.has_value()) {
|
||||
return;
|
||||
}
|
||||
static std::mutex local_mx;
|
||||
std::lock_guard<std::mutex> lock{local_mx};
|
||||
if (interpreter.has_value()) {
|
||||
return;
|
||||
}
|
||||
LOG(Gen, bts::Info, "Creating global py interpreter");
|
||||
// Don't let python handle the signals (otherwise it'll hang on ctrl+c)
|
||||
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
|
||||
interpreter.emplace(false);
|
||||
std::signal(SIGINT, orig_signal_handler);
|
||||
}
|
||||
|
||||
bool PythonInterpreter::parse_target(
|
||||
const std::string &target,
|
||||
std::string *module_name,
|
||||
std::string *type_name) {
|
||||
auto parts = split(target, ".");
|
||||
if (parts.empty()) {
|
||||
return false;
|
||||
}
|
||||
if (parts.size() == 1) {
|
||||
snake_to_pascal_case(parts.emplace_back(parts[0]));
|
||||
}
|
||||
*module_name = parts[0];
|
||||
*type_name = parts[1];
|
||||
return !module_name->empty() && !type_name->empty();
|
||||
}
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -1,125 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/utils/bts_assert.h"
|
||||
#include "spark/utils/string_utils.h"
|
||||
|
||||
#include <mutex>
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/gil.h>
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <string>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
class PythonInterpreter {
|
||||
static std::mutex mx;
|
||||
static std::optional<pybind11::scoped_interpreter> interpreter;
|
||||
static void init_interpreter();
|
||||
|
||||
public:
|
||||
/**
|
||||
* Parses module and type name from a \p target arg. The following formats are supported:
|
||||
* 1) <module_name>.<type_name>
|
||||
* 2) <module_name>
|
||||
* In this case, the type name defaults to the PascalCase equivalent of the module name.
|
||||
* For example, the module "foo_bar" would have type name "FooBar".
|
||||
*/
|
||||
static bool parse_target(
|
||||
const std::string &target,
|
||||
std::string *module_name,
|
||||
std::string *type_name);
|
||||
|
||||
static void exec(std::function<void()> fn) {
|
||||
init_interpreter();
|
||||
std::lock_guard<std::mutex> lock{mx};
|
||||
fn();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static T exec_t(std::function<T()> fn) {
|
||||
init_interpreter();
|
||||
std::lock_guard<std::mutex> lock{mx};
|
||||
pybind11::gil_scoped_acquire gil;
|
||||
T rv = fn();
|
||||
pybind11::gil_scoped_release gil_rel;
|
||||
return rv;
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static pybind11::object create_object(
|
||||
const std::vector<std::string> &module_paths,
|
||||
const std::string &target_name,
|
||||
Args &&...args) {
|
||||
std::string target_module_name;
|
||||
std::string target_type_name;
|
||||
bts_assert(
|
||||
parse_target(target_name, &target_module_name, &target_type_name));
|
||||
return create_object(
|
||||
module_paths, target_module_name, target_type_name,
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static pybind11::object create_object(
|
||||
const std::vector<std::string> &module_paths,
|
||||
const std::string &target_type_name,
|
||||
const std::string &target_module_name,
|
||||
Args &&...args) {
|
||||
namespace py = pybind11;
|
||||
return exec_t<py::object>([&]() {
|
||||
// Prepare module path
|
||||
py::module sys = py::module::import("sys");
|
||||
for (const auto &module_path : module_paths) {
|
||||
sys.attr("path").attr("append")(module_path.c_str());
|
||||
}
|
||||
// Import target module
|
||||
py::module target_module =
|
||||
py::module::import(target_module_name.c_str());
|
||||
// Instantiate target
|
||||
return target_module.attr(target_type_name.c_str())(
|
||||
std::forward<Args>(args)...);
|
||||
});
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static pybind11::object create_object_with_init(
|
||||
const std::vector<std::string> &module_paths,
|
||||
const std::string &target_name,
|
||||
std::function<void(pybind11::object &)> init_fn,
|
||||
Args &&...args) {
|
||||
std::string target_module_name;
|
||||
std::string target_type_name;
|
||||
bts_assert(
|
||||
parse_target(target_name, &target_module_name, &target_type_name));
|
||||
return create_object_with_init(
|
||||
module_paths, target_module_name, target_type_name, init_fn,
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static pybind11::object create_object_with_init(
|
||||
const std::vector<std::string> &module_paths,
|
||||
const std::string &target_type_name,
|
||||
const std::string &target_module_name,
|
||||
std::function<void(pybind11::object &)> init_fn,
|
||||
Args &&...args) {
|
||||
namespace py = pybind11;
|
||||
return exec_t<py::object>([&]() {
|
||||
// Prepare module path
|
||||
py::module sys = py::module::import("sys");
|
||||
for (const auto &module_path : module_paths) {
|
||||
sys.attr("path").attr("append")(module_path.c_str());
|
||||
}
|
||||
// Import target module
|
||||
py::module target_module =
|
||||
py::module::import(target_module_name.c_str());
|
||||
// Instantiate target
|
||||
py::object obj = target_module.attr(target_type_name.c_str())(
|
||||
std::forward<Args>(args)...);
|
||||
init_fn(obj);
|
||||
return obj;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -0,0 +1,20 @@
|
||||
#pragma once
|
||||
|
||||
#include <pybind11/pytypes.h>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
template <typename T>
|
||||
class PythonObjectT {
|
||||
pybind11::object obj_;
|
||||
T *tp_ = nullptr;
|
||||
|
||||
public:
|
||||
PythonObjectT(pybind11::object obj)
|
||||
: obj_(std::move(obj)), tp_(obj_.cast<T *>()) {}
|
||||
|
||||
T &operator*() { return *tp_; }
|
||||
T *operator->() { return tp_; }
|
||||
};
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -0,0 +1,95 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/core/TimeStamp.h"
|
||||
#include "spark/utils/bts_assert.h"
|
||||
#include "spark/utils/string_utils.h"
|
||||
|
||||
#include <functional>
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/gil.h>
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <pybind11/pytypes.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
struct PythonWorkRequest {
|
||||
virtual ~PythonWorkRequest();
|
||||
|
||||
static bool parse_target(
|
||||
const std::string &target,
|
||||
std::string *module_name,
|
||||
std::string *type_name);
|
||||
|
||||
virtual void invoke() = 0;
|
||||
};
|
||||
|
||||
template <typename... Args>
|
||||
struct PythonWorkRequestT : PythonWorkRequest {
|
||||
std::string target;
|
||||
std::vector<std::string> module_paths;
|
||||
std::function<void(pybind11::object)> callback;
|
||||
std::tuple<Args...> args;
|
||||
|
||||
PythonWorkRequestT(
|
||||
std::string target,
|
||||
std::vector<std::string> module_paths,
|
||||
std::function<void(pybind11::object)> callback,
|
||||
Args... args)
|
||||
: target(std::move(target)), module_paths(std::move(module_paths)),
|
||||
callback(callback), args(std::forward<Args>(args)...) {}
|
||||
|
||||
~PythonWorkRequestT() override = default;
|
||||
|
||||
void invoke() override;
|
||||
};
|
||||
|
||||
inline bool PythonWorkRequest::parse_target(
|
||||
const std::string &target,
|
||||
std::string *module_name,
|
||||
std::string *type_name) {
|
||||
auto parts = bts::system::string_utils::split(target, ".");
|
||||
if (parts.empty()) {
|
||||
return false;
|
||||
}
|
||||
if (parts.size() == 1) {
|
||||
bts::system::string_utils::snake_to_pascal_case(
|
||||
parts.emplace_back(parts[0]));
|
||||
}
|
||||
*module_name = parts[0];
|
||||
*type_name = parts[1];
|
||||
return !module_name->empty() && !type_name->empty();
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
inline void PythonWorkRequestT<Args...>::invoke() {
|
||||
std::string target_module_name;
|
||||
std::string target_type_name;
|
||||
bts_assert(parse_target(target, &target_module_name, &target_type_name));
|
||||
LOG(Gen, bts::Info,
|
||||
R"([py_worker.json]{{ "ev": "INVOKE", "mod": "{}", "typ": "{}", "args_t": "{}", "ts": {} }})",
|
||||
target_module_name, target_type_name,
|
||||
std::string(typeid(decltype(this->args)).name()),
|
||||
bts::core::TimeStamp::now_ns());
|
||||
try {
|
||||
pybind11::gil_scoped_acquire gil_acquire;
|
||||
pybind11::module sys = pybind11::module::import("sys");
|
||||
for (const auto &module_path : module_paths) {
|
||||
sys.attr("path").attr("append")(module_path.c_str());
|
||||
}
|
||||
pybind11::module target_module =
|
||||
pybind11::module::import(target_module_name.c_str());
|
||||
std::apply(
|
||||
[&](auto &&...unpacked_args) {
|
||||
callback(target_module.attr(target_type_name.c_str())(
|
||||
std::forward<decltype(unpacked_args)>(unpacked_args)...));
|
||||
},
|
||||
std::move(args));
|
||||
} catch (const std::exception &ex) {
|
||||
bts_assert_msgf(
|
||||
false, "PythonWorkRequest::invoke threw. {}", std::string(ex.what()));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -23,6 +23,7 @@ from python_interop_module import (
|
||||
PySecurity,
|
||||
PySecurityList,
|
||||
)
|
||||
from spark_enums import OrderStatus, ExecType
|
||||
import threading
|
||||
|
||||
|
||||
@@ -36,7 +37,8 @@ class OrderData:
|
||||
self.last_filled_quantity: float = 0
|
||||
self.avg_filled_price: float = 0
|
||||
self.last_filled_price: float = 0
|
||||
self.exec_type = "PENDING_NEW"
|
||||
self.exec_type: ExecType = ExecType.PENDING_NEW
|
||||
self.status: OrderStatus = OrderStatus.PENDING_NEW
|
||||
|
||||
def on_update(
|
||||
self,
|
||||
@@ -63,19 +65,29 @@ class OrderData:
|
||||
self.filled_quantity = filled_quantity
|
||||
self.avg_filled_price = avg_filled_price
|
||||
match status:
|
||||
case "New":
|
||||
self.exec_type = ExecType.NEW
|
||||
self.status = OrderStatus.NEW
|
||||
case "Filled":
|
||||
self.exec_type = "FILL"
|
||||
case "Canceled":
|
||||
self.exec_type = "CANCELED"
|
||||
self.exec_type = ExecType.FILL
|
||||
self.status = OrderStatus.FILLED
|
||||
case "PartiallyFilled":
|
||||
self.exec_type = ExecType.PARTIAL_FILL
|
||||
self.status = OrderStatus.PARTIALLY_FILLED
|
||||
case "Cancelled":
|
||||
self.exec_type = ExecType.CANCELED
|
||||
self.status = OrderStatus.CANCELED
|
||||
case "Rejected":
|
||||
self.exec_type = "REJECTED"
|
||||
self.exec_type = ExecType.REJECTED
|
||||
self.status = OrderStatus.REJECTED
|
||||
case _:
|
||||
self.exec_type = "NEW" if filled_quantity == 0 else "PARTIAL_FILL"
|
||||
raise ValueError(f"Unexpected order status '{status}'")
|
||||
|
||||
|
||||
class BybitExecutionGateway(ExecutionGateway):
|
||||
def __init__(self, name: str):
|
||||
def __init__(self):
|
||||
super(BybitExecutionGateway, self).__init__()
|
||||
name = "bybit_1"
|
||||
self.name = name
|
||||
self.logger = configure_logger(logging.getLogger(name), "OrderMan")
|
||||
self.logger.info("[bybit_gw_ctr.json]%s" % SLog(name=name))
|
||||
@@ -135,15 +147,6 @@ class BybitExecutionGateway(ExecutionGateway):
|
||||
total_filled_qty = parse_float(d["cumExecQty"])
|
||||
avg_filled_price = parse_float(d["avgPrice"])
|
||||
status: str = d["orderStatus"]
|
||||
# brits can't spell
|
||||
if status == "Cancelled":
|
||||
status = "Canceled"
|
||||
exec_type = ord_data.exec_type
|
||||
match ord_data.exec_type:
|
||||
case "PARTIAL_FILL":
|
||||
exec_type = "PARTIALLY_FILLED"
|
||||
case _:
|
||||
pass
|
||||
px = parse_float(d["price"])
|
||||
qty = parse_float(d["qty"])
|
||||
ord_data.on_update(
|
||||
@@ -162,8 +165,8 @@ class BybitExecutionGateway(ExecutionGateway):
|
||||
price=px,
|
||||
quantity=qty,
|
||||
time_in_force=d["timeInForce"],
|
||||
exec_type=exec_type,
|
||||
order_status=status,
|
||||
exec_type=ord_data.exec_type.name,
|
||||
order_status=ord_data.status.name,
|
||||
last_filled_price=ord_data.last_filled_price,
|
||||
last_filled_quantity=ord_data.last_filled_quantity,
|
||||
filled_quantity=total_filled_qty,
|
||||
@@ -198,7 +201,7 @@ class BybitExecutionGateway(ExecutionGateway):
|
||||
params = {
|
||||
"api_key": self.api_key,
|
||||
"api_secret": self.secret_key,
|
||||
"testnet": True,
|
||||
"testnet": self.use_testnet,
|
||||
}
|
||||
self.logger.info("[bybit_gw_start_trd_cli.json]%s" % json.dumps(params))
|
||||
self.trade_cli = WebSocketTrading(**params)
|
||||
@@ -206,6 +209,10 @@ class BybitExecutionGateway(ExecutionGateway):
|
||||
"category": "linear",
|
||||
}
|
||||
self.bridge.on_is_ready(True)
|
||||
while True:
|
||||
if not self.stream_cli.is_connected():
|
||||
# try reconnecting
|
||||
pass
|
||||
|
||||
def start(self, bridge: ExecutionGatewayBridge):
|
||||
def start_wrapper():
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class OrderStatus(Enum):
|
||||
NEW = 1
|
||||
PENDING_NEW = 2
|
||||
PARTIALLY_FILLED = 3
|
||||
FILLED = 4
|
||||
CANCELED = 5
|
||||
REJECTED = 6
|
||||
|
||||
|
||||
class ExecType(Enum):
|
||||
NEW = 1
|
||||
PENDING_NEW = 2
|
||||
PARTIAL_FILL = 3
|
||||
FILL = 4
|
||||
CANCELED = 5
|
||||
REJECTED = 6
|
||||
Reference in new issue
Block a user