diff --git a/src/app_sparkd/sparkd/cmds/Run.cpp b/src/app_sparkd/sparkd/cmds/Run.cpp index 53b7d17cd..df1d05cc7 100644 --- a/src/app_sparkd/sparkd/cmds/Run.cpp +++ b/src/app_sparkd/sparkd/cmds/Run.cpp @@ -1,4 +1,5 @@ #include "Run.h" +#include "python_interop/plugin_manager.h" #include "spark/api/public/ServiceContext.h" #include "spark/config/config.h" #include "spark/config/protos/ports.pb.h" @@ -53,6 +54,7 @@ using bts::orderman::ExecutionGatewayProvider; using bts::orderman::IExecutionGatewayProvider; using bts::orderman::OrderServiceServer; using bts::trace::PerfLog; +using std::shared_ptr; using std::unique_ptr; using std::vector; @@ -62,14 +64,16 @@ static OrderServiceServer create_order_service_server( DynamicOrderRegistry &dyn_order_registry, MetricsContainer &stats, PerfLog &perf_log, - TradeSummarySignalDispatcher &tss_dispatcher) { + TradeSummarySignalDispatcher &tss_dispatcher, + shared_ptr py_plugin_man) { vector> gw_pros; gw_pros.push_back(std::make_unique( &perf_log, mdclient.oracle(), config::get(), [](const std::string &name) { const spark::SparkDataDirectory data_dir; return data_dir.load_config_json(name); - })); + }, + std::move(py_plugin_man))); return OrderServiceServer( sc, mdclient, dyn_order_registry, std::move(gw_pros), stats, perf_log, tss_dispatcher); @@ -99,7 +103,11 @@ static void execute() { bts::md::MBOProcessor mbo_processor; bts::md::TradeSummarySignalDispatcher tss_dispatcher; - md::cmds::Run::run(); + // The py plugin manager is shared by md and execution, so it lives in this scope. + auto py_plugin_man = std::make_shared(); + // start work loop in background + py_plugin_man->start_work_thread(); + md::cmds::Run::run(py_plugin_man); api::synthetic::SyntheticSecurityService synthetic_service(sc); bts::md::MDClient mdclient( @@ -120,7 +128,8 @@ static void execute() { // finished initializing order plugins auto order_service_server = create_order_service_server( - sc, mdclient, dyn_order_registry, stats, perf_log, tss_dispatcher); + sc, mdclient, dyn_order_registry, stats, perf_log, tss_dispatcher, + py_plugin_man); support::ErrorReportServer error_report_service(sc); admin::AdminServiceServer admin_service(sc); bts::md::auxiliary::MarketDataAuxiliaryServiceServer md_aux_server(sc); diff --git a/src/app_sparkd/sparkd/cmds/crypto/LiveLadderDemo.cpp b/src/app_sparkd/sparkd/cmds/crypto/LiveLadderDemo.cpp index d137a2b05..74ad59993 100644 --- a/src/app_sparkd/sparkd/cmds/crypto/LiveLadderDemo.cpp +++ b/src/app_sparkd/sparkd/cmds/crypto/LiveLadderDemo.cpp @@ -1,4 +1,5 @@ #include "LiveLadderDemo.h" +#include "python_interop/plugin_manager.h" #include "spark/api/md/MarketDataConfig.pb.h" #include "spark/persistence/IDGenerator.h" #include "sparkd/md/MarketDataGateway.h" @@ -15,7 +16,9 @@ LiveLadderDemo::LiveLadderDemo(const MarketDataConfig &config) { char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt"; putenv(ssl_cert_file); - md::MarketDataGateway gw{config, IDGenerator::instance_ptr()}; + auto py_plugin_man = std::make_shared(); + py_plugin_man->start_work_thread(); + md::MarketDataGateway gw{config, IDGenerator::instance_ptr(), py_plugin_man}; md::listeners::LiveLadderPrinter ladder_printer; gw.start(); diff --git a/src/app_sparkd/sparkd/cmds/md/cmds/Run.cpp b/src/app_sparkd/sparkd/cmds/md/cmds/Run.cpp index 7ab9f6a83..ab4c3487f 100644 --- a/src/app_sparkd/sparkd/cmds/md/cmds/Run.cpp +++ b/src/app_sparkd/sparkd/cmds/md/cmds/Run.cpp @@ -1,5 +1,6 @@ #include "Run.h" #include "Cleanup.h" +#include "python_interop/plugin_manager.h" #include "spark/api/md/MarketDataConfig.pb.h" #include "spark/config/protos/market_data.pb.h" #include "spark/domain/SecurityCriteria.h" @@ -92,12 +93,13 @@ static void run_onixs( disp.run(); } -static void run_md_gwy() { +static void run_md_gwy( + std::shared_ptr py_worker) { LOG(md, Info, "Run::run_md_gwy t_id:{} t_name:{}", os::get_thread_id(), os::get_thread_name()); bts::md::MarketDataGateway gwy{ config::get(), - persistence::IDGenerator::instance_ptr()}; + persistence::IDGenerator::instance_ptr(), std::move(py_worker)}; gwy.start(); while (true) { using namespace std::chrono_literals; @@ -105,12 +107,15 @@ static void run_md_gwy() { } } -void Run::run() { +void Run::run(std::shared_ptr py_worker) { md::cmds::Cleanup::execute(); LOG(md, Info, "Run::run t_id:{} t_name:{}", os::get_thread_id(), os::get_thread_name()); // start a thread for the md - threading::make_thread("MarketDataGateway", run_md_gwy).detach(); + threading::make_thread( + "MarketDataGateway", + [py_worker = std::move(py_worker)]() { run_md_gwy(py_worker); }) + .detach(); boot_sleep(); } diff --git a/src/app_sparkd/sparkd/cmds/md/cmds/Run.h b/src/app_sparkd/sparkd/cmds/md/cmds/Run.h index 169a0c68a..c1d7846c8 100644 --- a/src/app_sparkd/sparkd/cmds/md/cmds/Run.h +++ b/src/app_sparkd/sparkd/cmds/md/cmds/Run.h @@ -2,6 +2,10 @@ #include +namespace python_interop { +class PluginManager; +} + namespace bts::cmds::md::cmds { /** @@ -9,18 +13,15 @@ namespace bts::cmds::md::cmds { * shared memory. */ struct Run : public CLI::App { - enum class MDType { - Configured, - Replay, - Onixs - }; + enum class MDType { Configured, Replay, Onixs }; Run(); /** * Run market data */ - static void run(); + static void run( + std::shared_ptr py_worker = nullptr); }; } // namespace bts::cmds::md::cmds diff --git a/src/app_sparkd/sparkd/execution/execution_gateway_provider.cpp b/src/app_sparkd/sparkd/execution/execution_gateway_provider.cpp index 280c2869a..68ba9cffe 100644 --- a/src/app_sparkd/sparkd/execution/execution_gateway_provider.cpp +++ b/src/app_sparkd/sparkd/execution/execution_gateway_provider.cpp @@ -5,6 +5,7 @@ #include "binance_execution/usds_futures/usds_futures_gateway.h" #include "okx_execution/okx_execution_gateway.h" #include "onixs_fix/OnixFIXAdapter.h" +#include "python_interop/plugin_manager.h" #include "spark/api/domain/Venue.h" #include "spark/domain/DomainSecurityUtils.h" #include "spark/domain_model/Security.pb.h" @@ -37,6 +38,7 @@ using bts::io::JsonReader; using nlohmann::json; using std::function; using std::optional; +using std::shared_ptr; using std::string; using std::vector; @@ -56,14 +58,17 @@ struct ExecutionGatewayProvider::PImpl { trace::PerfLog *l, md::SecurityOracle o, config::JetFIX config, - function(const string &)> load_config_json) + function(const string &)> load_config_json, + shared_ptr py_plugin_man) : l(l), o(std::move(o)), config(std::move(config)), - load_config_json(std::move(load_config_json)) {} + load_config_json(std::move(load_config_json)), + py_plugin_man(std::move(py_plugin_man)) {} trace::PerfLog *l; md::SecurityOracle o; config::JetFIX config; function(const string &)> load_config_json; + shared_ptr py_plugin_man; template T parse_config_json(const string &name) { @@ -87,6 +92,9 @@ struct ExecutionGatewayProvider::PImpl { // but with a "py_" prefix, assume that file defines python gateway // config for that connection. // ************************************* + if (SimExecutionGatewayFactory::is_sim_execution_on(conn.venue())) { + return make_gw(conn, conn.venue()); + } auto py_cfg_j = load_config_json(fmt::format("py_{}", conn.name())); if (py_cfg_j) { auto py_cfg = py_cfg_j->get(); @@ -94,10 +102,7 @@ struct ExecutionGatewayProvider::PImpl { domain::DomainSecurityUtils::convert(conn.venue()) == py_cfg.venue); return make_gw( - conn, o, std::move(py_cfg), conn); - } - if (SimExecutionGatewayFactory::is_sim_execution_on(conn.venue())) { - return make_gw(conn, conn.venue()); + conn, o, std::move(py_cfg), conn, py_plugin_man); } switch (conn.venue()) { case api::VENUE_ICE: @@ -161,9 +166,14 @@ ExecutionGatewayProvider::ExecutionGatewayProvider( md::SecurityOracle o, config::JetFIX config, std::function(const std::string &)> - load_config_json) + load_config_json, + shared_ptr py_plugin_man) : me_(std::make_unique( - l, std::move(o), std::move(config), std::move(load_config_json))) {} + l, + std::move(o), + std::move(config), + std::move(load_config_json), + std::move(py_plugin_man))) {} ExecutionGatewayProvider::~ExecutionGatewayProvider() = default; diff --git a/src/app_sparkd/sparkd/execution/execution_gateway_provider.h b/src/app_sparkd/sparkd/execution/execution_gateway_provider.h index c978f8678..5f23a2d9e 100644 --- a/src/app_sparkd/sparkd/execution/execution_gateway_provider.h +++ b/src/app_sparkd/sparkd/execution/execution_gateway_provider.h @@ -9,6 +9,10 @@ namespace bts::spark { class SparkDataDirectory; } +namespace python_interop { +class PluginManager; +} + namespace bts::orderman { class ExecutionGatewayProvider : public IExecutionGatewayProvider { @@ -18,7 +22,8 @@ class ExecutionGatewayProvider : public IExecutionGatewayProvider { md::SecurityOracle o, config::JetFIX config, std::function(const std::string &)> - load_config_json); + load_config_json, + std::shared_ptr py_plugin_man); ~ExecutionGatewayProvider() override; std::vector create() override; diff --git a/src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.cpp b/src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.cpp index 84bbf89b2..401521fe2 100644 --- a/src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.cpp +++ b/src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.cpp @@ -7,11 +7,12 @@ #include "python_interop/execution/py_create_order_request.h" #include "python_interop/execution/py_execution_gateway.h" #include "python_interop/execution/py_execution_report.h" +#include "python_interop/plugin_manager.h" #include "python_interop/py_contract_price_info.h" #include "python_interop/py_contract_quantity_info.h" #include "python_interop/py_contract_specification.h" #include "python_interop/py_security.h" -#include "python_interop/python_activator.h" +#include "python_interop/python_object.h" #include "spark/api/domain/Venue.h" #include "spark/api/service_helpers/AdminServiceUtils.h" #include "spark/api/services/AdminService.pb.h" @@ -59,6 +60,7 @@ #include "sparkd/execution/py_execution_gateway_config.h" #include +#include #include #include #include @@ -126,12 +128,13 @@ using bts::orderman::OrderMsg; using bts::orderman::RFQMsg; using bts::system::string_utils::parse_uint; using nlohmann::json; +using python_execution::ExecutionGateway; using python_execution::ExecutionGatewayBridgeImpl; using python_execution::PyCancelOrderRequest; using python_execution::PyCancelReplaceOrderRequest; using python_execution::PyCreateOrderRequest; -using python_execution::PyExecutionGateway; -using python_interop::PythonActivatorT; +using python_interop::PluginManager; +using python_interop::PythonObjectT; using std::int64_t; using std::optional; using std::shared_ptr; @@ -154,21 +157,20 @@ struct PyExecutionGatewayAdapter::PImpl { explicit PImpl( bts::md::SecurityOracle sec_oracle, PyExecutionGatewayConfig config, - bts::config::JetFIX::Connection jetfix_connection) + bts::config::JetFIX::Connection jetfix_connection, + shared_ptr plugin_man) : sec_oracle(std::move(sec_oracle)), config(std::move(config)), - bridge( - jetfix_connection.name(), - this->config.gateway_type, - this->config.module_path), + name(fmt::format("{}_py_gw", string{jetfix_connection.name()})), + bridge_ptr(std::make_shared( + jetfix_connection.name())), + bridge(*bridge_ptr), plugin_man(std::move(plugin_man)), jetfix_connection(std::make_shared( std::move(jetfix_connection))), orders(this->config.account, this->jetfix_connection) { bts_assert(this->jetfix_connection->venue() != api::VENUE_UNSET); bts_assert(!this->jetfix_connection->name().empty()); pos.exchange = this->jetfix_connection->venue(); - sm.set_name(fmt::format( - "{}_py_gw", - string{bts::api::Venue_Name(this->jetfix_connection->venue())})); + sm.set_name(name); sm.set_subsystem(bts::log::OrderMan); sm.start_core_impl.bind<&PImpl::start_impl>(*this); sm.stop_core_impl.bind<&PImpl::stop_impl>(*this); @@ -183,8 +185,10 @@ struct PyExecutionGatewayAdapter::PImpl { SecurityOracle sec_oracle; PyExecutionGatewayConfig config; - ExecutionGatewayBridgeImpl bridge; - optional> py_gw_activator; + std::string name; + std::shared_ptr bridge_ptr; + ExecutionGatewayBridgeImpl &bridge; + shared_ptr plugin_man; shared_ptr jetfix_connection; optional pos_snapshot_thread; optional py_thread; @@ -247,7 +251,17 @@ struct PyExecutionGatewayAdapter::PImpl { INFO_TRACE("start_impl"); bts_assert(!py_thread); py_keep_going = true; - py_thread.emplace([&]() { bridge.run_py(&py_keep_going); }); + py_thread.emplace([&]() { + auto orig_signal_handler = std::signal(SIGINT, SIG_DFL); + std::future>> gw_fut = + plugin_man->start_execution_gateway( + name, config.gateway_type, config.module_path, bridge_ptr); + gw_fut.wait(); + std::signal(SIGINT, orig_signal_handler); + bridge.run_py(&py_keep_going, gw_fut.get(), [&](auto callback) { + plugin_man->invoke(callback); + }); + }); } void stop() { @@ -519,11 +533,13 @@ struct PyExecutionGatewayAdapter::PImpl { PyExecutionGatewayAdapter::PyExecutionGatewayAdapter( SecurityOracle sec_oracle, PyExecutionGatewayConfig session_cfg, - bts::config::JetFIX::Connection jetfix_connection) + bts::config::JetFIX::Connection jetfix_connection, + shared_ptr plugin_man) : me(std::make_unique( std::move(sec_oracle), std::move(session_cfg), - std::move(jetfix_connection))) { + std::move(jetfix_connection), + std::move(plugin_man))) { bts_assert( me->config.account.size() <= ExecutionReport{}.account_name.value().max_size()); diff --git a/src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.h b/src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.h index 9e5e85d81..f6eadb317 100644 --- a/src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.h +++ b/src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.h @@ -6,6 +6,10 @@ #include +namespace python_interop { +class PluginManager; +} + namespace bts::orderman { struct PyExecutionGatewayConfig; @@ -15,7 +19,8 @@ class PyExecutionGatewayAdapter : public bts::orderman::IExecutionGateway { explicit PyExecutionGatewayAdapter( bts::md::SecurityOracle sec_oracle, PyExecutionGatewayConfig session_cfg, - bts::config::JetFIX::Connection jetfix_connection); + bts::config::JetFIX::Connection jetfix_connection, + std::shared_ptr plugin_man); ~PyExecutionGatewayAdapter() override; void start() override; diff --git a/src/app_sparkd/sparkd/md/MarketDataGateway.cpp b/src/app_sparkd/sparkd/md/MarketDataGateway.cpp index 4dff382af..e09c14611 100644 --- a/src/app_sparkd/sparkd/md/MarketDataGateway.cpp +++ b/src/app_sparkd/sparkd/md/MarketDataGateway.cpp @@ -1,4 +1,5 @@ #include "MarketDataGateway.h" +#include "python_interop/plugin_manager.h" #include "spark/api/domain/Venue.h" #include "spark/api/md/MarketDataConfig.pb.h" #include "spark/api/md/MarketDataConfigUtils.h" @@ -67,7 +68,10 @@ struct MarketDataGateway::PImpl { vector> sinks; vector> sources; - PImpl(MarketDataConfig config, shared_ptr sec_id_store) + PImpl( + MarketDataConfig config, + shared_ptr sec_id_store, + shared_ptr py_plugin_man) : config(std::move(config)), sec_id_store(std::move(sec_id_store)) { providers.push_back(std::make_unique( this->config, this->sec_id_store)); @@ -80,7 +84,7 @@ struct MarketDataGateway::PImpl { providers.push_back(std::make_unique( this->config, this->sec_id_store)); providers.push_back(std::make_unique( - this->config, this->sec_id_store)); + this->config, this->sec_id_store, std::move(py_plugin_man))); providers.push_back(std::make_unique( this->config, this->sec_id_store)); } @@ -126,9 +130,13 @@ struct MarketDataGateway::PImpl { }; MarketDataGateway::MarketDataGateway( - MarketDataConfig config, shared_ptr sec_id_store) + MarketDataConfig config, + shared_ptr sec_id_store, + std::shared_ptr py_plugin_man) : me_(std::make_unique( - std::move(config), std::move(sec_id_store))) {} + std::move(config), + std::move(sec_id_store), + std::move(py_plugin_man))) {} MarketDataGateway::~MarketDataGateway() = default; diff --git a/src/app_sparkd/sparkd/md/MarketDataGateway.h b/src/app_sparkd/sparkd/md/MarketDataGateway.h index 11e2bc3a0..66fa12aed 100644 --- a/src/app_sparkd/sparkd/md/MarketDataGateway.h +++ b/src/app_sparkd/sparkd/md/MarketDataGateway.h @@ -10,6 +10,10 @@ namespace bts::persistence { class IDGenerator; } +namespace python_interop { +class PluginManager; +} + namespace bts::md { /** @@ -20,7 +24,8 @@ class MarketDataGateway { public: explicit MarketDataGateway( api::md::MarketDataConfig config, - std::shared_ptr sec_id_store); + std::shared_ptr sec_id_store, + std::shared_ptr py_plugin_man); ~MarketDataGateway(); void start(); diff --git a/src/app_sparkd/sparkd/md/PyMarketDataProvider.cpp b/src/app_sparkd/sparkd/md/PyMarketDataProvider.cpp index 7169137e8..3c28e1153 100644 --- a/src/app_sparkd/sparkd/md/PyMarketDataProvider.cpp +++ b/src/app_sparkd/sparkd/md/PyMarketDataProvider.cpp @@ -1,4 +1,5 @@ #include "PyMarketDataProvider.h" +#include "python_interop/plugin_manager.h" #include "spark/api/md/MarketDataConfig.pb.h" #include "spark/md/IMarketDataSource.h" #include "spark/persistence/IDGenerator.h" @@ -19,16 +20,23 @@ using Path = boost::filesystem::path; struct PyMarketDataProvider::PImpl { MarketDataConfig config; shared_ptr sec_id_store; + shared_ptr py_worker; vector> py_md_subs; - PImpl(MarketDataConfig config, shared_ptr sec_id_store) - : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {} + PImpl( + MarketDataConfig config, + shared_ptr sec_id_store, + shared_ptr py_worker) + : config(std::move(config)), sec_id_store(std::move(sec_id_store)), + py_worker(std::move(py_worker)) {} }; PyMarketDataProvider::PyMarketDataProvider( - MarketDataConfig config, shared_ptr sec_id_store) - : me_(std::make_unique(std::move(config), std::move(sec_id_store))) { -} + MarketDataConfig config, + shared_ptr sec_id_store, + shared_ptr py_worker) + : me_(std::make_unique( + std::move(config), std::move(sec_id_store), std::move(py_worker))) {} PyMarketDataProvider::~PyMarketDataProvider() = default; @@ -38,7 +46,7 @@ PyMarketDataProvider::get_sources() { for (const auto &py_plugin : me_->config.python_md_providers()) { try { auto source = std::make_shared( - py_plugin, me_->sec_id_store.get()); + py_plugin, me_->sec_id_store.get(), me_->py_worker); me_->py_md_subs.push_back(source); rv.emplace_back(py_plugin.venue(), std::move(source)); } catch (const std::exception &ex) { diff --git a/src/app_sparkd/sparkd/md/PyMarketDataProvider.h b/src/app_sparkd/sparkd/md/PyMarketDataProvider.h index 245abc126..cf941329c 100644 --- a/src/app_sparkd/sparkd/md/PyMarketDataProvider.h +++ b/src/app_sparkd/sparkd/md/PyMarketDataProvider.h @@ -7,13 +7,18 @@ #include +namespace python_interop { +class PluginManager; +} + namespace bts::md { class PyMarketDataProvider : public IMarketDataProvider { public: explicit PyMarketDataProvider( api::md::MarketDataConfig config, - std::shared_ptr sec_id_store); + std::shared_ptr sec_id_store, + std::shared_ptr py_worker); ~PyMarketDataProvider(); std::vector< diff --git a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp index 86294ae5f..729f803e0 100644 --- a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp +++ b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp @@ -2,8 +2,10 @@ #include "python_interop/md/md_bridge_impl.h" #include "python_interop/md/py_book.h" #include "python_interop/md/py_trade.h" +#include "python_interop/plugin_manager.h" #include "python_interop/py_contract_specification.h" #include "python_interop/py_security.h" +#include "python_interop/python_object.h" #include "spark/api/domain/CallPut.pb.h" #include "spark/api/domain/SecurityType.h" #include "spark/api/domain/Venue.h" @@ -26,6 +28,7 @@ #include "spark/utils/bts_assert.h" #include "spark/utils/stopwatch.h" +#include #include #include #include @@ -45,6 +48,10 @@ namespace bts::md { #define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name) #define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name) +#define INFO_TRACE_PARAMS(name, params) \ + LOG_TRACE_PARAMS( \ + md, bts::Info, std::string("py_md_sub"), std::string(name), params) + #define LOG_DISPATCH_STATS(level, typ, n, dur) \ _LOG( \ level, \ @@ -101,11 +108,14 @@ struct TradeTuple { }; struct PyMarketDataSubscription::PImpl { - PImpl(PyMarketDataProviderConfig config, IDGenerator *sec_id_store) - : config(std::move(config)), bridge( - this->config.provider_type(), - {this->config.module_path().begin(), - this->config.module_path().end()}), + PImpl( + PyMarketDataProviderConfig config, + IDGenerator *sec_id_store, + shared_ptr py_plugin_man) + : config(std::move(config)), + name(fmt::format("{}_md_pro", string{this->config.provider_type()})), + bridge_ptr(std::make_shared()), + bridge(*bridge_ptr), py_plugin_man(std::move(py_plugin_man)), sec_id_store_(sec_id_store) {} size_t dispatch_securities() { @@ -185,7 +195,26 @@ struct PyMarketDataSubscription::PImpl { LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}", os::get_thread_id(), os::get_thread_name()); py_keep_going = true; - py_thread.emplace([&]() { bridge.run_py(&py_keep_going); }); + py_thread.emplace([&]() { + INFO_TRACE_PARAMS( + "run_py", json_formatter(std::map{ + {"name", name}, {"ev", "STARTING_PY_MD_PRO"}})); + utils::Stopwatch sw = utils::Stopwatch::start_new(); + std::future>> + md_pro_fut = py_plugin_man->start_md_provider( + name, config.provider_type(), + {config.module_path().begin(), config.module_path().end()}, + bridge_ptr); + md_pro_fut.wait(); + INFO_TRACE_PARAMS( + "run_py", + json_formatter(std::map{ + {"name", name}, + {"ev", "STARTED_PY_MD_PRO"}, + {"dur", + fmt::format("{:.3f}ms", sw.elapsed().total_millis())}})); + }); cpp_keep_going = true; cpp_thread.emplace([&]() { static const TimeSpan sec_warn_thresh = TimeSpan::from_millis(100); @@ -238,8 +267,11 @@ struct PyMarketDataSubscription::PImpl { private: PyMarketDataProviderConfig config; - py_interop::MdBridgeImpl bridge; + std::string name; + std::shared_ptr bridge_ptr; + py_interop::MdBridgeImpl &bridge; optional py_thread; + std::shared_ptr py_plugin_man; volatile bool py_keep_going = false; optional cpp_thread; volatile bool cpp_keep_going = false; @@ -373,9 +405,11 @@ struct PyMarketDataSubscription::PImpl { }; PyMarketDataSubscription::PyMarketDataSubscription( - PyMarketDataProviderConfig config, IDGenerator *sec_id_store) + PyMarketDataProviderConfig config, + IDGenerator *sec_id_store, + std::shared_ptr py_plugin_man) : me(std::make_unique( - std::move(config), sec_id_store)) {} + std::move(config), sec_id_store, std::move(py_plugin_man))) {} PyMarketDataSubscription::~PyMarketDataSubscription() { me->stop(); } diff --git a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.h b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.h index 1288270a3..a127fbb4d 100644 --- a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.h +++ b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.h @@ -1,5 +1,6 @@ #pragma once +#include "python_interop/plugin_manager.h" #include "spark/md/IMarketDataSource.h" #include @@ -18,7 +19,8 @@ class PyMarketDataSubscription : public IMarketDataSource { public: explicit PyMarketDataSubscription( api::md::PyMarketDataProviderConfig config, - persistence::IDGenerator *sec_id_store); + persistence::IDGenerator *sec_id_store, + std::shared_ptr py_worker); ~PyMarketDataSubscription(); void start(std::shared_ptr listener) override; diff --git a/src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.cpp b/src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.cpp index 42fea7354..e2e4e7818 100644 --- a/src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.cpp +++ b/src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.cpp @@ -3,31 +3,25 @@ #include "python_interop/execution/py_cancel_order_request.h" #include "python_interop/execution/py_create_order_request.h" #include "python_interop/execution/py_position_info.h" -#include "python_interop/python_activator.h" -#include "python_interop/python_interpreter.h" +#include "python_interop/python_object.h" #include "spark/core/TimeSpan.h" +#include "spark/domain_model/Venue.h" #include "spark/io/json_formatter.h" #include "spark/logging/logging.h" #include "spark/logging/trace_log.h" -#include "spark/logging/trace_log_var.h" -#include "spark/orderman/OrderMsg.h" -#include "spark/orderman/log_messages/execution_report_log.h" -#include "spark/orderman/orderman_utils/ExecutionReport.h" +#include "spark/utils/overloaded.h" #include "spark/utils/stopwatch.h" -#include +#include +#include using std::shared_ptr; -namespace py = pybind11; - namespace python_execution { using bts::core::Delegate; using bts::core::TimeSpan; using bts::io::json_formatter; -using bts::orderman::RFQMsg; -using python_interop::PythonActivatorT; using std::string; using std::vector; @@ -60,10 +54,8 @@ using std::vector; OrderMan, bts::Warn, std::string("exe_gw_bridge"), std::string(name), \ params) -ExecutionGatewayBridgeImpl::ExecutionGatewayBridgeImpl( - string name, string target, vector module_paths) - : name(std::move(name)), target(std::move(target)), - module_paths(std::move(module_paths)) {} +ExecutionGatewayBridgeImpl::ExecutionGatewayBridgeImpl(string name) + : name(std::move(name)) {} void ExecutionGatewayBridgeImpl::on_order_msg(PyOrderRequest msg) { INFO_TRACE("on_order_msg"); @@ -106,11 +98,11 @@ bool ExecutionGatewayBridgeImpl::pop_exe_report( void ExecutionGatewayBridgeImpl::on_securities( std::vector> securities) { INFO_TRACE_PARAMS( - "on_securities", - json_formatter( - fmt::format("Received {} securities", this->securities.size()))); + "on_securities", json_formatter(fmt::format( + "Received {} securities", securities.size()))); + std::lock_guard lock{securities_mx}; this->securities = std::move(securities); - this->is_securities_dirty = true; + is_securities_dirty = true; } shared_ptr @@ -119,12 +111,10 @@ ExecutionGatewayBridgeImpl::pop_exe_report() { return res ? *res : nullptr; } -int ExecutionGatewayBridgeImpl::run_py(volatile bool *keep_going) { - INFO_TRACE_PARAMS("run_py", json_formatter("Begin")); - bts_assert(keep_going != nullptr); - PythonActivatorT gw_activator{target, module_paths}; - gw = gw_activator.get_target( - [&](ExecutionGateway &local_gw) { local_gw.start(this); }, name); +int ExecutionGatewayBridgeImpl::run_py( + volatile bool *keep_going, + std::shared_ptr> gw_ptr, + std::function)> py_invoke) { INFO_TRACE_PARAMS("run_py", json_formatter("Waiting for ready...")); auto sw = bts::utils::Stopwatch::start_new(); while (!wait_for_ready(TimeSpan::from_seconds(1)) && *keep_going) { @@ -133,57 +123,100 @@ int ExecutionGatewayBridgeImpl::run_py(volatile bool *keep_going) { "PyExecutionGateway still not ready after {:.2f}s", sw.total_elapsed().total_seconds()))); } + INFO_TRACE_PARAMS( + "run", json_formatter(fmt::format( + "PyExecutionGateway ready after {:.2f}s", + sw.total_elapsed().total_seconds()))); + ExecutionGateway &gw = **gw_ptr; + std::vector> new_securities; while (is_ready.value() && *keep_going) { INFO_TRACE_PARAMS("run_py", json_formatter("Waiting for request...")); - if (is_securities_dirty) { + bool was_securities_dirty = false; + { + std::lock_guard lock{securities_mx}; + if (is_securities_dirty) { + was_securities_dirty = true; + new_securities = std::move(securities); + is_securities_dirty = false; + } + } + if (was_securities_dirty) { + INFO_TRACE_PARAMS( + "run_py", json_formatter(std::map{ + {"name", name}, {"ev", "SENDING_SECS_TO_PY"}})); + std::promise prom; + sw.restart(); + py_invoke([&]() { + try { + gw.on_securities(new_securities); + } catch (const std::exception &ex) { + bts_assert_msgf( + false, "py exe gw threw on secs. {}", + std::string(ex.what())); + } catch (...) { + bts_assert_msg(false, "py exe gw threw on secs. UNKNOWN"); + } + prom.set_value(); + }); + prom.get_future().wait(); INFO_TRACE_PARAMS( "run_py", - json_formatter(fmt::format( - "Sending {} securities to py gw", securities.size()))); - python_interop::PythonInterpreter::exec( - [&]() { gw->on_securities(securities); }); - is_securities_dirty = false; + json_formatter(std::map{ + {"name", name}, + {"ev", "SENT_SECS_TO_PY"}, + {"dur", + fmt::format("{:.3f}ms", sw.elapsed().total_millis())}})); } auto req = outgoing_reqs.wait_for_until_pop( bts::core::TimeSpan::from_seconds(1)); if (!req) { continue; } - INFO_TRACE_PARAMS( - "run_py", json_formatter("Forwarding request to py gateway")); enum ReqType { CREATE = 0, CANCEL_REPLACE = 1, CANCEL = 2 }; static const std::string req_type_names[]{"CREATE", "CXL_REP", "CXL"}; ReqType req_type; - sw.restart(); - python_interop::PythonInterpreter::exec([&]() { - std::visit( - bts::overloaded{ - [&](std::shared_ptr req) { - req_type = ReqType::CREATE; - gw->create_order(std::move(req)); - }, - [&](std::shared_ptr req) { - req_type = ReqType::CANCEL_REPLACE; - gw->cancel_replace_order(std::move(req)); - }, - [&](std::shared_ptr req) { - req_type = ReqType::CANCEL; - gw->cancel_order(std::move(req)); - }, - [&](const RFQMsg &) { - bts_assert_msg(false, "RFQ not implemented"); - }, - }, - *req); - }); - sw.stop(); - INFO_TRACE_PARAMS( - "run_py", - fmt::format( - R"({{ "typ": "{}", "hndl_dur": "{:.3f}ms" }})", - req_type_names[req_type], sw.total_elapsed().total_millis())); + { + INFO_TRACE_PARAMS( + "run_py", json_formatter(std::map{ + {"name", name}, {"ev", "SENDING_REQ_TO_PY"}})); + std::promise prom; + sw.restart(); + py_invoke([&, req = std::move(req)]() { + try { + std::visit( + bts::overloaded{ + [&](std::shared_ptr req) { + req_type = ReqType::CREATE; + gw.create_order(std::move(req)); + }, + [&](std::shared_ptr req) { + req_type = ReqType::CANCEL_REPLACE; + gw.cancel_replace_order(std::move(req)); + }, + [&](std::shared_ptr req) { + req_type = ReqType::CANCEL; + gw.cancel_order(std::move(req)); + }}, + *req); + } catch (const std::exception &ex) { + bts_assert_msgf( + false, "py exe gw threw on req. {}", std::string(ex.what())); + } catch (...) { + bts_assert_msg(false, "py exe gw threw on req. UNKNOWN"); + } + prom.set_value(); + }); + prom.get_future().wait(); + INFO_TRACE_PARAMS( + "run_py", + json_formatter(std::map{ + {"name", name}, + {"ev", "SENT_REQ_TO_PY"}, + {"typ", req_type_names[req_type]}, + {"dur", + fmt::format("{:.3f}ms", sw.elapsed().total_millis())}})); + } } - python_interop::PythonInterpreter::exec([&]() { gw->stop(); }); return 0; } diff --git a/src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.h b/src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.h index 6b334c966..c455ccd07 100644 --- a/src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.h +++ b/src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.h @@ -6,6 +6,7 @@ #include "python_interop/execution/py_order_request.h" #include "python_interop/execution/py_position_info.h" #include "python_interop/py_security.h" +#include "python_interop/python_object.h" #include "spark/collections/concurrent_queue.h" #include "spark/utils/Delegate.h" #include "spark/utils/concurrent_value_store.h" @@ -14,10 +15,7 @@ namespace python_execution { class ExecutionGatewayBridgeImpl : public ExecutionGatewayBridge { public: - ExecutionGatewayBridgeImpl( - std::string name, - std::string target, - std::vector module_paths); + ExecutionGatewayBridgeImpl(std::string name); ~ExecutionGatewayBridgeImpl() override; // Python interface @@ -35,21 +33,22 @@ class ExecutionGatewayBridgeImpl : public ExecutionGatewayBridge { bool wait_for_ready(bts::core::TimeSpan timeout); bool pop_exe_report(std::shared_ptr &result); std::shared_ptr pop_exe_report(); - int run_py(volatile bool *keep_going); + int run_py( + volatile bool *keep_going, + std::shared_ptr> gw, + std::function)> py_invoke); private: std::string name; - std::string target; - std::vector module_paths; bts::utils::ConcurrentValueStore is_ready; bts::collections::ConcurrentQueue> incoming_exe_reports; bts::collections::ConcurrentQueue> incoming_pos_infos; bts::collections::ConcurrentQueue outgoing_reqs; + std::mutex securities_mx; std::vector> securities; - volatile bool is_securities_dirty = false; - ExecutionGateway *gw; + bool is_securities_dirty = false; }; } // namespace python_execution diff --git a/src/lib_python_interop/python_interop/md/md_bridge_impl.cpp b/src/lib_python_interop/python_interop/md/md_bridge_impl.cpp index c157dcec8..95a451a1a 100644 --- a/src/lib_python_interop/python_interop/md/md_bridge_impl.cpp +++ b/src/lib_python_interop/python_interop/md/md_bridge_impl.cpp @@ -1,46 +1,18 @@ #include "python_interop/md/md_bridge_impl.h" -#include "python_interop/md/md_provider.h" -#include "python_interop/python_activator.h" #include "spark/io/json_formatter.h" #include "spark/logging/logging.h" #include "spark/logging/trace_log.h" #include "spark/utils/Delegate.h" -#include "spark/utils/stopwatch.h" - -#include -#include using std::shared_ptr; -namespace py = pybind11; - namespace bts::md::py_interop { using bts::core::Delegate; using bts::io::json_formatter; -using bts::utils::Stopwatch; -using python_interop::PythonActivatorT; using std::string; using std::vector; -#define ASSERT_FN(test, fn, ...) \ - bts_assert_msgf(test, "md_bridge::" fn ". " __VA_ARGS__) -#define LOG_FN(level, fn, ...) LOG(md, level, "md_bridge::" fn ". " __VA_ARGS__) -#define DIAG_FN(fn, ...) LOG_FN(bts::Diag, fn, __VA_ARGS__) -#define INFO_FN(fn, ...) LOG_FN(bts::Info, fn, __VA_ARGS__) -#define WARN_FN(fn, ...) LOG_FN(bts::Warn, fn, __VA_ARGS__) -#define ERROR_FN(fn, ...) LOG_FN(bts::Error, fn, __VA_ARGS__) - -#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__) -#define INFO(...) _LOG(bts::Info, __VA_ARGS__) -#define ERROR(...) _LOG(bts::Error, __VA_ARGS__) - -#define _LOG_TRACE(level, name) \ - LOG_TRACE(md, level, std::string("md_bridge"), std::string(name)) -#define INFO_TRACE(name) _LOG_TRACE(bts::Info, name) -#define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name) -#define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name) - #define DIAG_TRACE_PARAMS(name, params) \ LOG_TRACE_PARAMS( \ md, bts::Diag, std::string("md_bridge"), std::string(name), params) @@ -49,13 +21,6 @@ using std::vector; LOG_TRACE_PARAMS( \ md, bts::Info, std::string("md_bridge"), std::string(name), params) -#define WARN_TRACE_PARAMS(name, params) \ - LOG_TRACE_PARAMS( \ - md, bts::Warn, std::string("md_bridge"), std::string(name), params) - -MdBridgeImpl::MdBridgeImpl(string target, vector module_paths) - : target(std::move(target)), module_paths(std::move(module_paths)) {} - void MdBridgeImpl::on_securities( const std::vector> &securities) { INFO_TRACE_PARAMS("on_securities", securities.size()); @@ -63,7 +28,7 @@ void MdBridgeImpl::on_securities( } void MdBridgeImpl::on_books(const std::vector> &books) { - DIAG_TRACE_PARAMS("on_books", json_formatter(books.size())); + INFO_TRACE_PARAMS("on_books", json_formatter(books.size())); incoming_books.push_many(books.begin(), books.end()); } @@ -103,26 +68,6 @@ bool MdBridgeImpl::pop_trade(shared_ptr &result) { return false; } -int MdBridgeImpl::run_py(volatile bool *keep_going) { - bts_assert(keep_going != nullptr); - PythonActivatorT provider_activator{target, module_paths}; - provider = provider_activator.get_target([&](MdProvider &pro) { - // This will start another thread in python, and call back into c++ from there. - auto sw = Stopwatch::start_new(); - pro.start(this); - INFO_TRACE_PARAMS( - "run_py", - json_formatter(fmt::format( - "MdProvider.start took {:.2f}ms", sw.elapsed().total_millis()))); - }); - // Release gil to let python run - py::gil_scoped_release gil_rel; - while (*keep_going) { - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - return 0; -} - MdBridgeImpl::~MdBridgeImpl() = default; } // namespace bts::md::py_interop diff --git a/src/lib_python_interop/python_interop/md/md_bridge_impl.h b/src/lib_python_interop/python_interop/md/md_bridge_impl.h index 7d5a5027f..d0823fa84 100644 --- a/src/lib_python_interop/python_interop/md/md_bridge_impl.h +++ b/src/lib_python_interop/python_interop/md/md_bridge_impl.h @@ -1,15 +1,13 @@ #pragma once #include "python_interop/md/md_bridge.h" -#include "python_interop/md/md_provider.h" #include "spark/collections/concurrent_queue.h" namespace bts::md::py_interop { class MdBridgeImpl : public MdBridge { public: - MdBridgeImpl(std::string target, std::vector module_paths); - ~MdBridgeImpl(); + ~MdBridgeImpl() override; // Python interface void on_securities( @@ -21,16 +19,12 @@ class MdBridgeImpl : public MdBridge { bool pop_security(std::shared_ptr &result); bool pop_book(std::shared_ptr &result); bool pop_trade(std::shared_ptr &result); - int run_py(volatile bool *keep_going); private: - std::string target; - std::vector module_paths; collections::ConcurrentQueue> incoming_securities; collections::ConcurrentQueue> incoming_books; collections::ConcurrentQueue> incoming_trades; - MdProvider *provider; }; } // namespace bts::md::py_interop diff --git a/src/lib_python_interop/python_interop/plugin_manager.cpp b/src/lib_python_interop/python_interop/plugin_manager.cpp new file mode 100644 index 000000000..d084221da --- /dev/null +++ b/src/lib_python_interop/python_interop/plugin_manager.cpp @@ -0,0 +1,167 @@ +#include "plugin_manager.h" +#include "python_interop/execution/execution_gateway.h" +#include "python_interop/execution/execution_gateway_bridge.h" +#include "python_interop/md/md_bridge.h" +#include "python_interop/md/md_provider.h" +#include "python_interop/plugin_manager_logs.h" +#include "python_interop/python_object.h" +#include "spark/utils/stopwatch.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; + +namespace python_interop { + +using bts::md::py_interop::MdBridge; +using bts::utils::Stopwatch; +using python_execution::ExecutionGateway; +using python_execution::ExecutionGatewayBridge; +using std::shared_ptr; +using std::string; +using std::vector; + +PluginManager::~PluginManager() { stop_work_thread(); } + +PythonWorkRequest::~PythonWorkRequest() = default; + +void PluginManager::start_work_thread() { + mx.lock(); + if (work_thread) { + mx.unlock(); + return; + } + keep_going = true; + mx.unlock(); + work_thread.emplace([&]() { + pybind11::scoped_interpreter interpreter{false}; + try { + pybind11::gil_scoped_release gil_rel; + Stopwatch sw; + int n = 0; + while (keep_going) { + auto work_req = work_queue.wait_until_pop(); + if (!work_req) { + continue; + } + LOG_INVOKING(n); + sw.restart(); + (**work_req).invoke(); + LOG_INVOKED(n, sw.elapsed()); + n++; + } + } catch (const std::exception &ex) { + bts_assert_msgf( + false, "PythonWorker work_thread threw. {}", + std::string(ex.what())); + } + }); +} + +void PluginManager::stop_work_thread() { + mx.lock(); + if (work_thread && work_thread->joinable()) { + keep_going = false; + mx.unlock(); + work_thread->join(); + work_thread.reset(); + } else { + mx.unlock(); + } +} + +struct PythonWorkRequestDelegate : PythonWorkRequest { + std::function callback; + + PythonWorkRequestDelegate(std::function callback) + : callback(callback) {} + + void invoke() override { callback(); } +}; + +void PluginManager::invoke(std::function callback) { + work_queue.push(std::make_shared(callback)); +} + +std::future>> +PluginManager::start_md_provider( + string provider_name, + string target, + vector module_paths, + shared_ptr bridge) { + auto prom = std::make_shared>>>(); + std::function callback{ + [&, provider_name = std::move(provider_name), bridge = std::move(bridge), + prom = prom](py::object obj) { + auto [md_pro_it, md_pro_is_new] = md_providers.try_emplace( + provider_name, + std::make_shared>( + std::move(obj))); + bts_assert(md_pro_is_new); + LOG_STARTING("md_provider", provider_name); + Stopwatch sw = Stopwatch::start_new(); + shared_ptr> md_pro = + md_pro_it->second; + (*md_pro)->start(bridge.get()); + LOG_STARTED("md_provider", provider_name, sw.elapsed()); + prom->set_value(md_pro); + // Don't let python handle the signals (otherwise it'll hang on ctrl+c) + std::signal(SIGINT, SIG_DFL); + }}; + invoke_on_obj(std::move(target), std::move(module_paths), callback); + return prom->get_future(); +} + +void PluginManager::stop_md_provider(const std::string &provider_name) { + auto it = md_providers.find(provider_name); + if (it != md_providers.end()) { + (*it->second)->stop(); + md_providers.erase(it); + } +} + +std::future>> +PluginManager::start_execution_gateway( + string gateway_name, + string target, + vector module_paths, + shared_ptr bridge) { + auto prom = std::make_shared< + std::promise>>>(); + std::function callback{[&, gateway_name = gateway_name, + bridge = std::move(bridge), + prom = prom](py::object obj) { + auto [exe_gw_it, exe_gw_is_new] = execution_gateways.try_emplace( + gateway_name, + std::make_shared>(std::move(obj))); + bts_assert(exe_gw_is_new); + LOG_STARTING("execution_gateway", gateway_name); + Stopwatch sw = Stopwatch::start_new(); + shared_ptr> exe_gw = exe_gw_it->second; + (*exe_gw)->start(bridge.get()); + LOG_STARTED("execution_gateway", gateway_name, sw.elapsed()); + prom->set_value(exe_gw); + // Don't let python handle the signals (otherwise it'll hang on ctrl+c) + std::signal(SIGINT, SIG_DFL); + }}; + invoke_on_obj(std::move(target), std::move(module_paths), callback); + return prom->get_future(); +} + +void PluginManager::stop_execution_gateway(const std::string &provider_name) { + auto it = execution_gateways.find(provider_name); + if (it != execution_gateways.end()) { + (*it->second)->stop(); + execution_gateways.erase(it); + } +} + +} // namespace python_interop diff --git a/src/lib_python_interop/python_interop/plugin_manager.h b/src/lib_python_interop/python_interop/plugin_manager.h new file mode 100644 index 000000000..86de32b16 --- /dev/null +++ b/src/lib_python_interop/python_interop/plugin_manager.h @@ -0,0 +1,85 @@ +#pragma once + +#include "python_interop/execution/execution_gateway.h" +#include "python_interop/execution/execution_gateway_bridge.h" +#include "python_interop/md/md_bridge.h" +#include "python_interop/md/md_provider.h" +#include "python_interop/python_object.h" +#include "python_interop/python_work_request.h" +#include "spark/collections/concurrent_queue.h" +#include "spark/collections/concurrent_unordered_map.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace python_interop { + +class PluginManager { + volatile bool keep_going; + std::mutex mx; + std::optional work_thread; + bts::collections::ConcurrentQueue> + work_queue; + bts::collections::ConcurrentUnorderedMap< + std::string, + std::shared_ptr>> + md_providers; + bts::collections::ConcurrentUnorderedMap< + std::string, + std::shared_ptr>> + execution_gateways; + + public: + ~PluginManager(); + + void start_work_thread(); + void stop_work_thread(); + + std::future>> + start_md_provider( + std::string provider_name, + std::string target, + std::vector module_paths, + std::shared_ptr bridge); + void stop_md_provider(const std::string &provider_name); + + std::future< + std::shared_ptr>> + start_execution_gateway( + std::string gateway_name, + std::string target, + std::vector module_paths, + std::shared_ptr bridge); + void stop_execution_gateway(const std::string &gateway_name); + + void invoke(std::function callback); + + template + void invoke_on_obj( + std::string target, + std::vector module_path, + std::function callback, + Args &&...args); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +template +inline void PluginManager::invoke_on_obj( + std::string target, + std::vector module_path, + std::function callback, + Args &&...args) { + work_queue.push(std::make_shared>( + std::move(target), std::move(module_path), callback, + std::forward(args)...)); +} + +} // namespace python_interop diff --git a/src/lib_python_interop/python_interop/plugin_manager_logs.h b/src/lib_python_interop/python_interop/plugin_manager_logs.h new file mode 100644 index 000000000..fbdc843ae --- /dev/null +++ b/src/lib_python_interop/python_interop/plugin_manager_logs.h @@ -0,0 +1,115 @@ +#pragma once + +#include "spark/core/FMT_IMPL.h" +#include "spark/core/TimeSpan.h" +#include "spark/logging/trace_log.h" +#include "spark/os/os.h" +#include "spark/utils/thread_id.h" + +#include +#include +#include +#include + +namespace python_interop::logs { + +struct StartingLog { + std::string instance_name; + std::string thread_name; + std::int64_t thread_id; + + StartingLog(std::string instance_name) + : instance_name(std::move(instance_name)), + thread_name(bts::os::get_thread_name()), + thread_id(BTS_THIS_THREAD_ID) {} +}; + +struct StartedLog { + std::string instance_name; + std::string thread_name; + std::int64_t thread_id; + bts::core::TimeSpan duration; + + StartedLog(std::string instance_name, bts::core::TimeSpan duration) + : instance_name(std::move(instance_name)), + thread_name(bts::os::get_thread_name()), thread_id(BTS_THIS_THREAD_ID), + duration(duration) {} +}; + +struct InvokingLog { + int count; + std::string thread_name; + std::int64_t thread_id; + + InvokingLog(int count) + : count(count), thread_name(bts::os::get_thread_name()), + thread_id(BTS_THIS_THREAD_ID) {} +}; + +struct InvokedLog { + int count; + bts::core::TimeSpan duration; + std::string thread_name; + std::int64_t thread_id; + + InvokedLog(int count, bts::core::TimeSpan dequeue_duration) + : count(count), duration(dequeue_duration), + thread_name(bts::os::get_thread_name()), + thread_id(BTS_THIS_THREAD_ID) {} +}; + +} // namespace python_interop::logs + +FMT_IMPL( + python_interop::logs::StartingLog, + l, + R"({{ "name": "{}", "ev": "STARTING", "tn": "{}", "tid": "{}" }})", + l.instance_name, + l.thread_name, + l.thread_id); + +FMT_IMPL( + python_interop::logs::StartedLog, + l, + R"({{ "name": "{}", "ev": "STARTED", "dur": "{:.3f}ms", "tn": "{}", "tid": "{}" }})", + l.instance_name, + l.duration.total_millis(), + l.thread_name, + l.thread_id); + +FMT_IMPL( + python_interop::logs::InvokingLog, + l, + R"({{ "ev": "INVOKING", "count": {}, "tn": "{}", "tid": "{}" }})", + l.count, + l.thread_name, + l.thread_id); + +FMT_IMPL( + python_interop::logs::InvokedLog, + l, + R"({{ "ev": "INVOKED", "count": {}, "dur": "{:.3f}ms", "tn": "{}", "tid": "{}" }})", + l.count, + l.duration.total_millis(), + l.thread_name, + l.thread_id); + +#define LOG_STARTING(type_name, instance_name) \ + LOG_TRACE_PARAMS( \ + Gen, bts::Info, string("py_plugin"), string("start_" type_name), \ + python_interop::logs::StartingLog{instance_name}) + +#define LOG_STARTED(type_name, instance_name, dur) \ + LOG_TRACE_PARAMS( \ + Gen, bts::Info, string("py_plugin"), string("start_" type_name), \ + python_interop::logs::StartedLog(instance_name, dur)) + +#define LOG_INVOKING(count) \ + LOG_TRACE_PARAMS( \ + Gen, bts::Info, string("py_plugin"), string("work"), \ + python_interop::logs::InvokingLog(count)) + +#define LOG_INVOKED(count, duration) \ + LOG_TRACE_PARAMS( \ + Gen, bts::Info, string("py_plugin"), string("work"), \ + python_interop::logs::InvokedLog(count, duration)) diff --git a/src/lib_python_interop/python_interop/python_activator.h b/src/lib_python_interop/python_interop/python_activator.h deleted file mode 100644 index 66688b5c0..000000000 --- a/src/lib_python_interop/python_interop/python_activator.h +++ /dev/null @@ -1,93 +0,0 @@ -#pragma once - -#include "python_interop/python_activator_impl.h" - -#include -#include -#include -#include - -namespace python_interop { - -class PythonActivator { - public: - PythonActivator(std::string target, std::vector module_paths) - : me_(new PythonActivator_Impl( - std::move(target), std::move(module_paths))) {} - - PythonActivator() { - auto *me = static_cast(this->me_); - delete me; - } - - pybind11::object &get_target_obj() { - auto *me = static_cast(this->me_); - return me->get_target_obj(); - } - - pybind11::object &get_target_obj( - std::function init) { - auto *me = static_cast(this->me_); - return me->get_target_obj(init); - } - - template - pybind11::object &get_target_obj( - std::function init, Args &&...args) { - auto *me = static_cast(this->me_); - return me->get_target_obj(init, std::forward(args)...); - } - - void reset() { - auto *me = static_cast(this->me_); - me->reset(); - } - - private: - void *me_; -}; - -template -class PythonActivatorT { - public: - PythonActivatorT(std::string target, std::vector module_paths) - : activator_(std::move(target), std::move(module_paths)) {} - - pybind11::object &get_target_obj() { return activator_.get_target_obj(); } - - Target *get_target() { - if (target_ == nullptr) { - target_ = activator_.get_target_obj().cast(); - } - return target_; - } - - Target *get_target(std::function init) { - if (target_ == nullptr) { - activator_.get_target_obj([&](pybind11::object &obj) { - target_ = obj.cast(); - init(*target_); - }); - } - return target_; - } - - template - Target *get_target(std::function init, Args &&...args) { - if (target_ == nullptr) { - activator_.get_target_obj( - [&](pybind11::object &obj) { - target_ = obj.cast(); - init(*target_); - }, - std::forward(args)...); - } - return target_; - } - - private: - PythonActivator activator_; - Target *target_ = nullptr; -}; - -} // namespace python_interop diff --git a/src/lib_python_interop/python_interop/python_activator_impl.h b/src/lib_python_interop/python_interop/python_activator_impl.h deleted file mode 100644 index 87d28863d..000000000 --- a/src/lib_python_interop/python_interop/python_activator_impl.h +++ /dev/null @@ -1,118 +0,0 @@ -#pragma once - -#include "python_interop/python_interpreter.h" -#include "spark/core/TimeStamp.h" -#include "spark/io/json_formatter.h" -#include "spark/logging/logging.h" - -#include -#include -#include -#include - -namespace python_interop { - -struct PythonActivator_Impl { - std::string target; - std::vector module_paths; - std::optional target_obj; - - PythonActivator_Impl( - std::string target, std::vector module_paths) - : target(std::move(target)), module_paths(std::move(module_paths)) {} - - ~PythonActivator_Impl() { reset(); } - - pybind11::object &get_target_obj() { - return get_target_obj([](pybind11::object &) {}); - } - - template - pybind11::object &get_target_obj( - std::function init, Args &&...args) { - if (!target_obj) { - // Prevent python from overwriting our signal handler (so ctrl+c - // actually terminates the process). - auto orig_signal_handler = std::signal(SIGINT, SIG_DFL); - target_obj = create_target_with_args( - init, std::forward(args)...); - std::signal(SIGINT, orig_signal_handler); - } - return *target_obj; - } - -// Undef after use (this is a header) -#define LOG_EVENT(ev) \ - LOG(OrderMan, bts::Info, \ - R"([py_activator.json]{{ "tgt": "{}", "path": {}, "ev": )" ev \ - R"(, "ts": {} }})", \ - target, bts::io::json_formatter(module_paths), \ - bts::core::TimeStamp::now_ns()) -#define LOG_ERROR(err) \ - LOG(OrderMan, bts::Info, \ - R"([py_activator.json]{{ "tgt": "{}", "path": {}, "ev": "ERROR", "err": "{}", "ts": {} }})", \ - target, bts::io::json_formatter(module_paths), err, \ - bts::core::TimeStamp::now_ns()) -#define LOG_CREATE LOG_EVENT("CREATE") -#define LOG_RESET LOG_EVENT("RESET") - - void reset() { - LOG_RESET; - target_obj.reset(); - } - - private: - template - pybind11::object create_target_with_args( - std::function init, Args &&...args) { - LOG_CREATE; - return PythonInterpreter::exec_t( - [this, &init, - args_tuple = - std::make_tuple(std::forward(args)...)]() mutable { - std::string target_module_name; - std::string target_type_name; - PythonInterpreter::parse_target( - target, &target_module_name, &target_type_name); - pybind11::object obj; - try { - // Prepare module path - pybind11::module sys = pybind11::module::import("sys"); - for (const auto &module_path : module_paths) { - sys.attr("path").attr("append")(module_path.c_str()); - } - // Import target module - pybind11::module target_module = - pybind11::module::import(target_module_name.c_str()); - // Instantiate target using tuple expansion - obj = std::apply( - [&](auto &&...unpacked_args) { - return target_module.attr(target_type_name.c_str())( - std::forward( - unpacked_args)...); - }, - std::move(args_tuple)); - init(obj); - } catch (const std::exception &ex) { - LOG_ERROR(std::string{ex.what()}); - } - return obj; - }); - } - - pybind11::object create_target() { - return create_target_with_args([](pybind11::object &) {}); - } - - pybind11::object create_target( - std::function init) { - return create_target_with_args(init); - } - -#undef LOG_EVENT -#undef LOG_ERROR -#undef LOG_CREATE -#undef LOG_RESET -}; - -} // namespace python_interop diff --git a/src/lib_python_interop/python_interop/python_interpreter.cpp b/src/lib_python_interop/python_interop/python_interpreter.cpp deleted file mode 100644 index 5c9b450eb..000000000 --- a/src/lib_python_interop/python_interop/python_interpreter.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "python_interpreter.h" -#include "spark/logging/logging.h" -#include "spark/utils/string_utils.h" - -#include -#include -#include -#include - -namespace python_interop { - -using bts::system::string_utils::snake_to_pascal_case; -using bts::system::string_utils::split; - -std::mutex PythonInterpreter::mx; -std::optional PythonInterpreter::interpreter; - -void PythonInterpreter::init_interpreter() { - if (interpreter.has_value()) { - return; - } - static std::mutex local_mx; - std::lock_guard lock{local_mx}; - if (interpreter.has_value()) { - return; - } - LOG(Gen, bts::Info, "Creating global py interpreter"); - // Don't let python handle the signals (otherwise it'll hang on ctrl+c) - auto orig_signal_handler = std::signal(SIGINT, SIG_DFL); - interpreter.emplace(false); - std::signal(SIGINT, orig_signal_handler); -} - -bool PythonInterpreter::parse_target( - const std::string &target, - std::string *module_name, - std::string *type_name) { - auto parts = split(target, "."); - if (parts.empty()) { - return false; - } - if (parts.size() == 1) { - snake_to_pascal_case(parts.emplace_back(parts[0])); - } - *module_name = parts[0]; - *type_name = parts[1]; - return !module_name->empty() && !type_name->empty(); -} - -} // namespace python_interop diff --git a/src/lib_python_interop/python_interop/python_interpreter.h b/src/lib_python_interop/python_interop/python_interpreter.h deleted file mode 100644 index 71a3c744a..000000000 --- a/src/lib_python_interop/python_interop/python_interpreter.h +++ /dev/null @@ -1,125 +0,0 @@ -#pragma once - -#include "spark/utils/bts_assert.h" -#include "spark/utils/string_utils.h" - -#include -#include -#include -#include -#include - -namespace python_interop { - -class PythonInterpreter { - static std::mutex mx; - static std::optional interpreter; - static void init_interpreter(); - - public: - /** - * Parses module and type name from a \p target arg. The following formats are supported: - * 1) . - * 2) - * In this case, the type name defaults to the PascalCase equivalent of the module name. - * For example, the module "foo_bar" would have type name "FooBar". - */ - static bool parse_target( - const std::string &target, - std::string *module_name, - std::string *type_name); - - static void exec(std::function fn) { - init_interpreter(); - std::lock_guard lock{mx}; - fn(); - } - - template - static T exec_t(std::function fn) { - init_interpreter(); - std::lock_guard lock{mx}; - pybind11::gil_scoped_acquire gil; - T rv = fn(); - pybind11::gil_scoped_release gil_rel; - return rv; - } - - template - static pybind11::object create_object( - const std::vector &module_paths, - const std::string &target_name, - Args &&...args) { - std::string target_module_name; - std::string target_type_name; - bts_assert( - parse_target(target_name, &target_module_name, &target_type_name)); - return create_object( - module_paths, target_module_name, target_type_name, - std::forward(args)...); - } - - template - static pybind11::object create_object( - const std::vector &module_paths, - const std::string &target_type_name, - const std::string &target_module_name, - Args &&...args) { - namespace py = pybind11; - return exec_t([&]() { - // Prepare module path - py::module sys = py::module::import("sys"); - for (const auto &module_path : module_paths) { - sys.attr("path").attr("append")(module_path.c_str()); - } - // Import target module - py::module target_module = - py::module::import(target_module_name.c_str()); - // Instantiate target - return target_module.attr(target_type_name.c_str())( - std::forward(args)...); - }); - } - - template - static pybind11::object create_object_with_init( - const std::vector &module_paths, - const std::string &target_name, - std::function init_fn, - Args &&...args) { - std::string target_module_name; - std::string target_type_name; - bts_assert( - parse_target(target_name, &target_module_name, &target_type_name)); - return create_object_with_init( - module_paths, target_module_name, target_type_name, init_fn, - std::forward(args)...); - } - - template - static pybind11::object create_object_with_init( - const std::vector &module_paths, - const std::string &target_type_name, - const std::string &target_module_name, - std::function init_fn, - Args &&...args) { - namespace py = pybind11; - return exec_t([&]() { - // Prepare module path - py::module sys = py::module::import("sys"); - for (const auto &module_path : module_paths) { - sys.attr("path").attr("append")(module_path.c_str()); - } - // Import target module - py::module target_module = - py::module::import(target_module_name.c_str()); - // Instantiate target - py::object obj = target_module.attr(target_type_name.c_str())( - std::forward(args)...); - init_fn(obj); - return obj; - }); - } -}; - -} // namespace python_interop diff --git a/src/lib_python_interop/python_interop/python_object.h b/src/lib_python_interop/python_interop/python_object.h new file mode 100644 index 000000000..5cb799a7c --- /dev/null +++ b/src/lib_python_interop/python_interop/python_object.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace python_interop { + +template +class PythonObjectT { + pybind11::object obj_; + T *tp_ = nullptr; + + public: + PythonObjectT(pybind11::object obj) + : obj_(std::move(obj)), tp_(obj_.cast()) {} + + T &operator*() { return *tp_; } + T *operator->() { return tp_; } +}; + +} // namespace python_interop diff --git a/src/lib_python_interop/python_interop/python_work_request.h b/src/lib_python_interop/python_interop/python_work_request.h new file mode 100644 index 000000000..21821dff3 --- /dev/null +++ b/src/lib_python_interop/python_interop/python_work_request.h @@ -0,0 +1,95 @@ +#pragma once + +#include "spark/core/TimeStamp.h" +#include "spark/utils/bts_assert.h" +#include "spark/utils/string_utils.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace python_interop { + +struct PythonWorkRequest { + virtual ~PythonWorkRequest(); + + static bool parse_target( + const std::string &target, + std::string *module_name, + std::string *type_name); + + virtual void invoke() = 0; +}; + +template +struct PythonWorkRequestT : PythonWorkRequest { + std::string target; + std::vector module_paths; + std::function callback; + std::tuple args; + + PythonWorkRequestT( + std::string target, + std::vector module_paths, + std::function callback, + Args... args) + : target(std::move(target)), module_paths(std::move(module_paths)), + callback(callback), args(std::forward(args)...) {} + + ~PythonWorkRequestT() override = default; + + void invoke() override; +}; + +inline bool PythonWorkRequest::parse_target( + const std::string &target, + std::string *module_name, + std::string *type_name) { + auto parts = bts::system::string_utils::split(target, "."); + if (parts.empty()) { + return false; + } + if (parts.size() == 1) { + bts::system::string_utils::snake_to_pascal_case( + parts.emplace_back(parts[0])); + } + *module_name = parts[0]; + *type_name = parts[1]; + return !module_name->empty() && !type_name->empty(); +} + +template +inline void PythonWorkRequestT::invoke() { + std::string target_module_name; + std::string target_type_name; + bts_assert(parse_target(target, &target_module_name, &target_type_name)); + LOG(Gen, bts::Info, + R"([py_worker.json]{{ "ev": "INVOKE", "mod": "{}", "typ": "{}", "args_t": "{}", "ts": {} }})", + target_module_name, target_type_name, + std::string(typeid(decltype(this->args)).name()), + bts::core::TimeStamp::now_ns()); + try { + pybind11::gil_scoped_acquire gil_acquire; + pybind11::module sys = pybind11::module::import("sys"); + for (const auto &module_path : module_paths) { + sys.attr("path").attr("append")(module_path.c_str()); + } + pybind11::module target_module = + pybind11::module::import(target_module_name.c_str()); + std::apply( + [&](auto &&...unpacked_args) { + callback(target_module.attr(target_type_name.c_str())( + std::forward(unpacked_args)...)); + }, + std::move(args)); + } catch (const std::exception &ex) { + bts_assert_msgf( + false, "PythonWorkRequest::invoke threw. {}", std::string(ex.what())); + } +} + +} // namespace python_interop diff --git a/src/lib_python_interop/python_interop_plugins/bybit_execution_gateway.py b/src/lib_python_interop/python_interop_plugins/bybit_execution_gateway.py index 1398b18e2..3d6c4ff73 100644 --- a/src/lib_python_interop/python_interop_plugins/bybit_execution_gateway.py +++ b/src/lib_python_interop/python_interop_plugins/bybit_execution_gateway.py @@ -23,6 +23,7 @@ from python_interop_module import ( PySecurity, PySecurityList, ) +from spark_enums import OrderStatus, ExecType import threading @@ -36,7 +37,8 @@ class OrderData: self.last_filled_quantity: float = 0 self.avg_filled_price: float = 0 self.last_filled_price: float = 0 - self.exec_type = "PENDING_NEW" + self.exec_type: ExecType = ExecType.PENDING_NEW + self.status: OrderStatus = OrderStatus.PENDING_NEW def on_update( self, @@ -63,19 +65,29 @@ class OrderData: self.filled_quantity = filled_quantity self.avg_filled_price = avg_filled_price match status: + case "New": + self.exec_type = ExecType.NEW + self.status = OrderStatus.NEW case "Filled": - self.exec_type = "FILL" - case "Canceled": - self.exec_type = "CANCELED" + self.exec_type = ExecType.FILL + self.status = OrderStatus.FILLED + case "PartiallyFilled": + self.exec_type = ExecType.PARTIAL_FILL + self.status = OrderStatus.PARTIALLY_FILLED + case "Cancelled": + self.exec_type = ExecType.CANCELED + self.status = OrderStatus.CANCELED case "Rejected": - self.exec_type = "REJECTED" + self.exec_type = ExecType.REJECTED + self.status = OrderStatus.REJECTED case _: - self.exec_type = "NEW" if filled_quantity == 0 else "PARTIAL_FILL" + raise ValueError(f"Unexpected order status '{status}'") class BybitExecutionGateway(ExecutionGateway): - def __init__(self, name: str): + def __init__(self): super(BybitExecutionGateway, self).__init__() + name = "bybit_1" self.name = name self.logger = configure_logger(logging.getLogger(name), "OrderMan") self.logger.info("[bybit_gw_ctr.json]%s" % SLog(name=name)) @@ -135,15 +147,6 @@ class BybitExecutionGateway(ExecutionGateway): total_filled_qty = parse_float(d["cumExecQty"]) avg_filled_price = parse_float(d["avgPrice"]) status: str = d["orderStatus"] - # brits can't spell - if status == "Cancelled": - status = "Canceled" - exec_type = ord_data.exec_type - match ord_data.exec_type: - case "PARTIAL_FILL": - exec_type = "PARTIALLY_FILLED" - case _: - pass px = parse_float(d["price"]) qty = parse_float(d["qty"]) ord_data.on_update( @@ -162,8 +165,8 @@ class BybitExecutionGateway(ExecutionGateway): price=px, quantity=qty, time_in_force=d["timeInForce"], - exec_type=exec_type, - order_status=status, + exec_type=ord_data.exec_type.name, + order_status=ord_data.status.name, last_filled_price=ord_data.last_filled_price, last_filled_quantity=ord_data.last_filled_quantity, filled_quantity=total_filled_qty, @@ -198,7 +201,7 @@ class BybitExecutionGateway(ExecutionGateway): params = { "api_key": self.api_key, "api_secret": self.secret_key, - "testnet": True, + "testnet": self.use_testnet, } self.logger.info("[bybit_gw_start_trd_cli.json]%s" % json.dumps(params)) self.trade_cli = WebSocketTrading(**params) @@ -206,6 +209,10 @@ class BybitExecutionGateway(ExecutionGateway): "category": "linear", } self.bridge.on_is_ready(True) + while True: + if not self.stream_cli.is_connected(): + # try reconnecting + pass def start(self, bridge: ExecutionGatewayBridge): def start_wrapper(): diff --git a/src/lib_python_interop/python_interop_plugins/spark_enums.py b/src/lib_python_interop/python_interop_plugins/spark_enums.py new file mode 100644 index 000000000..e40040736 --- /dev/null +++ b/src/lib_python_interop/python_interop_plugins/spark_enums.py @@ -0,0 +1,19 @@ +from enum import Enum + + +class OrderStatus(Enum): + NEW = 1 + PENDING_NEW = 2 + PARTIALLY_FILLED = 3 + FILLED = 4 + CANCELED = 5 + REJECTED = 6 + + +class ExecType(Enum): + NEW = 1 + PENDING_NEW = 2 + PARTIAL_FILL = 3 + FILL = 4 + CANCELED = 5 + REJECTED = 6