From e7ee1e02dd536678438206f71c2b934133a9ea58 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sun, 25 May 2025 18:02:21 +0000 Subject: [PATCH] refactor: Redesign py md so it doesn't lock the global interpreter We can only embed one interpreter in the sparkd process, so all py plugins need to share it. * Move core py interop code to new lib_python_core. We're about to add a new lib for py execution, so it needs to be accesible from both execution and md libs * Fix typo for type hint in py_md_utils.py * Run py md providers on background thread * Rename MdSink -> MdBridge --- CMakeLists.txt | 1 + .../sparkd/md/PyMarketDataSubscription.cpp | 198 +++++++++++++----- src/lib_python_core/CMakeLists.txt | 30 +++ src/lib_python_core/Config.cmake.in | 0 .../python_core/python_activator.h | 93 ++++++++ .../python_core/python_activator_impl.h | 140 +++++++++++++ .../python_core/python_interpreter.cpp | 50 +++++ .../python_core/python_interpreter.h | 125 +++++++++++ src/lib_python_md/CMakeLists.txt | 10 +- .../python_md/{md_sink.cpp => md_bridge.cpp} | 4 +- src/lib_python_md/python_md/md_bridge.h | 23 ++ .../python_md/md_bridge_impl.cpp | 128 +++++++++++ src/lib_python_md/python_md/md_bridge_impl.h | 36 ++++ src/lib_python_md/python_md/md_provider.h | 4 +- src/lib_python_md/python_md/md_sink.h | 20 -- src/lib_python_md/python_md/md_sink_impl.cpp | 105 ---------- src/lib_python_md/python_md/md_sink_impl.h | 36 ---- .../python_md/py_md_executor.cpp | 143 ------------- src/lib_python_md/python_md/py_md_executor.h | 24 --- src/lib_python_md/python_md/py_md_provider.h | 8 +- src/lib_python_md/python_md/py_module.cpp | 39 ++-- .../python_md_providers/bybit_md_provider.py | 26 ++- .../python_md_providers/dummy_md_provider.py | 8 +- .../python_md_providers/gateio_md_provider.py | 21 +- .../python_md_providers/py_md_utils.py | 3 +- 25 files changed, 840 insertions(+), 435 deletions(-) create mode 100644 src/lib_python_core/CMakeLists.txt create mode 100644 src/lib_python_core/Config.cmake.in create mode 100644 src/lib_python_core/python_core/python_activator.h create mode 100644 src/lib_python_core/python_core/python_activator_impl.h create mode 100644 src/lib_python_core/python_core/python_interpreter.cpp create mode 100644 src/lib_python_core/python_core/python_interpreter.h rename src/lib_python_md/python_md/{md_sink.cpp => md_bridge.cpp} (55%) create mode 100644 src/lib_python_md/python_md/md_bridge.h create mode 100644 src/lib_python_md/python_md/md_bridge_impl.cpp create mode 100644 src/lib_python_md/python_md/md_bridge_impl.h delete mode 100644 src/lib_python_md/python_md/md_sink.h delete mode 100644 src/lib_python_md/python_md/md_sink_impl.cpp delete mode 100644 src/lib_python_md/python_md/md_sink_impl.h delete mode 100644 src/lib_python_md/python_md/py_md_executor.cpp delete mode 100644 src/lib_python_md/python_md/py_md_executor.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f73b0d56..e96fce553 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,6 +115,7 @@ else() # subdirs add_subdirectory(src/lib_spark) if (BUILD_SPARKD) + add_subdirectory(src/lib_python_core) add_subdirectory(src/lib_onixs_fix) add_subdirectory(src/lib_binance_execution) add_subdirectory(src/lib_okx_execution) diff --git a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp index 75ac9a529..23c9c3a87 100644 --- a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp +++ b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp @@ -1,14 +1,14 @@ #include "PyMarketDataSubscription.h" -#include "python_md/md_sink.h" +#include "python_md/md_bridge_impl.h" #include "python_md/py_book.h" #include "python_md/py_contract_specification.h" -#include "python_md/py_md_executor.h" #include "python_md/py_security.h" #include "python_md/py_trade.h" #include "spark/api/domain/CallPut.pb.h" #include "spark/api/domain/SecurityType.h" #include "spark/api/domain/Venue.h" #include "spark/api/md/PyMarketDataProviderConfig.pb.h" +#include "spark/core/TimeSpan.h" #include "spark/core/TimeStamp.h" #include "spark/domain/AggressorSide.h" #include "spark/domain/FutureSecurity.h" @@ -19,10 +19,12 @@ #include "spark/domain_model/SecurityType.h" #include "spark/io/json_formatter.h" #include "spark/logging/logging.h" +#include "spark/logging/trace_log.h" #include "spark/md/IMarketDataListener.h" #include "spark/os/os.h" #include "spark/persistence/IDGenerator.h" #include "spark/utils/bts_assert.h" +#include "spark/utils/stopwatch.h" #include #include @@ -33,12 +35,39 @@ namespace bts::md { -#define AGGRESSOR_SIDE(x) bts::domain::AGGRESSOR_SIDE_##x +#define _LOG(level, ...) LOG(md, level, __VA_ARGS__) +#define INFO(...) _LOG(bts::Info, __VA_ARGS__) +#define ERROR(...) _LOG(bts::Error, __VA_ARGS__) + +#define _LOG_TRACE(level, name) \ + LOG_TRACE(md, level, std::string("py_md_sub"), std::string(name)) +#define INFO_TRACE(name) _LOG_TRACE(bts::Info, name) +#define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name) +#define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name) + +#define LOG_DISPATCH_STATS(level, typ, n, dur) \ + _LOG( \ + level, \ + R"("[py_md_dispatch.json]{{ "typ": "{}", "n_recs": {}, "dur": "{:.2f}ms", "ts": {} }})", \ + n, dur.total_millis(), TimeStamp::now_ns()) + +#define LOG_SECURITY(level, py_sec) \ + LOG(md, level, R"([py_md_sec.json]{{ "sec": {}, "ts": {}}})", \ + json_formatter(py_sec), TimeStamp::now_ns()) + +#define LOG_BOOK(level, py_book) \ + LOG(md, level, R"([py_md_book.json]{{ "bk": {}, "ts": {}}})", \ + json_formatter(py_book), TimeStamp::now_ns()) + +#define LOG_TRADE(level, py_trade) \ + LOG(md, level, R"([py_md_trade.json]{{ "trd": {}, "ts": {}}})", \ + json_formatter(py_trade), TimeStamp::now_ns()) using bts::api::domain::CallPut; using bts::api::domain::SecurityType; using bts::api::domain::Venue; using bts::api::md::PyMarketDataProviderConfig; +using bts::core::TimeSpan; using bts::core::TimeStamp; using bts::domain::AggressorSide; using bts::domain::Book; @@ -52,13 +81,14 @@ using bts::domain::TradeRecord; using bts::io::json_formatter; using bts::md::py_interop::PyBook; using bts::md::py_interop::PyBookSide; -using bts::md::py_interop::PyMdExecutor; using bts::md::py_interop::PySecurity; using bts::md::py_interop::PyTrade; using bts::persistence::IDGenerator; using std::optional; using std::shared_ptr; +using std::size_t; using std::string; +using std::thread; using std::vector; using Path = boost::filesystem::path; @@ -69,48 +99,23 @@ struct TradeTuple { Venue venue; }; -struct PyMarketDataSubscription::PImpl : private py_interop::MdSink { +struct PyMarketDataSubscription::PImpl { PImpl(PyMarketDataProviderConfig config, IDGenerator *sec_id_store) - : config(std::move(config)), py_md_exec_( + : config(std::move(config)), bridge( this->config.provider_type(), {this->config.module_path().begin(), - this->config.module_path().end()}, - this), + this->config.module_path().end()}), sec_id_store_(sec_id_store) {} - void start(shared_ptr listener) { - bts_assert(listener != nullptr); - this->listener_ = std::move(listener); - LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}", - os::get_thread_id(), os::get_thread_name()); - py_md_exec_.start(); - } - - void stop() { py_md_exec_.stop(); } - - private: - PyMarketDataProviderConfig config; - PyMdExecutor py_md_exec_; - shared_ptr listener_; - - shared_ptr sec_id_store_; - // The main store for securities. - vector securities_; - vector security_ptrs_; - Book book_; - // Since the python md provider isn't tethered to a single venue, we - // must know the venue for each individual trade. - vector trades_; - vector trade_ptrs_; - - void on_securities(const vector &py_securities) override { - LOG(md, Info, R"([py_md_secs.json]{{ "secs": {}, "ts": {}}})", - json_formatter(py_securities), TimeStamp::now_ns()); + size_t dispatch_securities() { // `securities_` and `security_ptrs_` should already be empty, but no harm in clearing them // again. We keep them as member variables to avoid (re)allocating memory in each callback. securities_.clear(); security_ptrs_.clear(); - for (const auto &py_sec : py_securities) { + std::shared_ptr py_sec_p; + while (bridge.pop_security(py_sec_p)) { + const auto &py_sec = *py_sec_p; + LOG_SECURITY(Info, py_sec); auto sec = to_bts_security_(py_sec); if (!sec) { LOG(md, Error, @@ -123,47 +128,132 @@ struct PyMarketDataSubscription::PImpl : private py_interop::MdSink { } securities_.push_back(std::move(*sec)); } - // Populate vectors of security ptrs per each venue. - for (auto &sec : securities_) { - security_ptrs_.push_back(&sec); + const size_t n = securities_.size(); + if (!securities_.empty()) { + // Populate vectors of security ptrs per each venue. + for (auto &sec : securities_) { + security_ptrs_.push_back(&sec); + } + // Pass securities to the sinks (per venue). + listener_->on_securities(security_ptrs_); + securities_.clear(); + security_ptrs_.clear(); } - // Pass securities to the sinks (per venue). - listener_->on_securities(security_ptrs_); - securities_.clear(); - security_ptrs_.clear(); + return n; } - void on_books(const vector &py_books) override { - LOG(md, Info, R"([py_md_books.json]{{ "bks": {}, "ts": {}}})", - json_formatter(py_books), TimeStamp::now_ns()); - for (auto &py_book : py_books) { + size_t dispatch_books() { + size_t n = 0; + std::shared_ptr py_book_p; + while (bridge.pop_book(py_book_p)) { + const auto &py_book = *py_book_p; + LOG_BOOK(Diag, py_book); bts_assert(book_.count_populated() == 0); to_book_(py_book, book_); listener_->on_book(book_); book_.clear(); + n++; } + return n; } - void on_trades(const vector &py_trades) override { - LOG(md, Info, R"([py_md_trades.json]{{ "trds": {}, "ts": {}}})", - json_formatter(py_trades), TimeStamp::now_ns()); + size_t dispatch_trades() { // `trades_` and `trade_ptrs_` should already be empty, but no harm in clearing them again. // We keep them as member variables to avoid (re)allocating memory in each callback. trades_.clear(); trade_ptrs_.clear(); - for (const auto &py_trade : py_trades) { - LOG(md, Info, "PyMarketDataSubscription received trade {}", - py_trade.to_string()); + std::shared_ptr py_trade_p; + while (bridge.pop_trade(py_trade_p)) { + const auto &py_trade = *py_trade_p; + LOG_TRADE(Diag, py_trade); to_bts_trade_(py_trade, trades_.emplace_back()); } + const size_t n = trades_.size(); for (const auto &t : trades_) { trade_ptrs_.push_back(&t); } listener_->on_trades(trade_ptrs_); trades_.clear(); trade_ptrs_.clear(); + return n; } + void start(shared_ptr listener) { + bts_assert(listener != nullptr); + this->listener_ = std::move(listener); + LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}", + os::get_thread_id(), os::get_thread_name()); + py_keep_going = true; + py_thread.emplace([&]() { bridge.run_py(&py_keep_going); }); + cpp_keep_going = true; + cpp_thread.emplace([&]() { + static const TimeSpan sec_warn_thresh = TimeSpan::from_millis(100); + static const TimeSpan bk_warn_thresh = TimeSpan::from_millis(30); + static const TimeSpan trd_warn_thresh = TimeSpan::from_millis(30); + utils::Stopwatch sw; + TimeSpan dur; + size_t n; + while (cpp_keep_going) { + // Dispatch securities + sw.restart(); + n = dispatch_securities(); + dur = sw.elapsed(); + if (n > 0) { + LOG_DISPATCH_STATS( + dur > sec_warn_thresh ? Warn : Info, "sec", n, dur); + } + // Dispatch books + sw.restart(); + n = dispatch_books(); + dur = sw.elapsed(); + if (n > 0) { + LOG_DISPATCH_STATS( + dur > bk_warn_thresh ? Warn : Diag, "bk", n, dur); + } + // Dispatch trades + sw.restart(); + n = dispatch_trades(); + dur = sw.elapsed(); + if (n > 0) { + LOG_DISPATCH_STATS( + dur > trd_warn_thresh ? Warn : Diag, "trd", n, dur); + } + } + }); + } + + void stop() { + py_keep_going = false; + if (py_thread) { + py_thread->join(); + py_thread.reset(); + } + cpp_keep_going = false; + if (cpp_thread) { + cpp_thread->join(); + cpp_thread.reset(); + } + } + + private: + PyMarketDataProviderConfig config; + py_interop::MdBridgeImpl bridge; + optional py_thread; + volatile bool py_keep_going = false; + optional cpp_thread; + volatile bool cpp_keep_going = false; + shared_ptr listener_; + + shared_ptr sec_id_store_; + // The main store for securities. + vector securities_; + vector security_ptrs_; + Book book_; + // Since the python md provider isn't tethered to a single venue, we + // must know the venue for each individual trade. + vector trades_; + vector trade_ptrs_; + SecurityID get_sec_id_( Venue venue, SecurityType sec_type, const string &sec_name) { auto sec_id = sec_id_store_->get_id(venue, sec_type, sec_name, true); diff --git a/src/lib_python_core/CMakeLists.txt b/src/lib_python_core/CMakeLists.txt new file mode 100644 index 000000000..df6203c15 --- /dev/null +++ b/src/lib_python_core/CMakeLists.txt @@ -0,0 +1,30 @@ +cmake_minimum_required(VERSION 3.23) +project(python_core) +include(${CMAKE_SOURCE_DIR}/cmake/InitLibrary.cmake) +init_library() +target_link_libraries( + ${PROJECT_NAME} + PUBLIC + spark +) + +# Find Python from the virtual environment +find_package(Python3 REQUIRED COMPONENTS Interpreter Development) +find_package(pybind11 REQUIRED) + +target_include_directories( + ${PROJECT_NAME} + PUBLIC + ${Python3_INCLUDE_DIRS} +) +target_link_directories( + ${PROJECT_NAME} + PUBLIC + ${Python3_LIBRARY_DIRS} +) +target_link_libraries( + ${PROJECT_NAME} + PUBLIC + pybind11::embed + ${Python3_LIBRARIES} +) diff --git a/src/lib_python_core/Config.cmake.in b/src/lib_python_core/Config.cmake.in new file mode 100644 index 000000000..e69de29bb diff --git a/src/lib_python_core/python_core/python_activator.h b/src/lib_python_core/python_core/python_activator.h new file mode 100644 index 000000000..e693e162f --- /dev/null +++ b/src/lib_python_core/python_core/python_activator.h @@ -0,0 +1,93 @@ +#pragma once + +#include "python_core/python_activator_impl.h" + +#include +#include +#include +#include + +namespace python_core { + +class PythonActivator { + public: + PythonActivator(std::string target, std::vector module_paths) + : me_(new PythonActivator_Impl( + std::move(target), std::move(module_paths))) {} + + PythonActivator() { + auto *me = static_cast(this->me_); + delete me; + } + + pybind11::object &get_target_obj() { + auto *me = static_cast(this->me_); + return me->get_target_obj(); + } + + pybind11::object &get_target_obj( + std::function init) { + auto *me = static_cast(this->me_); + return me->get_target_obj(init); + } + + template + pybind11::object &get_target_obj( + std::function init, Args &&...args) { + auto *me = static_cast(this->me_); + return me->get_target_obj(init, std::forward(args)...); + } + + void reset() { + auto *me = static_cast(this->me_); + me->reset(); + } + + private: + void *me_; +}; + +template +class PythonActivatorT { + public: + PythonActivatorT(std::string target, std::vector module_paths) + : activator_(std::move(target), std::move(module_paths)) {} + + pybind11::object &get_target_obj() { return activator_.get_target_obj(); } + + Target *get_target() { + if (target_ == nullptr) { + target_ = activator_.get_target_obj().cast(); + } + return target_; + } + + Target *get_target(std::function init) { + if (target_ == nullptr) { + activator_.get_target_obj([&](pybind11::object &obj) { + target_ = obj.cast(); + init(*target_); + }); + } + return target_; + } + + template + Target *get_target(std::function init, Args &&...args) { + if (target_ == nullptr) { + activator_.get_target_obj( + [&](pybind11::object &obj) { + target_ = obj.cast(); + init(*target_); + }, + std::forward(args)...); + } + return target_; + } + + private: + PythonActivator activator_; + Target *target_ = nullptr; +}; + +} // namespace python_core diff --git a/src/lib_python_core/python_core/python_activator_impl.h b/src/lib_python_core/python_core/python_activator_impl.h new file mode 100644 index 000000000..8f4fc750c --- /dev/null +++ b/src/lib_python_core/python_core/python_activator_impl.h @@ -0,0 +1,140 @@ +#pragma once + +#include "python_core/python_interpreter.h" +#include "spark/logging/logging.h" + +#include +#include +#include +#include + +namespace python_core { + +struct PythonActivator_Impl { + std::string target; + std::vector module_paths; + std::optional target_obj; + + PythonActivator_Impl( + std::string target, std::vector module_paths) + : target(std::move(target)), module_paths(std::move(module_paths)) {} + + ~PythonActivator_Impl() { reset(); } + + template + pybind11::object create_target_with_args( + std::function init, Args &&...args) { + LOG(OrderMan, bts::Info, "PythonActivator::create_target. {}", target); + return PythonInterpreter::exec_t( + [this, &init, + args_tuple = + std::make_tuple(std::forward(args)...)]() mutable { + std::string target_module_name; + std::string target_type_name; + PythonInterpreter::parse_target( + target, &target_module_name, &target_type_name); + // Prepare module path + pybind11::module sys = pybind11::module::import("sys"); + for (const auto &module_path : module_paths) { + sys.attr("path").attr("append")(module_path.c_str()); + } + // Import target module + pybind11::module target_module = + pybind11::module::import(target_module_name.c_str()); + // Instantiate target using tuple expansion + pybind11::object obj = std::apply( + [&](auto &&...unpacked_args) { + return target_module.attr(target_type_name.c_str())( + std::forward( + unpacked_args)...); + }, + std::move(args_tuple)); + init(obj); + return obj; + }); + } + + pybind11::object &get_target_obj() { + if (!target_obj) { + // Prevent python from overwriting our signal handler (so ctrl+c + // actually terminates the process). + auto orig_signal_handler = std::signal(SIGINT, SIG_DFL); + target_obj = create_target(); + std::signal(SIGINT, orig_signal_handler); + } + return *target_obj; + } + + pybind11::object &get_target_obj( + std::function init) { + if (!target_obj) { + // Prevent python from overwriting our signal handler (so ctrl+c + // actually terminates the process). + auto orig_signal_handler = std::signal(SIGINT, SIG_DFL); + target_obj = create_target(init); + std::signal(SIGINT, orig_signal_handler); + } + return *target_obj; + } + + template + pybind11::object &get_target_obj( + std::function init, Args &&...args) { + if (!target_obj) { + // Prevent python from overwriting our signal handler (so ctrl+c + // actually terminates the process). + auto orig_signal_handler = std::signal(SIGINT, SIG_DFL); + target_obj = create_target_with_args( + init, std::forward(args)...); + std::signal(SIGINT, orig_signal_handler); + } + return *target_obj; + } + + pybind11::object create_target() { + LOG(OrderMan, bts::Info, "PythonActivator::create_target. {}", target); + return PythonInterpreter::exec_t([&]() { + std::string target_module_name; + std::string target_type_name; + PythonInterpreter::parse_target( + target, &target_module_name, &target_type_name); + // Prepare module path + pybind11::module sys = pybind11::module::import("sys"); + for (const auto &module_path : module_paths) { + sys.attr("path").attr("append")(module_path.c_str()); + } + // Import target module + pybind11::module target_module = + pybind11::module::import(target_module_name.c_str()); + // Instantiate target + return target_module.attr(target_type_name.c_str())(); + }); + } + + pybind11::object create_target( + std::function init) { + LOG(OrderMan, bts::Info, "PythonActivator::create_target. {}", target); + return PythonInterpreter::exec_t([&]() { + std::string target_module_name; + std::string target_type_name; + PythonInterpreter::parse_target( + target, &target_module_name, &target_type_name); + // Prepare module path + pybind11::module sys = pybind11::module::import("sys"); + for (const auto &module_path : module_paths) { + sys.attr("path").attr("append")(module_path.c_str()); + } + // Import target module + pybind11::module target_module = + pybind11::module::import(target_module_name.c_str()); + // Instantiate target + pybind11::object obj = target_module.attr(target_type_name.c_str())(); + init(obj); + return obj; + }); + } + + void reset() { target_obj.reset(); } +}; + +} // namespace python_core diff --git a/src/lib_python_core/python_core/python_interpreter.cpp b/src/lib_python_core/python_core/python_interpreter.cpp new file mode 100644 index 000000000..401609116 --- /dev/null +++ b/src/lib_python_core/python_core/python_interpreter.cpp @@ -0,0 +1,50 @@ +#include "python_interpreter.h" +#include "spark/logging/logging.h" +#include "spark/utils/string_utils.h" + +#include +#include +#include +#include + +namespace python_core { + +using bts::system::string_utils::snake_to_pascal_case; +using bts::system::string_utils::split; + +std::mutex PythonInterpreter::mx; +std::optional PythonInterpreter::interpreter; + +void PythonInterpreter::init_interpreter() { + if (interpreter.has_value()) { + return; + } + static std::mutex local_mx; + std::lock_guard lock{local_mx}; + if (interpreter.has_value()) { + return; + } + LOG(Gen, bts::Info, "Creating global py interpreter"); + // Don't let python handle the signals (otherwise it'll hang on ctrl+c) + auto orig_signal_handler = std::signal(SIGINT, SIG_DFL); + interpreter.emplace(false); + std::signal(SIGINT, orig_signal_handler); +} + +bool PythonInterpreter::parse_target( + const std::string &target, + std::string *module_name, + std::string *type_name) { + auto parts = split(target, "."); + if (parts.empty()) { + return false; + } + if (parts.size() == 1) { + snake_to_pascal_case(parts.emplace_back(parts[0])); + } + *module_name = parts[0]; + *type_name = parts[1]; + return !module_name->empty() && !type_name->empty(); +} + +} // namespace python_core diff --git a/src/lib_python_core/python_core/python_interpreter.h b/src/lib_python_core/python_core/python_interpreter.h new file mode 100644 index 000000000..3cb240f2d --- /dev/null +++ b/src/lib_python_core/python_core/python_interpreter.h @@ -0,0 +1,125 @@ +#pragma once + +#include "spark/utils/bts_assert.h" +#include "spark/utils/string_utils.h" + +#include +#include +#include +#include +#include + +namespace python_core { + +class PythonInterpreter { + static std::mutex mx; + static std::optional interpreter; + static void init_interpreter(); + + public: + /** + * Parses module and type name from a \p target arg. The following formats are supported: + * 1) . + * 2) + * In this case, the type name defaults to the PascalCase equivalent of the module name. + * For example, the module "foo_bar" would have type name "FooBar". + */ + static bool parse_target( + const std::string &target, + std::string *module_name, + std::string *type_name); + + static void exec(std::function fn) { + init_interpreter(); + std::lock_guard lock{mx}; + fn(); + } + + template + static T exec_t(std::function fn) { + init_interpreter(); + std::lock_guard lock{mx}; + pybind11::gil_scoped_acquire gil; + T rv = fn(); + pybind11::gil_scoped_release gil_rel; + return rv; + } + + template + static pybind11::object create_object( + const std::vector &module_paths, + const std::string &target_name, + Args &&...args) { + std::string target_module_name; + std::string target_type_name; + bts_assert( + parse_target(target_name, &target_module_name, &target_type_name)); + return create_object( + module_paths, target_module_name, target_type_name, + std::forward(args)...); + } + + template + static pybind11::object create_object( + const std::vector &module_paths, + const std::string &target_type_name, + const std::string &target_module_name, + Args &&...args) { + namespace py = pybind11; + return exec_t([&]() { + // Prepare module path + py::module sys = py::module::import("sys"); + for (const auto &module_path : module_paths) { + sys.attr("path").attr("append")(module_path.c_str()); + } + // Import target module + py::module target_module = + py::module::import(target_module_name.c_str()); + // Instantiate target + return target_module.attr(target_type_name.c_str())( + std::forward(args)...); + }); + } + + template + static pybind11::object create_object_with_init( + const std::vector &module_paths, + const std::string &target_name, + std::function init_fn, + Args &&...args) { + std::string target_module_name; + std::string target_type_name; + bts_assert( + parse_target(target_name, &target_module_name, &target_type_name)); + return create_object_with_init( + module_paths, target_module_name, target_type_name, init_fn, + std::forward(args)...); + } + + template + static pybind11::object create_object_with_init( + const std::vector &module_paths, + const std::string &target_type_name, + const std::string &target_module_name, + std::function init_fn, + Args &&...args) { + namespace py = pybind11; + return exec_t([&]() { + // Prepare module path + py::module sys = py::module::import("sys"); + for (const auto &module_path : module_paths) { + sys.attr("path").attr("append")(module_path.c_str()); + } + // Import target module + py::module target_module = + py::module::import(target_module_name.c_str()); + // Instantiate target + py::object obj = target_module.attr(target_type_name.c_str())( + std::forward(args)...); + init_fn(obj); + return obj; + }); + } +}; + +} // namespace python_core diff --git a/src/lib_python_md/CMakeLists.txt b/src/lib_python_md/CMakeLists.txt index 2da796d73..9cb509bf5 100644 --- a/src/lib_python_md/CMakeLists.txt +++ b/src/lib_python_md/CMakeLists.txt @@ -27,7 +27,12 @@ find_package(pybind11 REQUIRED) # 1) python_md init_library(SHARED) -target_link_libraries(${PROJECT_NAME} PUBLIC spark) +target_link_libraries( + ${PROJECT_NAME} + PUBLIC + spark + python_core +) target_include_directories( ${PROJECT_NAME} @@ -55,6 +60,7 @@ target_link_libraries( PUBLIC pybind11::embed spark + python_core ) target_include_directories( ${module_proj} @@ -70,7 +76,7 @@ target_include_directories( ) add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${PROJECT_NAME}.pyi - DEPENDS ${PROJECT_NAME} + DEPENDS ${PROJECT_NAME} ${module_proj} COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen ${PROJECT_NAME}_module COMMENT "Generating py stubs" ) diff --git a/src/lib_python_md/python_md/md_sink.cpp b/src/lib_python_md/python_md/md_bridge.cpp similarity index 55% rename from src/lib_python_md/python_md/md_sink.cpp rename to src/lib_python_md/python_md/md_bridge.cpp index c09561296..fa0a6a340 100644 --- a/src/lib_python_md/python_md/md_sink.cpp +++ b/src/lib_python_md/python_md/md_bridge.cpp @@ -1,7 +1,7 @@ -#include "md_sink.h" +#include "md_bridge.h" namespace bts::md::py_interop { -MdSink::~MdSink() = default; +MdBridge::~MdBridge() = default; } // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/md_bridge.h b/src/lib_python_md/python_md/md_bridge.h new file mode 100644 index 000000000..ecb97f818 --- /dev/null +++ b/src/lib_python_md/python_md/md_bridge.h @@ -0,0 +1,23 @@ +#pragma once + +#include "python_md/py_book.h" +#include "python_md/py_security.h" +#include "python_md/py_trade.h" + +#include + +namespace bts::md::py_interop { + +class MdBridge { + public: + virtual ~MdBridge(); + + // Python interface + virtual void on_securities( + const std::vector> &securities) = 0; + virtual void on_books(const std::vector> &books) = 0; + virtual void on_trades( + const std::vector> &trades) = 0; +}; + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/md_bridge_impl.cpp b/src/lib_python_md/python_md/md_bridge_impl.cpp new file mode 100644 index 000000000..ab9bff940 --- /dev/null +++ b/src/lib_python_md/python_md/md_bridge_impl.cpp @@ -0,0 +1,128 @@ +#include "python_md/md_bridge_impl.h" +#include "python_core/python_activator.h" +#include "python_md/md_provider.h" +#include "spark/io/json_formatter.h" +#include "spark/logging/logging.h" +#include "spark/logging/trace_log.h" +#include "spark/utils/Delegate.h" +#include "spark/utils/stopwatch.h" + +#include +#include + +using std::shared_ptr; + +namespace py = pybind11; + +namespace bts::md::py_interop { + +using bts::core::Delegate; +using bts::io::json_formatter; +using bts::utils::Stopwatch; +using python_core::PythonActivatorT; +using std::string; +using std::vector; + +#define ASSERT_FN(test, fn, ...) \ + bts_assert_msgf(test, "md_bridge::" fn ". " __VA_ARGS__) +#define LOG_FN(level, fn, ...) LOG(md, level, "md_bridge::" fn ". " __VA_ARGS__) +#define DIAG_FN(fn, ...) LOG_FN(bts::Diag, fn, __VA_ARGS__) +#define INFO_FN(fn, ...) LOG_FN(bts::Info, fn, __VA_ARGS__) +#define WARN_FN(fn, ...) LOG_FN(bts::Warn, fn, __VA_ARGS__) +#define ERROR_FN(fn, ...) LOG_FN(bts::Error, fn, __VA_ARGS__) + +#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__) +#define INFO(...) _LOG(bts::Info, __VA_ARGS__) +#define ERROR(...) _LOG(bts::Error, __VA_ARGS__) + +#define _LOG_TRACE(level, name) \ + LOG_TRACE(md, level, std::string("md_bridge"), std::string(name)) +#define INFO_TRACE(name) _LOG_TRACE(bts::Info, name) +#define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name) +#define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name) + +#define DIAG_TRACE_PARAMS(name, params) \ + LOG_TRACE_PARAMS( \ + md, bts::Diag, std::string("md_bridge"), std::string(name), params) + +#define INFO_TRACE_PARAMS(name, params) \ + LOG_TRACE_PARAMS( \ + md, bts::Info, std::string("md_bridge"), std::string(name), params) + +#define WARN_TRACE_PARAMS(name, params) \ + LOG_TRACE_PARAMS( \ + md, bts::Warn, std::string("md_bridge"), std::string(name), params) + +MdBridgeImpl::MdBridgeImpl(string target, vector module_paths) + : target(std::move(target)), module_paths(std::move(module_paths)) {} + +void MdBridgeImpl::on_securities( + const std::vector> &securities) { + INFO_TRACE_PARAMS("on_securities", securities.size()); + incoming_securities.push_many(securities.begin(), securities.end()); +} + +void MdBridgeImpl::on_books(const std::vector> &books) { + DIAG_TRACE_PARAMS("on_books", json_formatter(books.size())); + incoming_books.push_many(books.begin(), books.end()); +} + +void MdBridgeImpl::on_trades( + const std::vector> &trades) { + DIAG_TRACE_PARAMS("on_trades", json_formatter(trades.size())); + incoming_trades.push_many(trades.begin(), trades.end()); +} + +bool MdBridgeImpl::pop_security(shared_ptr &result) { + auto res = incoming_securities.pop(); + if (res) { + result = std::move(*res); + return true; + } + result = nullptr; + return false; +} + +bool MdBridgeImpl::pop_book(shared_ptr &result) { + auto res = incoming_books.pop(); + if (res) { + result = std::move(*res); + return true; + } + result = nullptr; + return false; +} + +bool MdBridgeImpl::pop_trade(shared_ptr &result) { + auto res = incoming_trades.pop(); + if (res) { + result = std::move(*res); + return true; + } + result = nullptr; + return false; +} + +int MdBridgeImpl::run_py(volatile bool *keep_going) { + bts_assert(keep_going != nullptr); + PythonActivatorT provider_activator{target, module_paths}; + provider = provider_activator.get_target([&](MdProvider &pro) { + // This will start another thread in python, and call back into c++ from there. + auto sw = Stopwatch::start_new(); + pro.start(this); + INFO_TRACE_PARAMS( + "run_py", + json_formatter(fmt::format( + "MdProvider.start took {:.2f}ms", sw.elapsed().total_millis()))); + }); + // Release gil to let python run + py::gil_scoped_release gil_rel; + while (*keep_going) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + return 0; +} + +MdBridgeImpl::~MdBridgeImpl() = default; + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/md_bridge_impl.h b/src/lib_python_md/python_md/md_bridge_impl.h new file mode 100644 index 000000000..b3472bea6 --- /dev/null +++ b/src/lib_python_md/python_md/md_bridge_impl.h @@ -0,0 +1,36 @@ +#pragma once + +#include "python_md/md_bridge.h" +#include "python_md/md_provider.h" +#include "spark/collections/concurrent_queue.h" + +namespace bts::md::py_interop { + +class MdBridgeImpl : public MdBridge { + public: + MdBridgeImpl(std::string target, std::vector module_paths); + ~MdBridgeImpl(); + + // Python interface + void on_securities( + const std::vector> &securities) override; + void on_books(const std::vector> &books) override; + void on_trades(const std::vector> &trades) override; + + // C++ interface + bool pop_security(std::shared_ptr &result); + bool pop_book(std::shared_ptr &result); + bool pop_trade(std::shared_ptr &result); + int run_py(volatile bool *keep_going); + + private: + std::string target; + std::vector module_paths; + collections::ConcurrentQueue> + incoming_securities; + collections::ConcurrentQueue> incoming_books; + collections::ConcurrentQueue> incoming_trades; + MdProvider *provider; +}; + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/md_provider.h b/src/lib_python_md/python_md/md_provider.h index 74ad68109..f23848156 100644 --- a/src/lib_python_md/python_md/md_provider.h +++ b/src/lib_python_md/python_md/md_provider.h @@ -1,13 +1,13 @@ #pragma once -#include "python_md/md_sink.h" +#include "python_md/md_bridge.h" namespace bts::md::py_interop { class MdProvider { public: virtual ~MdProvider(); - virtual void start(MdSink *sink) = 0; + virtual void start(MdBridge *bridge) = 0; virtual void stop() = 0; }; diff --git a/src/lib_python_md/python_md/md_sink.h b/src/lib_python_md/python_md/md_sink.h deleted file mode 100644 index ee24e99d8..000000000 --- a/src/lib_python_md/python_md/md_sink.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include "python_md/py_book.h" -#include "python_md/py_security.h" -#include "python_md/py_trade.h" - -#include - -namespace bts::md::py_interop { - -class MdSink { - public: - virtual ~MdSink(); - - virtual void on_securities(const std::vector &securities) = 0; - virtual void on_books(const std::vector &books) = 0; - virtual void on_trades(const std::vector &trades) = 0; -}; - -} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/md_sink_impl.cpp b/src/lib_python_md/python_md/md_sink_impl.cpp deleted file mode 100644 index 8147d4990..000000000 --- a/src/lib_python_md/python_md/md_sink_impl.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#include "python_md/md_sink_impl.h" -#include "python_md/py_security.h" -#include "python_md/py_trade.h" -#include "spark/logging/logging.h" -#include "spark/utils/formatters/exception_formatter.h" - -#include - -using std::vector; - -namespace bts::md::py_interop { - -#define ASSERT_FN(test, fn, ...) \ - bts_assert_msgf(test, "md_sink_impl::" fn ". " __VA_ARGS__) -#define LOG_FN(level, fn, ...) \ - LOG(md, level, "md_sink_impl::" fn ". " __VA_ARGS__) -#define DIAG_FN(fn, ...) LOG_FN(bts::Diag, fn, __VA_ARGS__) -#define INFO_FN(fn, ...) LOG_FN(bts::Info, fn, __VA_ARGS__) -#define WARN_FN(fn, ...) LOG_FN(bts::Warn, fn, __VA_ARGS__) -#define ERROR_FN(fn, ...) LOG_FN(bts::Error, fn, __VA_ARGS__) - -#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__) -#define DIAG(...) _LOG(bts::Diag, __VA_ARGS__) -#define INFO(...) _LOG(bts::Info, __VA_ARGS__) -#define ERROR(...) _LOG(bts::Error, __VA_ARGS__) - -struct MdSinkImpl::PImpl { - std::function &)> securities_callback; - std::function &)> books_callback; - std::function &)> trades_callback; -}; - -MdSinkImpl::MdSinkImpl() : me(std::make_unique()) {} - -MdSinkImpl::MdSinkImpl(MdSinkImpl &&) = default; -MdSinkImpl &MdSinkImpl::operator=(MdSinkImpl &&) = default; -MdSinkImpl::~MdSinkImpl() = default; - -void MdSinkImpl::on_books(const vector &books) { - DIAG( - "[py_md_sink_books.json][{}]", - fmt::format("{}", fmt::join(books, ", "))); - try { - if (books.empty()) { - WARN_FN("on_books", "books empty"); - return; - } - if (me->books_callback) { - me->books_callback(books); - } - } catch (const std::exception &ex) { - ERROR_FN("on_books", "ex: {}", ex); - } -} - -void MdSinkImpl::on_securities(const vector &securities) { - DIAG( - "[py_md_sink_secs.json][{}]", - fmt::format("{}", fmt::join(securities, ", "))); - try { - if (securities.empty()) { - WARN_FN("on_securities", "securities empty"); - return; - } - if (me->securities_callback) { - me->securities_callback(securities); - } - } catch (const std::exception &ex) { - ERROR_FN("on_securities", "ex: {}", ex); - } -} - -void MdSinkImpl::on_trades(const vector &trades) { - DIAG( - "[py_md_sink_trds.json][{}]", - fmt::format("{}", fmt::join(trades, ", "))); - try { - if (trades.empty()) { - WARN_FN("on_trades", "trades empty"); - return; - } - if (me->trades_callback) { - me->trades_callback(trades); - } - } catch (const std::exception &ex) { - ERROR_FN("on_trades", "ex: {}", ex); - } -} - -void MdSinkImpl::set_books_callback( - std::function &)> callback) { - me->books_callback = callback; -} - -void MdSinkImpl::set_trades_callback( - std::function &)> callback) { - me->trades_callback = callback; -} - -void MdSinkImpl::set_securities_callback( - std::function &)> callback) { - me->securities_callback = callback; -} - -} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/md_sink_impl.h b/src/lib_python_md/python_md/md_sink_impl.h deleted file mode 100644 index e6ca05a6d..000000000 --- a/src/lib_python_md/python_md/md_sink_impl.h +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once - -#include "python_md/md_sink.h" -#include "python_md/py_book.h" -#include "python_md/py_trade.h" - -#include -#include - -namespace bts::md::py_interop { - -class MdSinkImpl : public MdSink { - public: - MdSinkImpl(); - MdSinkImpl(MdSinkImpl &&); - ~MdSinkImpl() override; - - MdSinkImpl &operator=(MdSinkImpl &&); - - void on_books(const std::vector &books) override; - void on_trades(const std::vector &trades) override; - void on_securities(const std::vector &securities) override; - - void set_books_callback( - std::function &)> callback); - void set_trades_callback( - std::function &)> callback); - void set_securities_callback( - std::function &)> callback); - - private: - struct PImpl; - std::unique_ptr me; -}; - -} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_md_executor.cpp b/src/lib_python_md/python_md/py_md_executor.cpp deleted file mode 100644 index d873273eb..000000000 --- a/src/lib_python_md/python_md/py_md_executor.cpp +++ /dev/null @@ -1,143 +0,0 @@ -#include "py_md_executor.h" -#include "python_md/md_provider.h" -#include "python_md/md_sink.h" -#include "spark/utils/string_utils.h" -#include "spark/utils/thread_id.h" - -#include -#include -#include -#include -#include -#include - -using bts::md::py_interop::PyBook; -using bts::md::py_interop::PySecurity; -using bts::md::py_interop::PyTrade; -using std::vector; - -namespace bts::md::py_interop { - -namespace py = pybind11; - -using bts::system::string_utils::snake_to_pascal_case; -using bts::system::string_utils::split; -using std::int64_t; -using std::size_t; -using std::string; - -/** - * Parses module and type name from a \p py_md_provider arg. The following formats are supported: - * 1) . - * 2) - * In this case, the type name defaults to the PascalCase equivalent of the module name. - * For example, the module "foo_bar" would have type name "FooBar". - */ -static void parse_py_md_provider( - const std::string &py_md_provider, - std::string *module_name, - std::string *type_name) { - auto parts = split(py_md_provider, "."); - if (parts.empty()) { - return; - } - if (parts.size() == 1) { - snake_to_pascal_case(parts.emplace_back(parts[0])); - } - *module_name = parts[0]; - *type_name = parts[1]; -} - -struct __attribute__((visibility("hidden"))) PyMdExecutor_PImpl { - std::string py_md_provider; - std::vector module_paths; - std::unique_ptr provider; - MdSink *sink; - std::optional provider_obj; - bool is_running = false; - size_t thread_id = BTS_THIS_THREAD_ID; - std::mutex instance_mx; - std::optional interpreter{}; - std::function &)> securities_callback; - std::function &)> books_callback; - std::function &)> trades_callback; - - void start() { - check_access(); - { - std::lock_guard lock{instance_mx}; - if (is_running) { - return; - } - interpreter.emplace(false); - is_running = true; - } - provider_obj = create_provider(); - MdProvider *provider = provider_obj->cast(); - provider->start(sink); - } - - void stop() { - check_access(); - std::lock_guard lock{instance_mx}; - if (!provider_obj) { - return; - } - MdProvider *providerx = provider_obj->cast(); - providerx->stop(); - provider_obj = std::nullopt; - interpreter.reset(); - } - - py::object create_provider() { - string provider_module_name; - string provider_type_name; - parse_py_md_provider( - py_md_provider, &provider_module_name, &provider_type_name); - // Prepare module path - py::module sys = py::module::import("sys"); - for (const auto &module_path : module_paths) { - sys.attr("path").attr("append")(module_path.c_str()); - } - // Import provider module - py::module provider_module = - py::module::import(provider_module_name.c_str()); - // Instantiate provider - return provider_module.attr(provider_type_name.c_str())(); - } - - void check_access() { - if (BTS_THIS_THREAD_ID != thread_id) { - throw std::runtime_error{ - "All calls to PyMdExecutor must be on the same thread."}; - } - } -}; - -PyMdExecutor::PyMdExecutor( - std::string py_md_provider, - std::vector module_paths, - MdSink *sink) - : me_(new PyMdExecutor_PImpl()) { - auto *me = static_cast(me_); - me->py_md_provider = std::move(py_md_provider); - me->module_paths = std::move(module_paths); - me->sink = sink; -} - -PyMdExecutor::~PyMdExecutor() { - auto *me = static_cast(me_); - delete me; -} - -void PyMdExecutor::start() { - auto *me = static_cast(me_); - me->start(); -} - -void PyMdExecutor::stop() { - auto *me = static_cast(me_); - me->stop(); -} - -} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_md_executor.h b/src/lib_python_md/python_md/py_md_executor.h deleted file mode 100644 index 72b05b686..000000000 --- a/src/lib_python_md/python_md/py_md_executor.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include "python_md/md_sink.h" - -#include - -namespace bts::md::py_interop { - -class PyMdExecutor { - public: - PyMdExecutor( - std::string py_md_provider, - std::vector module_paths, - MdSink *sink); - ~PyMdExecutor(); - - void start(); - void stop(); - - private: - void *me_; -}; - -} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_md_provider.h b/src/lib_python_md/python_md/py_md_provider.h index d1aff32cc..a76147737 100644 --- a/src/lib_python_md/python_md/py_md_provider.h +++ b/src/lib_python_md/python_md/py_md_provider.h @@ -1,7 +1,7 @@ #pragma once +#include "python_md/md_bridge.h" #include "python_md/md_provider.h" -#include "python_md/md_sink.h" #include @@ -11,7 +11,7 @@ namespace bts::md::py_interop { * An implementation of `bts::md::py_interop::MdProvider` that binds to a python object at runtime. * * @note The python object must derive from 'python_md_module.MdProvider', and define methods: - * 1. 'start(md_sink: python_md_module.MdSink)' + * 1. 'start(md_bridge: python_md_module.MdBridge)' * 2. 'stop' * @note 'python_md_module' is a python module defined by us in c++ (see 'py_module.cpp'). */ @@ -21,8 +21,8 @@ class PyMdProvider : public MdProvider { ~PyMdProvider() override; - void start(MdSink *sink) override { - PYBIND11_OVERLOAD_PURE(void, MdProvider, start, sink); + void start(MdBridge *bridge) override { + PYBIND11_OVERLOAD_PURE(void, MdProvider, start, bridge); } void stop() override { PYBIND11_OVERLOAD_PURE(void, MdProvider, stop); } diff --git a/src/lib_python_md/python_md/py_module.cpp b/src/lib_python_md/python_md/py_module.cpp index 91b6bfb3e..b139ecff0 100644 --- a/src/lib_python_md/python_md/py_module.cpp +++ b/src/lib_python_md/python_md/py_module.cpp @@ -1,6 +1,5 @@ #include "md_provider.h" -#include "python_md/md_sink.h" -#include "python_md/md_sink_impl.h" +#include "python_md/md_bridge_impl.h" #include "python_md/py_book_level.h" #include "python_md/py_book_side.h" #include "python_md/py_contract_price_info.h" @@ -10,12 +9,12 @@ #include "python_md/py_security.h" #include "python_md/py_trade.h" +#include #include #include +using bts::md::py_interop::MdBridge; using bts::md::py_interop::MdProvider; -using bts::md::py_interop::MdSink; -using bts::md::py_interop::MdSinkImpl; using bts::md::py_interop::PyBook; using bts::md::py_interop::PyBookLevel; using bts::md::py_interop::PyBookSide; @@ -28,6 +27,9 @@ using bts::md::py_interop::PyOption; using bts::md::py_interop::PySecurity; using bts::md::py_interop::PySpot; using bts::md::py_interop::PyTrade; +using std::shared_ptr; + +PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr) namespace py = pybind11; @@ -62,7 +64,7 @@ PYBIND11_MODULE(python_md_module, m) { .PROP(PyBookSide, is_ascending) .def("__len__", &PyBookSide::size) .def("__str__", &PyBookSide::to_string); - py::class_(m, "PyBook") + py::class_>(m, "PyBook") .def(py::init<>()) .PROP(PyBook, venue) .PROP(PyBook, security_type) @@ -73,9 +75,9 @@ PYBIND11_MODULE(python_md_module, m) { .DEF(PyBook, set_asks) .def("__len__", &PyBook::size) .def("__str__", &PyBook::to_string); - py::bind_vector>(m, "PyBookList"); + py::bind_vector>>(m, "PyBookList"); // trade - py::class_(m, "PyTrade") + py::class_>(m, "PyTrade") .def(py::init<>()) .PROP(PyTrade, venue) .PROP(PyTrade, security_type) @@ -84,7 +86,7 @@ PYBIND11_MODULE(python_md_module, m) { .PROP(PyTrade, quantity) .PROP(PyTrade, aggressor_side) .PROP(PyTrade, timestamp); - py::bind_vector>(m, "PyTradeList"); + py::bind_vector>>(m, "PyTradeList"); // price info py::class_(m, "PyContractPriceInfo") .def(py::init<>()) @@ -129,7 +131,7 @@ PYBIND11_MODULE(python_md_module, m) { .PROP(PyOption, is_call) .PROP(PyOption, strike) .def("__str__", &PyOption::to_string); - py::class_(m, "PySecurity") + py::class_>(m, "PySecurity") .def(py::init<>()) .PROP(PySecurity, id) .PROP(PySecurity, exchange_id) @@ -143,21 +145,16 @@ PYBIND11_MODULE(python_md_module, m) { .PROP(PySecurity, future) .PROP(PySecurity, option) .def("__str__", &PySecurity::to_string); - py::bind_vector>(m, "PySecurityList"); - // md sink - py::class_(m, "MdSink") - .DEF_ARGS(MdSink, on_securities, py::arg("securities")) - .DEF_ARGS(MdSink, on_books, py::arg("books")) - .DEF_ARGS(MdSink, on_trades, py::arg("trades")); - py::class_(m, "MdSinkImpl") - .def(py::init<>()) - .DEF_ARGS(MdSinkImpl, on_securities, py::arg("securities")) - .DEF_ARGS(MdSinkImpl, on_books, py::arg("books")) - .DEF_ARGS(MdSinkImpl, on_trades, py::arg("trades")); + py::bind_vector>>(m, "PySecurityList"); + // md bridge + py::class_(m, "MdBridge") + .DEF_ARGS(MdBridge, on_securities, py::arg("securities")) + .DEF_ARGS(MdBridge, on_books, py::arg("books")) + .DEF_ARGS(MdBridge, on_trades, py::arg("trades")); // md provider py::class_(m, "MdProvider") .def(py::init<>()) - .def("start", &MdProvider::start, py::arg("sink")) + .DEF_ARGS(MdProvider, start, py::arg("bridge")) .DEF(MdProvider, stop); } diff --git a/src/lib_python_md/python_md_providers/bybit_md_provider.py b/src/lib_python_md/python_md_providers/bybit_md_provider.py index 6380e15a5..5c625f02f 100644 --- a/src/lib_python_md/python_md_providers/bybit_md_provider.py +++ b/src/lib_python_md/python_md_providers/bybit_md_provider.py @@ -4,6 +4,7 @@ import logging import sys import threading import signal +from typing import Optional from pybit.unified_trading import WebSocket import python_md_module as pymd from bybit_conf import CONTRACTS @@ -73,7 +74,8 @@ class BybitMdProvider(pymd.MdProvider): def __init__(self): super().__init__() self.ws = None - self.sink = None + self.ws_thread: Optional[threading.Thread] = None + self.bridge: Optional[pymd.MdBridge] = None self.stop_event = threading.Event() self._setup_signal_handlers() @@ -87,8 +89,8 @@ class BybitMdProvider(pymd.MdProvider): signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGTSTP, signal_handler) - def start(self, sink: pymd.MdSink): - self.sink = sink + def start_impl(self, bridge: pymd.MdBridge): + self.bridge = bridge logging.info("Starting BybitMdProvider") contract_specs = fetch_contract_specs() @@ -139,8 +141,8 @@ class BybitMdProvider(pymd.MdProvider): logging.error(f"Failed to create security for contract: {contract}, error: {e}") continue - sink.on_securities(sec_list) - logging.info("Sent securities list--sink.on_securities called.") + bridge.on_securities(sec_list) + logging.info("Sent securities list--bridge.on_securities called.") try: self.ws = WebSocket( @@ -176,6 +178,10 @@ class BybitMdProvider(pymd.MdProvider): logging.error(f"Bybit WebSocket fatal error: {e}", exc_info=True) self.stop() + def start(self, bridge: pymd.MdBridge): + self.ws_thread = threading.Thread(target=lambda: self.start_impl(bridge)) + self.ws_thread.start() + def _on_book_update(self, data: dict): try: topic = data.get("topic", "") @@ -204,11 +210,11 @@ class BybitMdProvider(pymd.MdProvider): book_list = pymd.PyBookList() book_list.append(book) - if self.sink: - self.sink.on_books(book_list) + if self.bridge: + self.bridge.on_books(book_list) logging.info(f"✔ on_books called for {contract}") else: - logging.warning("Sink is None, cannot call on_books") + logging.warning("Bridge is None, cannot call on_books") except Exception as e: logging.error(f"Error processing book update for {contract}: {e}", exc_info=True) @@ -233,7 +239,9 @@ class BybitMdProvider(pymd.MdProvider): logging.error(f"Error while closing WebSocket: {e}", exc_info=True) self.ws = None - self.sink = None + if self.ws_thread: + self.ws_thread.join() + self.bridge = None logging.info("✔ Stopped BybitMdProvider.") logging.debug("Threads after stop:") diff --git a/src/lib_python_md/python_md_providers/dummy_md_provider.py b/src/lib_python_md/python_md_providers/dummy_md_provider.py index b20cac8a7..22983a692 100644 --- a/src/lib_python_md/python_md_providers/dummy_md_provider.py +++ b/src/lib_python_md/python_md_providers/dummy_md_provider.py @@ -7,7 +7,7 @@ class DummyMdProvider(pymd.MdProvider): super(DummyMdProvider, self).__init__() pass - def start(self, sink: pymd.MdSink): + def start(self, bridge: pymd.MdBridge): # start subscriptions, bind callbacks etc. def add_fake_securities(): sec_list = pymd.PySecurityList() @@ -28,7 +28,7 @@ class DummyMdProvider(pymd.MdProvider): qty_info=get_quantity_info(tick_unit=1, tick_base=1), ), )) - sink.on_securities(sec_list) + bridge.on_securities(sec_list) def add_fake_books(): bids = get_book_side([(99.75, 8), (99.50, 11)], is_bid_side=True) @@ -42,7 +42,7 @@ class DummyMdProvider(pymd.MdProvider): # pass book to sink book_list = pymd.PyBookList() book_list.append(book) - sink.on_books(book_list) + bridge.on_books(book_list) def add_fake_trades(): trade = get_trade( @@ -55,7 +55,7 @@ class DummyMdProvider(pymd.MdProvider): ) trade_list = pymd.PyTradeList() trade_list.append(trade) - sink.on_trades(trade_list) + bridge.on_trades(trade_list) add_fake_securities() add_fake_books() diff --git a/src/lib_python_md/python_md_providers/gateio_md_provider.py b/src/lib_python_md/python_md_providers/gateio_md_provider.py index 99a99f607..9002d1f16 100644 --- a/src/lib_python_md/python_md_providers/gateio_md_provider.py +++ b/src/lib_python_md/python_md_providers/gateio_md_provider.py @@ -61,8 +61,9 @@ class GateIoMdProvider(pymd.MdProvider): def __init__(self): super().__init__() self.ws = None + self.ws_thread: Optional[threading.Thread] = None + self.bridge: Optional[pymd.MdBridge] = None self.stop_event = threading.Event() - self.sink = None self.ping_thread = None def send_ping(self): @@ -82,8 +83,8 @@ class GateIoMdProvider(pymd.MdProvider): break time.sleep(15) - def start(self, sink: pymd.MdSink): - self.sink = sink + def start_impl(self, bridge: pymd.MdBridge): + self.bridge = bridge logging.info(f"Connecting to {WS_URL}") self.ws = create_connection(WS_URL) self.ws.settimeout(5) @@ -126,7 +127,7 @@ class GateIoMdProvider(pymd.MdProvider): ) sec_list.append(sec) logging.info(f"Created security: {contract} with specs: {specs}") - sink.on_securities(sec_list) + bridge.on_securities(sec_list) logging.info("Sent securities list--sink.on_securities called.") # 2) Subscribe to order books @@ -185,7 +186,7 @@ class GateIoMdProvider(pymd.MdProvider): book_list = pymd.PyBookList() book_list.append(book) - self.sink.on_books(book_list) + self.bridge.on_books(book_list) logging.info(f"✔ on_books called for {contract}") else: logging.debug("… no bids/asks in payload; skipping") @@ -194,8 +195,10 @@ class GateIoMdProvider(pymd.MdProvider): logging.info("WebSocket closed.") except Exception as e: logging.error("Fatal error in listen loop:", exc_info=True) - finally: - self.stop() + + def start(self, bridge: pymd.MdBridge): + self.ws_thread = threading.Thread(target=lambda: self.start_impl(bridge)) + self.ws_thread.start() def stop(self): """Disconnect and cleanup.""" @@ -213,5 +216,7 @@ class GateIoMdProvider(pymd.MdProvider): pass self.ws = None - self.sink = None + if self.ws_thread: + self.ws_thread.join() + self.bridge = None logging.info("✔ Stopped GateIoMdProvider.") diff --git a/src/lib_python_md/python_md_providers/py_md_utils.py b/src/lib_python_md/python_md_providers/py_md_utils.py index 6b666f28f..68f291b59 100644 --- a/src/lib_python_md/python_md_providers/py_md_utils.py +++ b/src/lib_python_md/python_md_providers/py_md_utils.py @@ -50,6 +50,7 @@ def get_tick_factors(tick_sz: float): case _: raise ValueError(f"Invalid tick size {tick_sz}") + def get_price_info( tick_unit: float, tick_base: float, @@ -173,7 +174,7 @@ def get_security( return sec -def get_book_side(levels: list[(float, float)], is_bid_side: bool): +def get_book_side(levels: list[tuple[float, float]], is_bid_side: bool): book_side = pymd.PyBookSide() book_side.set_is_ascending(not is_bid_side) for price, qty in levels: