#include "plugin_manager.h" #include "python_interop/execution/execution_gateway.h" #include "python_interop/execution/execution_gateway_bridge.h" #include "python_interop/md/md_bridge.h" #include "python_interop/md/md_provider.h" #include "python_interop/plugin_manager_logs.h" #include "python_interop/python_object.h" #include "spark/collections/concurrent_queue.h" #include "spark/collections/concurrent_unordered_map.h" #include "spark/utils/stopwatch.h" #include "spark/utils/string_utils.h" #include #include #include #include #include #include #include #include #include #include #include #include namespace py = pybind11; namespace python_interop { using bts::collections::ConcurrentQueue; using bts::collections::ConcurrentUnorderedMap; using bts::md::py_interop::MdBridge; using bts::utils::Stopwatch; using python_execution::ExecutionGateway; using python_execution::ExecutionGatewayBridge; using std::function; using std::future; using std::mutex; using std::optional; using std::shared_ptr; using std::string; using std::thread; static bool parse_target( const std::string &target, std::string *module_name, std::string *type_name) { auto parts = bts::system::string_utils::split(target, "."); if (parts.empty()) { return false; } if (parts.size() == 1) { bts::system::string_utils::snake_to_pascal_case( parts.emplace_back(parts[0])); } *module_name = parts[0]; *type_name = parts[1]; return !module_name->empty() && !type_name->empty(); } struct __attribute__((visibility("hidden"))) PluginManager_SessionData { py::scoped_interpreter interpreter{false}; std::unordered_map> modules; py::module &import(const std::string &name) { auto [it, is_new] = modules.try_emplace(name, nullptr); if (is_new) { it->second = std::make_unique(py::module::import(name.c_str())); } return *it->second; } }; struct __attribute__((visibility("hidden"))) PluginManager_Impl { volatile bool keep_going; mutex mx; optional work_thread; ConcurrentQueue> work_queue; ConcurrentUnorderedMap< string, shared_ptr>> md_providers; ConcurrentUnorderedMap< string, shared_ptr>> execution_gateways; PluginManager_SessionData *session_data; void start_work_thread() { std::lock_guard lock{mx}; if (work_thread) { return; } keep_going = true; work_thread.emplace([&]() { PluginManager_SessionData sd; mx.lock(); session_data = &sd; mx.unlock(); try { sd.import("sys").attr("path").attr("append")( "python_interop_plugins"); sd.import("spark_core").attr("configure_default_logger")(); sd.import("bybit_md_provider"); sd.import("bybit_execution_gateway"); py::gil_scoped_release gil_rel; Stopwatch sw; int n = 0; while (keep_going) { auto work_req = work_queue.wait_until_pop(); if (!work_req) { continue; } LOG_INVOKING(n); sw.restart(); { py::gil_scoped_acquire gil_acquire; (*work_req)(); } LOG_INVOKED(n, sw.elapsed()); n++; } } catch (const std::exception &ex) { bts_assert_msgf( false, "PythonWorker work_thread threw. {}", std::string(ex.what())); return; } catch (...) { bts_assert_msg(false, "PythonWorker work_thread threw unknown ex."); return; } mx.lock(); session_data = nullptr; mx.unlock(); }); } void stop_work_thread() { mx.lock(); if (work_thread && work_thread->joinable()) { keep_going = false; mx.unlock(); work_thread->join(); work_thread.reset(); } else { mx.unlock(); } } future>> start_md_provider( string provider_name, string target, shared_ptr bridge) { auto prom = std::make_shared>>>(); work_queue.push([this, target = std::move(target), provider_name = std::move(provider_name), bridge = std::move(bridge), prom = prom]() { string module_name; string type_name; if (!parse_target(target, &module_name, &type_name)) { LOG_WORK_ERROR( md, fmt::format("Failed to parse module/type from '{}'", target)); return; } py::object obj = session_data->import(module_name.c_str()) .attr(type_name.c_str())(); auto [md_pro_it, md_pro_is_new] = md_providers.try_emplace( provider_name, std::make_shared>( std::move(obj))); bts_assert(md_pro_is_new); LOG_STARTING("md_provider", provider_name); Stopwatch sw = Stopwatch::start_new(); shared_ptr> md_pro = md_pro_it->second; (*md_pro)->start(bridge.get()); LOG_STARTED("md_provider", provider_name, sw.elapsed()); prom->set_value(md_pro); // Don't let python handle the signals (otherwise it'll hang on ctrl+c) std::signal(SIGINT, SIG_DFL); }); return prom->get_future(); } void stop_md_provider(const std::string &provider_name) { auto it = md_providers.find(provider_name); if (it != md_providers.end()) { (*it->second)->stop(); md_providers.erase(it); } } future>> start_execution_gateway( string gateway_name, string target, shared_ptr bridge) { auto prom = std::make_shared< std::promise>>>(); work_queue.push([this, target = std::move(target), gateway_name = std::move(gateway_name), bridge = std::move(bridge), prom = prom]() { string module_name; string type_name; if (!parse_target(target, &module_name, &type_name)) { LOG_WORK_ERROR( OrderMan, fmt::format("Failed to parse module/type from '{}'", target)); return; } py::object obj = session_data->import(module_name.c_str()) .attr(type_name.c_str())(); auto [exe_gw_it, exe_gw_is_new] = execution_gateways.try_emplace( gateway_name, std::make_shared>(std::move(obj))); bts_assert(exe_gw_is_new); LOG_STARTING("execution_gateway", gateway_name); Stopwatch sw = Stopwatch::start_new(); shared_ptr> exe_gw = exe_gw_it->second; (*exe_gw)->start(bridge.get()); LOG_STARTED("execution_gateway", gateway_name, sw.elapsed()); prom->set_value(exe_gw); // Don't let python handle the signals (otherwise it'll hang on ctrl+c) std::signal(SIGINT, SIG_DFL); }); return prom->get_future(); } void stop_execution_gateway(const std::string &provider_name) { auto it = execution_gateways.find(provider_name); if (it != execution_gateways.end()) { (*it->second)->stop(); execution_gateways.erase(it); } } }; PluginManager::PluginManager() : me_(new PluginManager_Impl()) {} PluginManager::~PluginManager() { stop_work_thread(); delete static_cast(me_); me_ = nullptr; } void PluginManager::start_work_thread() { static_cast(me_)->start_work_thread(); } void PluginManager::stop_work_thread() { static_cast(me_)->stop_work_thread(); } void PluginManager::invoke(std::function callback) { auto *me = static_cast(me_); me->work_queue.push(callback); } std::future>> PluginManager::start_md_provider( string provider_name, string target, shared_ptr bridge) { return static_cast(me_)->start_md_provider( std::move(provider_name), std::move(target), std::move(bridge)); } void PluginManager::stop_md_provider(const std::string &provider_name) { static_cast(me_)->stop_md_provider(provider_name); } std::future>> PluginManager::start_execution_gateway( string gateway_name, string target, shared_ptr bridge) { return static_cast(me_)->start_execution_gateway( std::move(gateway_name), std::move(target), std::move(bridge)); } void PluginManager::stop_execution_gateway(const std::string &provider_name) { return static_cast(me_)->stop_execution_gateway( provider_name); } } // namespace python_interop