refactor: Manage all py plugins (md and exe) in one class

All calls to/from python are performed on one thread via a simple
message queue (necessary because of the global interpreter lock).
This commit is contained in:
John Colvin committed 2025-06-16 11:08:58 -04:00
1 parent c7db105533
commit 8c23665d65
29 files changed
+855 -602

No files matched your search

@@ -3,31 +3,25 @@
#include "python_interop/execution/py_cancel_order_request.h"
#include "python_interop/execution/py_create_order_request.h"
#include "python_interop/execution/py_position_info.h"
#include "python_interop/python_activator.h"
#include "python_interop/python_interpreter.h"
#include "python_interop/python_object.h"
#include "spark/core/TimeSpan.h"
#include "spark/domain_model/Venue.h"
#include "spark/io/json_formatter.h"
#include "spark/logging/logging.h"
#include "spark/logging/trace_log.h"
#include "spark/logging/trace_log_var.h"
#include "spark/orderman/OrderMsg.h"
#include "spark/orderman/log_messages/execution_report_log.h"
#include "spark/orderman/orderman_utils/ExecutionReport.h"
#include "spark/utils/overloaded.h"
#include "spark/utils/stopwatch.h"
#include <pybind11/pybind11.h>
#include <future>
#include <memory>
using std::shared_ptr;
namespace py = pybind11;
namespace python_execution {
using bts::core::Delegate;
using bts::core::TimeSpan;
using bts::io::json_formatter;
using bts::orderman::RFQMsg;
using python_interop::PythonActivatorT;
using std::string;
using std::vector;
@@ -60,10 +54,8 @@ using std::vector;
OrderMan, bts::Warn, std::string("exe_gw_bridge"), std::string(name), \
params)
ExecutionGatewayBridgeImpl::ExecutionGatewayBridgeImpl(
string name, string target, vector<string> module_paths)
: name(std::move(name)), target(std::move(target)),
module_paths(std::move(module_paths)) {}
ExecutionGatewayBridgeImpl::ExecutionGatewayBridgeImpl(string name)
: name(std::move(name)) {}
void ExecutionGatewayBridgeImpl::on_order_msg(PyOrderRequest msg) {
INFO_TRACE("on_order_msg");
@@ -106,11 +98,11 @@ bool ExecutionGatewayBridgeImpl::pop_exe_report(
void ExecutionGatewayBridgeImpl::on_securities(
std::vector<std::shared_ptr<bts::md::py_interop::PySecurity>> securities) {
INFO_TRACE_PARAMS(
"on_securities",
json_formatter(
fmt::format("Received {} securities", this->securities.size())));
"on_securities", json_formatter(fmt::format(
"Received {} securities", securities.size())));
std::lock_guard<std::mutex> lock{securities_mx};
this->securities = std::move(securities);
this->is_securities_dirty = true;
is_securities_dirty = true;
}
shared_ptr<const PyExecutionReport>
@@ -119,12 +111,10 @@ ExecutionGatewayBridgeImpl::pop_exe_report() {
return res ? *res : nullptr;
}
int ExecutionGatewayBridgeImpl::run_py(volatile bool *keep_going) {
INFO_TRACE_PARAMS("run_py", json_formatter("Begin"));
bts_assert(keep_going != nullptr);
PythonActivatorT<ExecutionGateway> gw_activator{target, module_paths};
gw = gw_activator.get_target(
[&](ExecutionGateway &local_gw) { local_gw.start(this); }, name);
int ExecutionGatewayBridgeImpl::run_py(
volatile bool *keep_going,
std::shared_ptr<python_interop::PythonObjectT<ExecutionGateway>> gw_ptr,
std::function<void(std::function<void()>)> py_invoke) {
INFO_TRACE_PARAMS("run_py", json_formatter("Waiting for ready..."));
auto sw = bts::utils::Stopwatch::start_new();
while (!wait_for_ready(TimeSpan::from_seconds(1)) && *keep_going) {
@@ -133,57 +123,100 @@ int ExecutionGatewayBridgeImpl::run_py(volatile bool *keep_going) {
"PyExecutionGateway still not ready after {:.2f}s",
sw.total_elapsed().total_seconds())));
}
INFO_TRACE_PARAMS(
"run", json_formatter(fmt::format(
"PyExecutionGateway ready after {:.2f}s",
sw.total_elapsed().total_seconds())));
ExecutionGateway &gw = **gw_ptr;
std::vector<std::shared_ptr<bts::md::py_interop::PySecurity>> new_securities;
while (is_ready.value() && *keep_going) {
INFO_TRACE_PARAMS("run_py", json_formatter("Waiting for request..."));
if (is_securities_dirty) {
bool was_securities_dirty = false;
{
std::lock_guard<std::mutex> lock{securities_mx};
if (is_securities_dirty) {
was_securities_dirty = true;
new_securities = std::move(securities);
is_securities_dirty = false;
}
}
if (was_securities_dirty) {
INFO_TRACE_PARAMS(
"run_py", json_formatter(std::map<string, string>{
{"name", name}, {"ev", "SENDING_SECS_TO_PY"}}));
std::promise<void> prom;
sw.restart();
py_invoke([&]() {
try {
gw.on_securities(new_securities);
} catch (const std::exception &ex) {
bts_assert_msgf(
false, "py exe gw threw on secs. {}",
std::string(ex.what()));
} catch (...) {
bts_assert_msg(false, "py exe gw threw on secs. UNKNOWN");
}
prom.set_value();
});
prom.get_future().wait();
INFO_TRACE_PARAMS(
"run_py",
json_formatter(fmt::format(
"Sending {} securities to py gw", securities.size())));
python_interop::PythonInterpreter::exec(
[&]() { gw->on_securities(securities); });
is_securities_dirty = false;
json_formatter(std::map<string, string>{
{"name", name},
{"ev", "SENT_SECS_TO_PY"},
{"dur",
fmt::format("{:.3f}ms", sw.elapsed().total_millis())}}));
}
auto req = outgoing_reqs.wait_for_until_pop(
bts::core::TimeSpan::from_seconds(1));
if (!req) {
continue;
}
INFO_TRACE_PARAMS(
"run_py", json_formatter("Forwarding request to py gateway"));
enum ReqType { CREATE = 0, CANCEL_REPLACE = 1, CANCEL = 2 };
static const std::string req_type_names[]{"CREATE", "CXL_REP", "CXL"};
ReqType req_type;
sw.restart();
python_interop::PythonInterpreter::exec([&]() {
std::visit(
bts::overloaded{
[&](std::shared_ptr<PyCreateOrderRequest> req) {
req_type = ReqType::CREATE;
gw->create_order(std::move(req));
},
[&](std::shared_ptr<PyCancelReplaceOrderRequest> req) {
req_type = ReqType::CANCEL_REPLACE;
gw->cancel_replace_order(std::move(req));
},
[&](std::shared_ptr<PyCancelOrderRequest> req) {
req_type = ReqType::CANCEL;
gw->cancel_order(std::move(req));
},
[&](const RFQMsg &) {
bts_assert_msg(false, "RFQ not implemented");
},
},
*req);
});
sw.stop();
INFO_TRACE_PARAMS(
"run_py",
fmt::format(
R"({{ "typ": "{}", "hndl_dur": "{:.3f}ms" }})",
req_type_names[req_type], sw.total_elapsed().total_millis()));
{
INFO_TRACE_PARAMS(
"run_py", json_formatter(std::map<string, string>{
{"name", name}, {"ev", "SENDING_REQ_TO_PY"}}));
std::promise<void> prom;
sw.restart();
py_invoke([&, req = std::move(req)]() {
try {
std::visit(
bts::overloaded{
[&](std::shared_ptr<PyCreateOrderRequest> req) {
req_type = ReqType::CREATE;
gw.create_order(std::move(req));
},
[&](std::shared_ptr<PyCancelReplaceOrderRequest> req) {
req_type = ReqType::CANCEL_REPLACE;
gw.cancel_replace_order(std::move(req));
},
[&](std::shared_ptr<PyCancelOrderRequest> req) {
req_type = ReqType::CANCEL;
gw.cancel_order(std::move(req));
}},
*req);
} catch (const std::exception &ex) {
bts_assert_msgf(
false, "py exe gw threw on req. {}", std::string(ex.what()));
} catch (...) {
bts_assert_msg(false, "py exe gw threw on req. UNKNOWN");
}
prom.set_value();
});
prom.get_future().wait();
INFO_TRACE_PARAMS(
"run_py",
json_formatter(std::map<string, string>{
{"name", name},
{"ev", "SENT_REQ_TO_PY"},
{"typ", req_type_names[req_type]},
{"dur",
fmt::format("{:.3f}ms", sw.elapsed().total_millis())}}));
}
}
python_interop::PythonInterpreter::exec([&]() { gw->stop(); });
return 0;
}
@@ -6,6 +6,7 @@
#include "python_interop/execution/py_order_request.h"
#include "python_interop/execution/py_position_info.h"
#include "python_interop/py_security.h"
#include "python_interop/python_object.h"
#include "spark/collections/concurrent_queue.h"
#include "spark/utils/Delegate.h"
#include "spark/utils/concurrent_value_store.h"
@@ -14,10 +15,7 @@ namespace python_execution {
class ExecutionGatewayBridgeImpl : public ExecutionGatewayBridge {
public:
ExecutionGatewayBridgeImpl(
std::string name,
std::string target,
std::vector<std::string> module_paths);
ExecutionGatewayBridgeImpl(std::string name);
~ExecutionGatewayBridgeImpl() override;
// Python interface
@@ -35,21 +33,22 @@ class ExecutionGatewayBridgeImpl : public ExecutionGatewayBridge {
bool wait_for_ready(bts::core::TimeSpan timeout);
bool pop_exe_report(std::shared_ptr<const PyExecutionReport> &result);
std::shared_ptr<const PyExecutionReport> pop_exe_report();
int run_py(volatile bool *keep_going);
int run_py(
volatile bool *keep_going,
std::shared_ptr<python_interop::PythonObjectT<ExecutionGateway>> gw,
std::function<void(std::function<void()>)> py_invoke);
private:
std::string name;
std::string target;
std::vector<std::string> module_paths;
bts::utils::ConcurrentValueStore<bool> is_ready;
bts::collections::ConcurrentQueue<std::shared_ptr<PyExecutionReport>>
incoming_exe_reports;
bts::collections::ConcurrentQueue<std::shared_ptr<PyPositionInfo>>
incoming_pos_infos;
bts::collections::ConcurrentQueue<PyOrderRequest> outgoing_reqs;
std::mutex securities_mx;
std::vector<std::shared_ptr<bts::md::py_interop::PySecurity>> securities;
volatile bool is_securities_dirty = false;
ExecutionGateway *gw;
bool is_securities_dirty = false;
};
} // namespace python_execution
@@ -1,46 +1,18 @@
#include "python_interop/md/md_bridge_impl.h"
#include "python_interop/md/md_provider.h"
#include "python_interop/python_activator.h"
#include "spark/io/json_formatter.h"
#include "spark/logging/logging.h"
#include "spark/logging/trace_log.h"
#include "spark/utils/Delegate.h"
#include "spark/utils/stopwatch.h"
#include <pybind11/pybind11.h>
#include <thread>
using std::shared_ptr;
namespace py = pybind11;
namespace bts::md::py_interop {
using bts::core::Delegate;
using bts::io::json_formatter;
using bts::utils::Stopwatch;
using python_interop::PythonActivatorT;
using std::string;
using std::vector;
#define ASSERT_FN(test, fn, ...) \
bts_assert_msgf(test, "md_bridge::" fn ". " __VA_ARGS__)
#define LOG_FN(level, fn, ...) LOG(md, level, "md_bridge::" fn ". " __VA_ARGS__)
#define DIAG_FN(fn, ...) LOG_FN(bts::Diag, fn, __VA_ARGS__)
#define INFO_FN(fn, ...) LOG_FN(bts::Info, fn, __VA_ARGS__)
#define WARN_FN(fn, ...) LOG_FN(bts::Warn, fn, __VA_ARGS__)
#define ERROR_FN(fn, ...) LOG_FN(bts::Error, fn, __VA_ARGS__)
#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__)
#define INFO(...) _LOG(bts::Info, __VA_ARGS__)
#define ERROR(...) _LOG(bts::Error, __VA_ARGS__)
#define _LOG_TRACE(level, name) \
LOG_TRACE(md, level, std::string("md_bridge"), std::string(name))
#define INFO_TRACE(name) _LOG_TRACE(bts::Info, name)
#define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name)
#define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name)
#define DIAG_TRACE_PARAMS(name, params) \
LOG_TRACE_PARAMS( \
md, bts::Diag, std::string("md_bridge"), std::string(name), params)
@@ -49,13 +21,6 @@ using std::vector;
LOG_TRACE_PARAMS( \
md, bts::Info, std::string("md_bridge"), std::string(name), params)
#define WARN_TRACE_PARAMS(name, params) \
LOG_TRACE_PARAMS( \
md, bts::Warn, std::string("md_bridge"), std::string(name), params)
MdBridgeImpl::MdBridgeImpl(string target, vector<string> module_paths)
: target(std::move(target)), module_paths(std::move(module_paths)) {}
void MdBridgeImpl::on_securities(
const std::vector<std::shared_ptr<PySecurity>> &securities) {
INFO_TRACE_PARAMS("on_securities", securities.size());
@@ -63,7 +28,7 @@ void MdBridgeImpl::on_securities(
}
void MdBridgeImpl::on_books(const std::vector<std::shared_ptr<PyBook>> &books) {
DIAG_TRACE_PARAMS("on_books", json_formatter(books.size()));
INFO_TRACE_PARAMS("on_books", json_formatter(books.size()));
incoming_books.push_many(books.begin(), books.end());
}
@@ -103,26 +68,6 @@ bool MdBridgeImpl::pop_trade(shared_ptr<const PyTrade> &result) {
return false;
}
int MdBridgeImpl::run_py(volatile bool *keep_going) {
bts_assert(keep_going != nullptr);
PythonActivatorT<MdProvider> provider_activator{target, module_paths};
provider = provider_activator.get_target([&](MdProvider &pro) {
// This will start another thread in python, and call back into c++ from there.
auto sw = Stopwatch::start_new();
pro.start(this);
INFO_TRACE_PARAMS(
"run_py",
json_formatter(fmt::format(
"MdProvider.start took {:.2f}ms", sw.elapsed().total_millis())));
});
// Release gil to let python run
py::gil_scoped_release gil_rel;
while (*keep_going) {
std::this_thread::sleep_for(std::chrono::seconds(1));
}
return 0;
}
MdBridgeImpl::~MdBridgeImpl() = default;
} // namespace bts::md::py_interop
@@ -1,15 +1,13 @@
#pragma once
#include "python_interop/md/md_bridge.h"
#include "python_interop/md/md_provider.h"
#include "spark/collections/concurrent_queue.h"
namespace bts::md::py_interop {
class MdBridgeImpl : public MdBridge {
public:
MdBridgeImpl(std::string target, std::vector<std::string> module_paths);
~MdBridgeImpl();
~MdBridgeImpl() override;
// Python interface
void on_securities(
@@ -21,16 +19,12 @@ class MdBridgeImpl : public MdBridge {
bool pop_security(std::shared_ptr<const PySecurity> &result);
bool pop_book(std::shared_ptr<const PyBook> &result);
bool pop_trade(std::shared_ptr<const PyTrade> &result);
int run_py(volatile bool *keep_going);
private:
std::string target;
std::vector<std::string> module_paths;
collections::ConcurrentQueue<std::shared_ptr<const PySecurity>>
incoming_securities;
collections::ConcurrentQueue<std::shared_ptr<const PyBook>> incoming_books;
collections::ConcurrentQueue<std::shared_ptr<const PyTrade>> incoming_trades;
MdProvider *provider;
};
} // namespace bts::md::py_interop
@@ -0,0 +1,287 @@
#include "plugin_manager.h"
#include "python_interop/execution/execution_gateway.h"
#include "python_interop/execution/execution_gateway_bridge.h"
#include "python_interop/md/md_bridge.h"
#include "python_interop/md/md_provider.h"
#include "python_interop/plugin_manager_logs.h"
#include "python_interop/python_object.h"
#include "spark/collections/concurrent_queue.h"
#include "spark/collections/concurrent_unordered_map.h"
#include "spark/utils/stopwatch.h"
#include "spark/utils/string_utils.h"
#include <csignal>
#include <fmt/core.h>
#include <functional>
#include <future>
#include <memory>
#include <mutex>
#include <pybind11/embed.h>
#include <pybind11/gil.h>
#include <pybind11/pytypes.h>
#include <string>
#include <thread>
#include <utility>
namespace py = pybind11;
namespace python_interop {
using bts::collections::ConcurrentQueue;
using bts::collections::ConcurrentUnorderedMap;
using bts::md::py_interop::MdBridge;
using bts::utils::Stopwatch;
using python_execution::ExecutionGateway;
using python_execution::ExecutionGatewayBridge;
using std::function;
using std::future;
using std::mutex;
using std::optional;
using std::shared_ptr;
using std::string;
using std::thread;
static bool parse_target(
const std::string &target,
std::string *module_name,
std::string *type_name) {
auto parts = bts::system::string_utils::split(target, ".");
if (parts.empty()) {
return false;
}
if (parts.size() == 1) {
bts::system::string_utils::snake_to_pascal_case(
parts.emplace_back(parts[0]));
}
*module_name = parts[0];
*type_name = parts[1];
return !module_name->empty() && !type_name->empty();
}
struct __attribute__((visibility("hidden"))) PluginManager_SessionData {
py::scoped_interpreter interpreter{false};
std::unordered_map<std::string, std::unique_ptr<py::module>> modules;
py::module &import(const std::string &name) {
auto [it, is_new] = modules.try_emplace(name, nullptr);
if (is_new) {
it->second =
std::make_unique<py::module>(py::module::import(name.c_str()));
}
return *it->second;
}
};
struct __attribute__((visibility("hidden"))) PluginManager_Impl {
volatile bool keep_going;
mutex mx;
optional<thread> work_thread;
ConcurrentQueue<function<void()>> work_queue;
ConcurrentUnorderedMap<
string,
shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>
md_providers;
ConcurrentUnorderedMap<
string,
shared_ptr<PythonObjectT<python_execution::ExecutionGateway>>>
execution_gateways;
PluginManager_SessionData *session_data;
void start_work_thread() {
mx.lock();
if (work_thread) {
mx.unlock();
return;
}
keep_going = true;
mx.unlock();
work_thread.emplace([&]() {
PluginManager_SessionData sd;
mx.lock();
session_data = &sd;
mx.unlock();
try {
sd.import("sys").attr("path").attr("append")(
"python_interop_plugins");
sd.import("spark_core").attr("configure_default_logger")();
sd.import("bybit_md_provider");
sd.import("bybit_execution_gateway");
py::gil_scoped_release gil_rel;
Stopwatch sw;
int n = 0;
while (keep_going) {
auto work_req = work_queue.wait_until_pop();
if (!work_req) {
continue;
}
LOG_INVOKING(n);
sw.restart();
{
py::gil_scoped_acquire gil_acquire;
(*work_req)();
}
LOG_INVOKED(n, sw.elapsed());
n++;
}
} catch (const std::exception &ex) {
bts_assert_msgf(
false, "PythonWorker work_thread threw. {}",
std::string(ex.what()));
return;
} catch (...) {
bts_assert_msg(false, "PythonWorker work_thread threw unknown ex.");
return;
}
mx.lock();
session_data = nullptr;
mx.unlock();
});
}
void stop_work_thread() {
mx.lock();
if (work_thread && work_thread->joinable()) {
keep_going = false;
mx.unlock();
work_thread->join();
work_thread.reset();
} else {
mx.unlock();
}
}
future<shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>
start_md_provider(
string provider_name, string target, shared_ptr<MdBridge> bridge) {
auto prom = std::make_shared<std::promise<
shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>>();
work_queue.push([this, target = std::move(target),
provider_name = std::move(provider_name),
bridge = std::move(bridge), prom = prom]() {
string module_name;
string type_name;
if (!parse_target(target, &module_name, &type_name)) {
LOG_WORK_ERROR(
md,
fmt::format("Failed to parse module/type from '{}'", target));
return;
}
py::object obj = session_data->import(module_name.c_str())
.attr(type_name.c_str())();
auto [md_pro_it, md_pro_is_new] = md_providers.try_emplace(
provider_name,
std::make_shared<PythonObjectT<bts::md::py_interop::MdProvider>>(
std::move(obj)));
bts_assert(md_pro_is_new);
LOG_STARTING("md_provider", provider_name);
Stopwatch sw = Stopwatch::start_new();
shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>> md_pro =
md_pro_it->second;
(*md_pro)->start(bridge.get());
LOG_STARTED("md_provider", provider_name, sw.elapsed());
prom->set_value(md_pro);
// Don't let python handle the signals (otherwise it'll hang on ctrl+c)
std::signal(SIGINT, SIG_DFL);
});
return prom->get_future();
}
void stop_md_provider(const std::string &provider_name) {
auto it = md_providers.find(provider_name);
if (it != md_providers.end()) {
(*it->second)->stop();
md_providers.erase(it);
}
}
future<shared_ptr<PythonObjectT<ExecutionGateway>>> start_execution_gateway(
string gateway_name,
string target,
shared_ptr<ExecutionGatewayBridge> bridge) {
auto prom = std::make_shared<
std::promise<shared_ptr<PythonObjectT<ExecutionGateway>>>>();
work_queue.push([this, target = std::move(target),
gateway_name = std::move(gateway_name),
bridge = std::move(bridge), prom = prom]() {
string module_name;
string type_name;
if (!parse_target(target, &module_name, &type_name)) {
LOG_WORK_ERROR(
OrderMan,
fmt::format("Failed to parse module/type from '{}'", target));
return;
}
py::object obj = session_data->import(module_name.c_str())
.attr(type_name.c_str())();
auto [exe_gw_it, exe_gw_is_new] = execution_gateways.try_emplace(
gateway_name,
std::make_shared<PythonObjectT<ExecutionGateway>>(std::move(obj)));
bts_assert(exe_gw_is_new);
LOG_STARTING("execution_gateway", gateway_name);
Stopwatch sw = Stopwatch::start_new();
shared_ptr<PythonObjectT<ExecutionGateway>> exe_gw = exe_gw_it->second;
(*exe_gw)->start(bridge.get());
LOG_STARTED("execution_gateway", gateway_name, sw.elapsed());
prom->set_value(exe_gw);
// Don't let python handle the signals (otherwise it'll hang on ctrl+c)
std::signal(SIGINT, SIG_DFL);
});
return prom->get_future();
}
void stop_execution_gateway(const std::string &provider_name) {
auto it = execution_gateways.find(provider_name);
if (it != execution_gateways.end()) {
(*it->second)->stop();
execution_gateways.erase(it);
}
}
};
PluginManager::PluginManager() : me_(new PluginManager_Impl()) {}
PluginManager::~PluginManager() {
stop_work_thread();
delete static_cast<PluginManager_Impl *>(me_);
me_ = nullptr;
}
void PluginManager::start_work_thread() {
static_cast<PluginManager_Impl *>(me_)->start_work_thread();
}
void PluginManager::stop_work_thread() {
static_cast<PluginManager_Impl *>(me_)->stop_work_thread();
}
void PluginManager::invoke(std::function<void()> callback) {
auto *me = static_cast<PluginManager_Impl *>(me_);
me->work_queue.push(callback);
}
std::future<shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>
PluginManager::start_md_provider(
string provider_name, string target, shared_ptr<MdBridge> bridge) {
return static_cast<PluginManager_Impl *>(me_)->start_md_provider(
std::move(provider_name), std::move(target), std::move(bridge));
}
void PluginManager::stop_md_provider(const std::string &provider_name) {
static_cast<PluginManager_Impl *>(me_)->stop_md_provider(provider_name);
}
std::future<shared_ptr<PythonObjectT<ExecutionGateway>>>
PluginManager::start_execution_gateway(
string gateway_name,
string target,
shared_ptr<ExecutionGatewayBridge> bridge) {
return static_cast<PluginManager_Impl *>(me_)->start_execution_gateway(
std::move(gateway_name), std::move(target), std::move(bridge));
}
void PluginManager::stop_execution_gateway(const std::string &provider_name) {
return static_cast<PluginManager_Impl *>(me_)->stop_execution_gateway(
provider_name);
}
} // namespace python_interop
@@ -0,0 +1,45 @@
#pragma once
#include "python_interop/execution/execution_gateway.h"
#include "python_interop/execution/execution_gateway_bridge.h"
#include "python_interop/md/md_bridge.h"
#include "python_interop/md/md_provider.h"
#include "python_interop/python_object.h"
#include <functional>
#include <future>
#include <memory>
#include <pybind11/pytypes.h>
#include <string>
namespace python_interop {
class PluginManager {
void *me_;
public:
PluginManager();
~PluginManager();
void start_work_thread();
void stop_work_thread();
std::future<std::shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>
start_md_provider(
std::string provider_name,
std::string target,
std::shared_ptr<bts::md::py_interop::MdBridge> bridge);
void stop_md_provider(const std::string &provider_name);
std::future<
std::shared_ptr<PythonObjectT<python_execution::ExecutionGateway>>>
start_execution_gateway(
std::string gateway_name,
std::string target,
std::shared_ptr<python_execution::ExecutionGatewayBridge> bridge);
void stop_execution_gateway(const std::string &gateway_name);
void invoke(std::function<void()> callback);
};
} // namespace python_interop
@@ -0,0 +1,116 @@
#pragma once
#include "spark/core/FMT_IMPL.h"
#include "spark/core/TimeSpan.h"
#include "spark/logging/trace_log.h"
#include "spark/os/os.h"
#include "spark/utils/thread_id.h"
#include <fmt/core.h>
#include <string>
namespace python_interop::logs {
struct StartingLog {
std::string instance_name;
std::string thread_name;
std::int64_t thread_id;
StartingLog(std::string instance_name)
: instance_name(std::move(instance_name)),
thread_name(bts::os::get_thread_name()),
thread_id(BTS_THIS_THREAD_ID) {}
};
struct StartedLog {
std::string instance_name;
std::string thread_name;
std::int64_t thread_id;
bts::core::TimeSpan duration;
StartedLog(std::string instance_name, bts::core::TimeSpan duration)
: instance_name(std::move(instance_name)),
thread_name(bts::os::get_thread_name()), thread_id(BTS_THIS_THREAD_ID),
duration(duration) {}
};
struct InvokingLog {
int count;
std::string thread_name;
std::int64_t thread_id;
InvokingLog(int count)
: count(count), thread_name(bts::os::get_thread_name()),
thread_id(BTS_THIS_THREAD_ID) {}
};
struct InvokedLog {
int count;
bts::core::TimeSpan duration;
std::string thread_name;
std::int64_t thread_id;
InvokedLog(int count, bts::core::TimeSpan dequeue_duration)
: count(count), duration(dequeue_duration),
thread_name(bts::os::get_thread_name()),
thread_id(BTS_THIS_THREAD_ID) {}
};
} // namespace python_interop::logs
FMT_IMPL(
python_interop::logs::StartingLog,
l,
R"({{ "name": "{}", "ev": "STARTING", "tn": "{}", "tid": "{}" }})",
l.instance_name,
l.thread_name,
l.thread_id);
FMT_IMPL(
python_interop::logs::StartedLog,
l,
R"({{ "name": "{}", "ev": "STARTED", "dur": "{:.3f}ms", "tn": "{}", "tid": "{}" }})",
l.instance_name,
l.duration.total_millis(),
l.thread_name,
l.thread_id);
FMT_IMPL(
python_interop::logs::InvokingLog,
l,
R"({{ "ev": "INVOKING", "count": {}, "tn": "{}", "tid": "{}" }})",
l.count,
l.thread_name,
l.thread_id);
FMT_IMPL(
python_interop::logs::InvokedLog,
l,
R"({{ "ev": "INVOKED", "count": {}, "dur": "{:.3f}ms", "tn": "{}", "tid": "{}" }})",
l.count,
l.duration.total_millis(),
l.thread_name,
l.thread_id);
#define LOG_STARTING(type_name, instance_name) \
LOG_TRACE_PARAMS( \
Gen, bts::Info, string("py_plugin"), string("start_" type_name), \
python_interop::logs::StartingLog{instance_name})
#define LOG_STARTED(type_name, instance_name, dur) \
LOG_TRACE_PARAMS( \
Gen, bts::Info, string("py_plugin"), string("start_" type_name), \
python_interop::logs::StartedLog(instance_name, dur))
#define LOG_INVOKING(count) \
LOG_TRACE_PARAMS( \
Gen, bts::Info, string("py_plugin"), string("work"), \
python_interop::logs::InvokingLog(count))
#define LOG_INVOKED(count, duration) \
LOG_TRACE_PARAMS( \
Gen, bts::Info, string("py_plugin"), string("work"), \
python_interop::logs::InvokedLog(count, duration))
#define LOG_WORK_ERROR(subsystem, params) \
LOG_TRACE_PARAMS(subsystem, bts::Error, string("py_plugin"), string("work"), params)
@@ -1,93 +0,0 @@
#pragma once
#include "python_interop/python_activator_impl.h"
#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
#include <string>
#include <vector>
namespace python_interop {
class PythonActivator {
public:
PythonActivator(std::string target, std::vector<std::string> module_paths)
: me_(new PythonActivator_Impl(
std::move(target), std::move(module_paths))) {}
PythonActivator() {
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
delete me;
}
pybind11::object &get_target_obj() {
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
return me->get_target_obj();
}
pybind11::object &get_target_obj(
std::function<void(pybind11::object &)> init) {
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
return me->get_target_obj(init);
}
template <typename... Args>
pybind11::object &get_target_obj(
std::function<void(pybind11::object &)> init, Args &&...args) {
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
return me->get_target_obj<Args...>(init, std::forward<Args>(args)...);
}
void reset() {
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
me->reset();
}
private:
void *me_;
};
template <typename Target>
class PythonActivatorT {
public:
PythonActivatorT(std::string target, std::vector<std::string> module_paths)
: activator_(std::move(target), std::move(module_paths)) {}
pybind11::object &get_target_obj() { return activator_.get_target_obj(); }
Target *get_target() {
if (target_ == nullptr) {
target_ = activator_.get_target_obj().cast<Target *>();
}
return target_;
}
Target *get_target(std::function<void(Target &)> init) {
if (target_ == nullptr) {
activator_.get_target_obj([&](pybind11::object &obj) {
target_ = obj.cast<Target *>();
init(*target_);
});
}
return target_;
}
template <typename... Args>
Target *get_target(std::function<void(Target &)> init, Args &&...args) {
if (target_ == nullptr) {
activator_.get_target_obj<Args...>(
[&](pybind11::object &obj) {
target_ = obj.cast<Target *>();
init(*target_);
},
std::forward<Args>(args)...);
}
return target_;
}
private:
PythonActivator activator_;
Target *target_ = nullptr;
};
} // namespace python_interop
@@ -1,118 +0,0 @@
#pragma once
#include "python_interop/python_interpreter.h"
#include "spark/core/TimeStamp.h"
#include "spark/io/json_formatter.h"
#include "spark/logging/logging.h"
#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
#include <string>
#include <vector>
namespace python_interop {
struct PythonActivator_Impl {
std::string target;
std::vector<std::string> module_paths;
std::optional<pybind11::object> target_obj;
PythonActivator_Impl(
std::string target, std::vector<std::string> module_paths)
: target(std::move(target)), module_paths(std::move(module_paths)) {}
~PythonActivator_Impl() { reset(); }
pybind11::object &get_target_obj() {
return get_target_obj([](pybind11::object &) {});
}
template <typename... Args>
pybind11::object &get_target_obj(
std::function<void(pybind11::object &)> init, Args &&...args) {
if (!target_obj) {
// Prevent python from overwriting our signal handler (so ctrl+c
// actually terminates the process).
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
target_obj = create_target_with_args<Args...>(
init, std::forward<Args>(args)...);
std::signal(SIGINT, orig_signal_handler);
}
return *target_obj;
}
// Undef after use (this is a header)
#define LOG_EVENT(ev) \
LOG(OrderMan, bts::Info, \
R"([py_activator.json]{{ "tgt": "{}", "path": {}, "ev": )" ev \
R"(, "ts": {} }})", \
target, bts::io::json_formatter(module_paths), \
bts::core::TimeStamp::now_ns())
#define LOG_ERROR(err) \
LOG(OrderMan, bts::Info, \
R"([py_activator.json]{{ "tgt": "{}", "path": {}, "ev": "ERROR", "err": "{}", "ts": {} }})", \
target, bts::io::json_formatter(module_paths), err, \
bts::core::TimeStamp::now_ns())
#define LOG_CREATE LOG_EVENT("CREATE")
#define LOG_RESET LOG_EVENT("RESET")
void reset() {
LOG_RESET;
target_obj.reset();
}
private:
template <typename... Args>
pybind11::object create_target_with_args(
std::function<void(pybind11::object &)> init, Args &&...args) {
LOG_CREATE;
return PythonInterpreter::exec_t<pybind11::object>(
[this, &init,
args_tuple =
std::make_tuple(std::forward<Args>(args)...)]() mutable {
std::string target_module_name;
std::string target_type_name;
PythonInterpreter::parse_target(
target, &target_module_name, &target_type_name);
pybind11::object obj;
try {
// Prepare module path
pybind11::module sys = pybind11::module::import("sys");
for (const auto &module_path : module_paths) {
sys.attr("path").attr("append")(module_path.c_str());
}
// Import target module
pybind11::module target_module =
pybind11::module::import(target_module_name.c_str());
// Instantiate target using tuple expansion
obj = std::apply(
[&](auto &&...unpacked_args) {
return target_module.attr(target_type_name.c_str())(
std::forward<decltype(unpacked_args)>(
unpacked_args)...);
},
std::move(args_tuple));
init(obj);
} catch (const std::exception &ex) {
LOG_ERROR(std::string{ex.what()});
}
return obj;
});
}
pybind11::object create_target() {
return create_target_with_args([](pybind11::object &) {});
}
pybind11::object create_target(
std::function<void(pybind11::object &)> init) {
return create_target_with_args(init);
}
#undef LOG_EVENT
#undef LOG_ERROR
#undef LOG_CREATE
#undef LOG_RESET
};
} // namespace python_interop
@@ -1,50 +0,0 @@
#include "python_interpreter.h"
#include "spark/logging/logging.h"
#include "spark/utils/string_utils.h"
#include <csignal>
#include <mutex>
#include <pybind11/embed.h>
#include <pybind11/gil.h>
namespace python_interop {
using bts::system::string_utils::snake_to_pascal_case;
using bts::system::string_utils::split;
std::mutex PythonInterpreter::mx;
std::optional<pybind11::scoped_interpreter> PythonInterpreter::interpreter;
void PythonInterpreter::init_interpreter() {
if (interpreter.has_value()) {
return;
}
static std::mutex local_mx;
std::lock_guard<std::mutex> lock{local_mx};
if (interpreter.has_value()) {
return;
}
LOG(Gen, bts::Info, "Creating global py interpreter");
// Don't let python handle the signals (otherwise it'll hang on ctrl+c)
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
interpreter.emplace(false);
std::signal(SIGINT, orig_signal_handler);
}
bool PythonInterpreter::parse_target(
const std::string &target,
std::string *module_name,
std::string *type_name) {
auto parts = split(target, ".");
if (parts.empty()) {
return false;
}
if (parts.size() == 1) {
snake_to_pascal_case(parts.emplace_back(parts[0]));
}
*module_name = parts[0];
*type_name = parts[1];
return !module_name->empty() && !type_name->empty();
}
} // namespace python_interop
@@ -1,125 +0,0 @@
#pragma once
#include "spark/utils/bts_assert.h"
#include "spark/utils/string_utils.h"
#include <mutex>
#include <pybind11/embed.h>
#include <pybind11/gil.h>
#include <pybind11/pybind11.h>
#include <string>
namespace python_interop {
class PythonInterpreter {
static std::mutex mx;
static std::optional<pybind11::scoped_interpreter> interpreter;
static void init_interpreter();
public:
/**
* Parses module and type name from a \p target arg. The following formats are supported:
* 1) <module_name>.<type_name>
* 2) <module_name>
* In this case, the type name defaults to the PascalCase equivalent of the module name.
* For example, the module "foo_bar" would have type name "FooBar".
*/
static bool parse_target(
const std::string &target,
std::string *module_name,
std::string *type_name);
static void exec(std::function<void()> fn) {
init_interpreter();
std::lock_guard<std::mutex> lock{mx};
fn();
}
template <typename T>
static T exec_t(std::function<T()> fn) {
init_interpreter();
std::lock_guard<std::mutex> lock{mx};
pybind11::gil_scoped_acquire gil;
T rv = fn();
pybind11::gil_scoped_release gil_rel;
return rv;
}
template <typename... Args>
static pybind11::object create_object(
const std::vector<std::string> &module_paths,
const std::string &target_name,
Args &&...args) {
std::string target_module_name;
std::string target_type_name;
bts_assert(
parse_target(target_name, &target_module_name, &target_type_name));
return create_object(
module_paths, target_module_name, target_type_name,
std::forward<Args>(args)...);
}
template <typename... Args>
static pybind11::object create_object(
const std::vector<std::string> &module_paths,
const std::string &target_type_name,
const std::string &target_module_name,
Args &&...args) {
namespace py = pybind11;
return exec_t<py::object>([&]() {
// Prepare module path
py::module sys = py::module::import("sys");
for (const auto &module_path : module_paths) {
sys.attr("path").attr("append")(module_path.c_str());
}
// Import target module
py::module target_module =
py::module::import(target_module_name.c_str());
// Instantiate target
return target_module.attr(target_type_name.c_str())(
std::forward<Args>(args)...);
});
}
template <typename... Args>
static pybind11::object create_object_with_init(
const std::vector<std::string> &module_paths,
const std::string &target_name,
std::function<void(pybind11::object &)> init_fn,
Args &&...args) {
std::string target_module_name;
std::string target_type_name;
bts_assert(
parse_target(target_name, &target_module_name, &target_type_name));
return create_object_with_init(
module_paths, target_module_name, target_type_name, init_fn,
std::forward<Args>(args)...);
}
template <typename... Args>
static pybind11::object create_object_with_init(
const std::vector<std::string> &module_paths,
const std::string &target_type_name,
const std::string &target_module_name,
std::function<void(pybind11::object &)> init_fn,
Args &&...args) {
namespace py = pybind11;
return exec_t<py::object>([&]() {
// Prepare module path
py::module sys = py::module::import("sys");
for (const auto &module_path : module_paths) {
sys.attr("path").attr("append")(module_path.c_str());
}
// Import target module
py::module target_module =
py::module::import(target_module_name.c_str());
// Instantiate target
py::object obj = target_module.attr(target_type_name.c_str())(
std::forward<Args>(args)...);
init_fn(obj);
return obj;
});
}
};
} // namespace python_interop
@@ -0,0 +1,45 @@
#include "python_object.h"
#include "python_interop/execution/execution_gateway.h"
#include "python_interop/md/md_provider.h"
namespace python_interop {
template <typename T>
struct __attribute__((visibility("hidden"))) PythonObjectT_Impl {
pybind11::object obj_;
T *tp_ = nullptr;
PythonObjectT_Impl(pybind11::object &&obj)
: obj_(std::move(obj)), tp_(obj_.cast<T *>()) {}
T &operator*();
T *operator->();
};
template <typename T>
PythonObjectT<T>::PythonObjectT(pybind11::object &&obj)
: me(new PythonObjectT_Impl<T>(std::move(obj))) {}
template <typename T>
PythonObjectT<T>::~PythonObjectT() {
delete static_cast<PythonObjectT_Impl<T> *>(me);
}
template <typename T>
T &PythonObjectT<T>::operator*() {
return *static_cast<PythonObjectT_Impl<T> *>(me)->tp_;
}
template <typename T>
T *PythonObjectT<T>::operator->() {
return static_cast<PythonObjectT_Impl<T> *>(me)->tp_;
}
#define PYTHON_OBJECT_DEF(T) \
template struct PythonObjectT_Impl<T>; \
template class PythonObjectT<T>;
PYTHON_OBJECT_DEF(bts::md::py_interop::MdProvider);
PYTHON_OBJECT_DEF(python_execution::ExecutionGateway);
} // namespace python_interop
@@ -0,0 +1,19 @@
#pragma once
#include <pybind11/pytypes.h>
namespace python_interop {
template <typename T>
class PythonObjectT {
void *me;
public:
__attribute__((visibility("hidden"))) PythonObjectT(pybind11::object &&obj);
~PythonObjectT();
T &operator*();
T *operator->();
};
} // namespace python_interop