refactor: Manage all py plugins (md and exe) in one class
All calls to/from python are performed on one thread via a simple message queue (necessary because of the global interpreter lock).
This commit is contained in:
1 parent
c7db105533
commit
8c23665d65
29 files changed
+855
-602
No files matched your search
@@ -3,31 +3,25 @@
|
||||
#include "python_interop/execution/py_cancel_order_request.h"
|
||||
#include "python_interop/execution/py_create_order_request.h"
|
||||
#include "python_interop/execution/py_position_info.h"
|
||||
#include "python_interop/python_activator.h"
|
||||
#include "python_interop/python_interpreter.h"
|
||||
#include "python_interop/python_object.h"
|
||||
#include "spark/core/TimeSpan.h"
|
||||
#include "spark/domain_model/Venue.h"
|
||||
#include "spark/io/json_formatter.h"
|
||||
#include "spark/logging/logging.h"
|
||||
#include "spark/logging/trace_log.h"
|
||||
#include "spark/logging/trace_log_var.h"
|
||||
#include "spark/orderman/OrderMsg.h"
|
||||
#include "spark/orderman/log_messages/execution_report_log.h"
|
||||
#include "spark/orderman/orderman_utils/ExecutionReport.h"
|
||||
#include "spark/utils/overloaded.h"
|
||||
#include "spark/utils/stopwatch.h"
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <future>
|
||||
#include <memory>
|
||||
|
||||
using std::shared_ptr;
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace python_execution {
|
||||
|
||||
using bts::core::Delegate;
|
||||
using bts::core::TimeSpan;
|
||||
using bts::io::json_formatter;
|
||||
using bts::orderman::RFQMsg;
|
||||
using python_interop::PythonActivatorT;
|
||||
using std::string;
|
||||
using std::vector;
|
||||
|
||||
@@ -60,10 +54,8 @@ using std::vector;
|
||||
OrderMan, bts::Warn, std::string("exe_gw_bridge"), std::string(name), \
|
||||
params)
|
||||
|
||||
ExecutionGatewayBridgeImpl::ExecutionGatewayBridgeImpl(
|
||||
string name, string target, vector<string> module_paths)
|
||||
: name(std::move(name)), target(std::move(target)),
|
||||
module_paths(std::move(module_paths)) {}
|
||||
ExecutionGatewayBridgeImpl::ExecutionGatewayBridgeImpl(string name)
|
||||
: name(std::move(name)) {}
|
||||
|
||||
void ExecutionGatewayBridgeImpl::on_order_msg(PyOrderRequest msg) {
|
||||
INFO_TRACE("on_order_msg");
|
||||
@@ -106,11 +98,11 @@ bool ExecutionGatewayBridgeImpl::pop_exe_report(
|
||||
void ExecutionGatewayBridgeImpl::on_securities(
|
||||
std::vector<std::shared_ptr<bts::md::py_interop::PySecurity>> securities) {
|
||||
INFO_TRACE_PARAMS(
|
||||
"on_securities",
|
||||
json_formatter(
|
||||
fmt::format("Received {} securities", this->securities.size())));
|
||||
"on_securities", json_formatter(fmt::format(
|
||||
"Received {} securities", securities.size())));
|
||||
std::lock_guard<std::mutex> lock{securities_mx};
|
||||
this->securities = std::move(securities);
|
||||
this->is_securities_dirty = true;
|
||||
is_securities_dirty = true;
|
||||
}
|
||||
|
||||
shared_ptr<const PyExecutionReport>
|
||||
@@ -119,12 +111,10 @@ ExecutionGatewayBridgeImpl::pop_exe_report() {
|
||||
return res ? *res : nullptr;
|
||||
}
|
||||
|
||||
int ExecutionGatewayBridgeImpl::run_py(volatile bool *keep_going) {
|
||||
INFO_TRACE_PARAMS("run_py", json_formatter("Begin"));
|
||||
bts_assert(keep_going != nullptr);
|
||||
PythonActivatorT<ExecutionGateway> gw_activator{target, module_paths};
|
||||
gw = gw_activator.get_target(
|
||||
[&](ExecutionGateway &local_gw) { local_gw.start(this); }, name);
|
||||
int ExecutionGatewayBridgeImpl::run_py(
|
||||
volatile bool *keep_going,
|
||||
std::shared_ptr<python_interop::PythonObjectT<ExecutionGateway>> gw_ptr,
|
||||
std::function<void(std::function<void()>)> py_invoke) {
|
||||
INFO_TRACE_PARAMS("run_py", json_formatter("Waiting for ready..."));
|
||||
auto sw = bts::utils::Stopwatch::start_new();
|
||||
while (!wait_for_ready(TimeSpan::from_seconds(1)) && *keep_going) {
|
||||
@@ -133,57 +123,100 @@ int ExecutionGatewayBridgeImpl::run_py(volatile bool *keep_going) {
|
||||
"PyExecutionGateway still not ready after {:.2f}s",
|
||||
sw.total_elapsed().total_seconds())));
|
||||
}
|
||||
INFO_TRACE_PARAMS(
|
||||
"run", json_formatter(fmt::format(
|
||||
"PyExecutionGateway ready after {:.2f}s",
|
||||
sw.total_elapsed().total_seconds())));
|
||||
ExecutionGateway &gw = **gw_ptr;
|
||||
std::vector<std::shared_ptr<bts::md::py_interop::PySecurity>> new_securities;
|
||||
while (is_ready.value() && *keep_going) {
|
||||
INFO_TRACE_PARAMS("run_py", json_formatter("Waiting for request..."));
|
||||
if (is_securities_dirty) {
|
||||
bool was_securities_dirty = false;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock{securities_mx};
|
||||
if (is_securities_dirty) {
|
||||
was_securities_dirty = true;
|
||||
new_securities = std::move(securities);
|
||||
is_securities_dirty = false;
|
||||
}
|
||||
}
|
||||
if (was_securities_dirty) {
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py", json_formatter(std::map<string, string>{
|
||||
{"name", name}, {"ev", "SENDING_SECS_TO_PY"}}));
|
||||
std::promise<void> prom;
|
||||
sw.restart();
|
||||
py_invoke([&]() {
|
||||
try {
|
||||
gw.on_securities(new_securities);
|
||||
} catch (const std::exception &ex) {
|
||||
bts_assert_msgf(
|
||||
false, "py exe gw threw on secs. {}",
|
||||
std::string(ex.what()));
|
||||
} catch (...) {
|
||||
bts_assert_msg(false, "py exe gw threw on secs. UNKNOWN");
|
||||
}
|
||||
prom.set_value();
|
||||
});
|
||||
prom.get_future().wait();
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py",
|
||||
json_formatter(fmt::format(
|
||||
"Sending {} securities to py gw", securities.size())));
|
||||
python_interop::PythonInterpreter::exec(
|
||||
[&]() { gw->on_securities(securities); });
|
||||
is_securities_dirty = false;
|
||||
json_formatter(std::map<string, string>{
|
||||
{"name", name},
|
||||
{"ev", "SENT_SECS_TO_PY"},
|
||||
{"dur",
|
||||
fmt::format("{:.3f}ms", sw.elapsed().total_millis())}}));
|
||||
}
|
||||
auto req = outgoing_reqs.wait_for_until_pop(
|
||||
bts::core::TimeSpan::from_seconds(1));
|
||||
if (!req) {
|
||||
continue;
|
||||
}
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py", json_formatter("Forwarding request to py gateway"));
|
||||
enum ReqType { CREATE = 0, CANCEL_REPLACE = 1, CANCEL = 2 };
|
||||
static const std::string req_type_names[]{"CREATE", "CXL_REP", "CXL"};
|
||||
ReqType req_type;
|
||||
sw.restart();
|
||||
python_interop::PythonInterpreter::exec([&]() {
|
||||
std::visit(
|
||||
bts::overloaded{
|
||||
[&](std::shared_ptr<PyCreateOrderRequest> req) {
|
||||
req_type = ReqType::CREATE;
|
||||
gw->create_order(std::move(req));
|
||||
},
|
||||
[&](std::shared_ptr<PyCancelReplaceOrderRequest> req) {
|
||||
req_type = ReqType::CANCEL_REPLACE;
|
||||
gw->cancel_replace_order(std::move(req));
|
||||
},
|
||||
[&](std::shared_ptr<PyCancelOrderRequest> req) {
|
||||
req_type = ReqType::CANCEL;
|
||||
gw->cancel_order(std::move(req));
|
||||
},
|
||||
[&](const RFQMsg &) {
|
||||
bts_assert_msg(false, "RFQ not implemented");
|
||||
},
|
||||
},
|
||||
*req);
|
||||
});
|
||||
sw.stop();
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py",
|
||||
fmt::format(
|
||||
R"({{ "typ": "{}", "hndl_dur": "{:.3f}ms" }})",
|
||||
req_type_names[req_type], sw.total_elapsed().total_millis()));
|
||||
{
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py", json_formatter(std::map<string, string>{
|
||||
{"name", name}, {"ev", "SENDING_REQ_TO_PY"}}));
|
||||
std::promise<void> prom;
|
||||
sw.restart();
|
||||
py_invoke([&, req = std::move(req)]() {
|
||||
try {
|
||||
std::visit(
|
||||
bts::overloaded{
|
||||
[&](std::shared_ptr<PyCreateOrderRequest> req) {
|
||||
req_type = ReqType::CREATE;
|
||||
gw.create_order(std::move(req));
|
||||
},
|
||||
[&](std::shared_ptr<PyCancelReplaceOrderRequest> req) {
|
||||
req_type = ReqType::CANCEL_REPLACE;
|
||||
gw.cancel_replace_order(std::move(req));
|
||||
},
|
||||
[&](std::shared_ptr<PyCancelOrderRequest> req) {
|
||||
req_type = ReqType::CANCEL;
|
||||
gw.cancel_order(std::move(req));
|
||||
}},
|
||||
*req);
|
||||
} catch (const std::exception &ex) {
|
||||
bts_assert_msgf(
|
||||
false, "py exe gw threw on req. {}", std::string(ex.what()));
|
||||
} catch (...) {
|
||||
bts_assert_msg(false, "py exe gw threw on req. UNKNOWN");
|
||||
}
|
||||
prom.set_value();
|
||||
});
|
||||
prom.get_future().wait();
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py",
|
||||
json_formatter(std::map<string, string>{
|
||||
{"name", name},
|
||||
{"ev", "SENT_REQ_TO_PY"},
|
||||
{"typ", req_type_names[req_type]},
|
||||
{"dur",
|
||||
fmt::format("{:.3f}ms", sw.elapsed().total_millis())}}));
|
||||
}
|
||||
}
|
||||
python_interop::PythonInterpreter::exec([&]() { gw->stop(); });
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#include "python_interop/execution/py_order_request.h"
|
||||
#include "python_interop/execution/py_position_info.h"
|
||||
#include "python_interop/py_security.h"
|
||||
#include "python_interop/python_object.h"
|
||||
#include "spark/collections/concurrent_queue.h"
|
||||
#include "spark/utils/Delegate.h"
|
||||
#include "spark/utils/concurrent_value_store.h"
|
||||
@@ -14,10 +15,7 @@ namespace python_execution {
|
||||
|
||||
class ExecutionGatewayBridgeImpl : public ExecutionGatewayBridge {
|
||||
public:
|
||||
ExecutionGatewayBridgeImpl(
|
||||
std::string name,
|
||||
std::string target,
|
||||
std::vector<std::string> module_paths);
|
||||
ExecutionGatewayBridgeImpl(std::string name);
|
||||
~ExecutionGatewayBridgeImpl() override;
|
||||
|
||||
// Python interface
|
||||
@@ -35,21 +33,22 @@ class ExecutionGatewayBridgeImpl : public ExecutionGatewayBridge {
|
||||
bool wait_for_ready(bts::core::TimeSpan timeout);
|
||||
bool pop_exe_report(std::shared_ptr<const PyExecutionReport> &result);
|
||||
std::shared_ptr<const PyExecutionReport> pop_exe_report();
|
||||
int run_py(volatile bool *keep_going);
|
||||
int run_py(
|
||||
volatile bool *keep_going,
|
||||
std::shared_ptr<python_interop::PythonObjectT<ExecutionGateway>> gw,
|
||||
std::function<void(std::function<void()>)> py_invoke);
|
||||
|
||||
private:
|
||||
std::string name;
|
||||
std::string target;
|
||||
std::vector<std::string> module_paths;
|
||||
bts::utils::ConcurrentValueStore<bool> is_ready;
|
||||
bts::collections::ConcurrentQueue<std::shared_ptr<PyExecutionReport>>
|
||||
incoming_exe_reports;
|
||||
bts::collections::ConcurrentQueue<std::shared_ptr<PyPositionInfo>>
|
||||
incoming_pos_infos;
|
||||
bts::collections::ConcurrentQueue<PyOrderRequest> outgoing_reqs;
|
||||
std::mutex securities_mx;
|
||||
std::vector<std::shared_ptr<bts::md::py_interop::PySecurity>> securities;
|
||||
volatile bool is_securities_dirty = false;
|
||||
ExecutionGateway *gw;
|
||||
bool is_securities_dirty = false;
|
||||
};
|
||||
|
||||
} // namespace python_execution
|
||||
@@ -1,46 +1,18 @@
|
||||
#include "python_interop/md/md_bridge_impl.h"
|
||||
#include "python_interop/md/md_provider.h"
|
||||
#include "python_interop/python_activator.h"
|
||||
#include "spark/io/json_formatter.h"
|
||||
#include "spark/logging/logging.h"
|
||||
#include "spark/logging/trace_log.h"
|
||||
#include "spark/utils/Delegate.h"
|
||||
#include "spark/utils/stopwatch.h"
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <thread>
|
||||
|
||||
using std::shared_ptr;
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace bts::md::py_interop {
|
||||
|
||||
using bts::core::Delegate;
|
||||
using bts::io::json_formatter;
|
||||
using bts::utils::Stopwatch;
|
||||
using python_interop::PythonActivatorT;
|
||||
using std::string;
|
||||
using std::vector;
|
||||
|
||||
#define ASSERT_FN(test, fn, ...) \
|
||||
bts_assert_msgf(test, "md_bridge::" fn ". " __VA_ARGS__)
|
||||
#define LOG_FN(level, fn, ...) LOG(md, level, "md_bridge::" fn ". " __VA_ARGS__)
|
||||
#define DIAG_FN(fn, ...) LOG_FN(bts::Diag, fn, __VA_ARGS__)
|
||||
#define INFO_FN(fn, ...) LOG_FN(bts::Info, fn, __VA_ARGS__)
|
||||
#define WARN_FN(fn, ...) LOG_FN(bts::Warn, fn, __VA_ARGS__)
|
||||
#define ERROR_FN(fn, ...) LOG_FN(bts::Error, fn, __VA_ARGS__)
|
||||
|
||||
#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__)
|
||||
#define INFO(...) _LOG(bts::Info, __VA_ARGS__)
|
||||
#define ERROR(...) _LOG(bts::Error, __VA_ARGS__)
|
||||
|
||||
#define _LOG_TRACE(level, name) \
|
||||
LOG_TRACE(md, level, std::string("md_bridge"), std::string(name))
|
||||
#define INFO_TRACE(name) _LOG_TRACE(bts::Info, name)
|
||||
#define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name)
|
||||
#define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name)
|
||||
|
||||
#define DIAG_TRACE_PARAMS(name, params) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
md, bts::Diag, std::string("md_bridge"), std::string(name), params)
|
||||
@@ -49,13 +21,6 @@ using std::vector;
|
||||
LOG_TRACE_PARAMS( \
|
||||
md, bts::Info, std::string("md_bridge"), std::string(name), params)
|
||||
|
||||
#define WARN_TRACE_PARAMS(name, params) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
md, bts::Warn, std::string("md_bridge"), std::string(name), params)
|
||||
|
||||
MdBridgeImpl::MdBridgeImpl(string target, vector<string> module_paths)
|
||||
: target(std::move(target)), module_paths(std::move(module_paths)) {}
|
||||
|
||||
void MdBridgeImpl::on_securities(
|
||||
const std::vector<std::shared_ptr<PySecurity>> &securities) {
|
||||
INFO_TRACE_PARAMS("on_securities", securities.size());
|
||||
@@ -63,7 +28,7 @@ void MdBridgeImpl::on_securities(
|
||||
}
|
||||
|
||||
void MdBridgeImpl::on_books(const std::vector<std::shared_ptr<PyBook>> &books) {
|
||||
DIAG_TRACE_PARAMS("on_books", json_formatter(books.size()));
|
||||
INFO_TRACE_PARAMS("on_books", json_formatter(books.size()));
|
||||
incoming_books.push_many(books.begin(), books.end());
|
||||
}
|
||||
|
||||
@@ -103,26 +68,6 @@ bool MdBridgeImpl::pop_trade(shared_ptr<const PyTrade> &result) {
|
||||
return false;
|
||||
}
|
||||
|
||||
int MdBridgeImpl::run_py(volatile bool *keep_going) {
|
||||
bts_assert(keep_going != nullptr);
|
||||
PythonActivatorT<MdProvider> provider_activator{target, module_paths};
|
||||
provider = provider_activator.get_target([&](MdProvider &pro) {
|
||||
// This will start another thread in python, and call back into c++ from there.
|
||||
auto sw = Stopwatch::start_new();
|
||||
pro.start(this);
|
||||
INFO_TRACE_PARAMS(
|
||||
"run_py",
|
||||
json_formatter(fmt::format(
|
||||
"MdProvider.start took {:.2f}ms", sw.elapsed().total_millis())));
|
||||
});
|
||||
// Release gil to let python run
|
||||
py::gil_scoped_release gil_rel;
|
||||
while (*keep_going) {
|
||||
std::this_thread::sleep_for(std::chrono::seconds(1));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
MdBridgeImpl::~MdBridgeImpl() = default;
|
||||
|
||||
} // namespace bts::md::py_interop
|
||||
@@ -1,15 +1,13 @@
|
||||
#pragma once
|
||||
|
||||
#include "python_interop/md/md_bridge.h"
|
||||
#include "python_interop/md/md_provider.h"
|
||||
#include "spark/collections/concurrent_queue.h"
|
||||
|
||||
namespace bts::md::py_interop {
|
||||
|
||||
class MdBridgeImpl : public MdBridge {
|
||||
public:
|
||||
MdBridgeImpl(std::string target, std::vector<std::string> module_paths);
|
||||
~MdBridgeImpl();
|
||||
~MdBridgeImpl() override;
|
||||
|
||||
// Python interface
|
||||
void on_securities(
|
||||
@@ -21,16 +19,12 @@ class MdBridgeImpl : public MdBridge {
|
||||
bool pop_security(std::shared_ptr<const PySecurity> &result);
|
||||
bool pop_book(std::shared_ptr<const PyBook> &result);
|
||||
bool pop_trade(std::shared_ptr<const PyTrade> &result);
|
||||
int run_py(volatile bool *keep_going);
|
||||
|
||||
private:
|
||||
std::string target;
|
||||
std::vector<std::string> module_paths;
|
||||
collections::ConcurrentQueue<std::shared_ptr<const PySecurity>>
|
||||
incoming_securities;
|
||||
collections::ConcurrentQueue<std::shared_ptr<const PyBook>> incoming_books;
|
||||
collections::ConcurrentQueue<std::shared_ptr<const PyTrade>> incoming_trades;
|
||||
MdProvider *provider;
|
||||
};
|
||||
|
||||
} // namespace bts::md::py_interop
|
||||
@@ -0,0 +1,287 @@
|
||||
#include "plugin_manager.h"
|
||||
#include "python_interop/execution/execution_gateway.h"
|
||||
#include "python_interop/execution/execution_gateway_bridge.h"
|
||||
#include "python_interop/md/md_bridge.h"
|
||||
#include "python_interop/md/md_provider.h"
|
||||
#include "python_interop/plugin_manager_logs.h"
|
||||
#include "python_interop/python_object.h"
|
||||
#include "spark/collections/concurrent_queue.h"
|
||||
#include "spark/collections/concurrent_unordered_map.h"
|
||||
#include "spark/utils/stopwatch.h"
|
||||
#include "spark/utils/string_utils.h"
|
||||
|
||||
#include <csignal>
|
||||
#include <fmt/core.h>
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/gil.h>
|
||||
#include <pybind11/pytypes.h>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
using bts::collections::ConcurrentQueue;
|
||||
using bts::collections::ConcurrentUnorderedMap;
|
||||
using bts::md::py_interop::MdBridge;
|
||||
using bts::utils::Stopwatch;
|
||||
using python_execution::ExecutionGateway;
|
||||
using python_execution::ExecutionGatewayBridge;
|
||||
using std::function;
|
||||
using std::future;
|
||||
using std::mutex;
|
||||
using std::optional;
|
||||
using std::shared_ptr;
|
||||
using std::string;
|
||||
using std::thread;
|
||||
|
||||
static bool parse_target(
|
||||
const std::string &target,
|
||||
std::string *module_name,
|
||||
std::string *type_name) {
|
||||
auto parts = bts::system::string_utils::split(target, ".");
|
||||
if (parts.empty()) {
|
||||
return false;
|
||||
}
|
||||
if (parts.size() == 1) {
|
||||
bts::system::string_utils::snake_to_pascal_case(
|
||||
parts.emplace_back(parts[0]));
|
||||
}
|
||||
*module_name = parts[0];
|
||||
*type_name = parts[1];
|
||||
return !module_name->empty() && !type_name->empty();
|
||||
}
|
||||
|
||||
struct __attribute__((visibility("hidden"))) PluginManager_SessionData {
|
||||
py::scoped_interpreter interpreter{false};
|
||||
std::unordered_map<std::string, std::unique_ptr<py::module>> modules;
|
||||
|
||||
py::module &import(const std::string &name) {
|
||||
auto [it, is_new] = modules.try_emplace(name, nullptr);
|
||||
if (is_new) {
|
||||
it->second =
|
||||
std::make_unique<py::module>(py::module::import(name.c_str()));
|
||||
}
|
||||
return *it->second;
|
||||
}
|
||||
};
|
||||
|
||||
struct __attribute__((visibility("hidden"))) PluginManager_Impl {
|
||||
volatile bool keep_going;
|
||||
mutex mx;
|
||||
optional<thread> work_thread;
|
||||
ConcurrentQueue<function<void()>> work_queue;
|
||||
ConcurrentUnorderedMap<
|
||||
string,
|
||||
shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>
|
||||
md_providers;
|
||||
ConcurrentUnorderedMap<
|
||||
string,
|
||||
shared_ptr<PythonObjectT<python_execution::ExecutionGateway>>>
|
||||
execution_gateways;
|
||||
PluginManager_SessionData *session_data;
|
||||
|
||||
void start_work_thread() {
|
||||
mx.lock();
|
||||
if (work_thread) {
|
||||
mx.unlock();
|
||||
return;
|
||||
}
|
||||
keep_going = true;
|
||||
mx.unlock();
|
||||
work_thread.emplace([&]() {
|
||||
PluginManager_SessionData sd;
|
||||
mx.lock();
|
||||
session_data = &sd;
|
||||
mx.unlock();
|
||||
try {
|
||||
sd.import("sys").attr("path").attr("append")(
|
||||
"python_interop_plugins");
|
||||
sd.import("spark_core").attr("configure_default_logger")();
|
||||
sd.import("bybit_md_provider");
|
||||
sd.import("bybit_execution_gateway");
|
||||
py::gil_scoped_release gil_rel;
|
||||
Stopwatch sw;
|
||||
int n = 0;
|
||||
while (keep_going) {
|
||||
auto work_req = work_queue.wait_until_pop();
|
||||
if (!work_req) {
|
||||
continue;
|
||||
}
|
||||
LOG_INVOKING(n);
|
||||
sw.restart();
|
||||
{
|
||||
py::gil_scoped_acquire gil_acquire;
|
||||
(*work_req)();
|
||||
}
|
||||
LOG_INVOKED(n, sw.elapsed());
|
||||
n++;
|
||||
}
|
||||
} catch (const std::exception &ex) {
|
||||
bts_assert_msgf(
|
||||
false, "PythonWorker work_thread threw. {}",
|
||||
std::string(ex.what()));
|
||||
return;
|
||||
} catch (...) {
|
||||
bts_assert_msg(false, "PythonWorker work_thread threw unknown ex.");
|
||||
return;
|
||||
}
|
||||
mx.lock();
|
||||
session_data = nullptr;
|
||||
mx.unlock();
|
||||
});
|
||||
}
|
||||
|
||||
void stop_work_thread() {
|
||||
mx.lock();
|
||||
if (work_thread && work_thread->joinable()) {
|
||||
keep_going = false;
|
||||
mx.unlock();
|
||||
work_thread->join();
|
||||
work_thread.reset();
|
||||
} else {
|
||||
mx.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
future<shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>
|
||||
start_md_provider(
|
||||
string provider_name, string target, shared_ptr<MdBridge> bridge) {
|
||||
auto prom = std::make_shared<std::promise<
|
||||
shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>>();
|
||||
work_queue.push([this, target = std::move(target),
|
||||
provider_name = std::move(provider_name),
|
||||
bridge = std::move(bridge), prom = prom]() {
|
||||
string module_name;
|
||||
string type_name;
|
||||
if (!parse_target(target, &module_name, &type_name)) {
|
||||
LOG_WORK_ERROR(
|
||||
md,
|
||||
fmt::format("Failed to parse module/type from '{}'", target));
|
||||
return;
|
||||
}
|
||||
py::object obj = session_data->import(module_name.c_str())
|
||||
.attr(type_name.c_str())();
|
||||
auto [md_pro_it, md_pro_is_new] = md_providers.try_emplace(
|
||||
provider_name,
|
||||
std::make_shared<PythonObjectT<bts::md::py_interop::MdProvider>>(
|
||||
std::move(obj)));
|
||||
bts_assert(md_pro_is_new);
|
||||
LOG_STARTING("md_provider", provider_name);
|
||||
Stopwatch sw = Stopwatch::start_new();
|
||||
shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>> md_pro =
|
||||
md_pro_it->second;
|
||||
(*md_pro)->start(bridge.get());
|
||||
LOG_STARTED("md_provider", provider_name, sw.elapsed());
|
||||
prom->set_value(md_pro);
|
||||
// Don't let python handle the signals (otherwise it'll hang on ctrl+c)
|
||||
std::signal(SIGINT, SIG_DFL);
|
||||
});
|
||||
return prom->get_future();
|
||||
}
|
||||
|
||||
void stop_md_provider(const std::string &provider_name) {
|
||||
auto it = md_providers.find(provider_name);
|
||||
if (it != md_providers.end()) {
|
||||
(*it->second)->stop();
|
||||
md_providers.erase(it);
|
||||
}
|
||||
}
|
||||
|
||||
future<shared_ptr<PythonObjectT<ExecutionGateway>>> start_execution_gateway(
|
||||
string gateway_name,
|
||||
string target,
|
||||
shared_ptr<ExecutionGatewayBridge> bridge) {
|
||||
auto prom = std::make_shared<
|
||||
std::promise<shared_ptr<PythonObjectT<ExecutionGateway>>>>();
|
||||
work_queue.push([this, target = std::move(target),
|
||||
gateway_name = std::move(gateway_name),
|
||||
bridge = std::move(bridge), prom = prom]() {
|
||||
string module_name;
|
||||
string type_name;
|
||||
if (!parse_target(target, &module_name, &type_name)) {
|
||||
LOG_WORK_ERROR(
|
||||
OrderMan,
|
||||
fmt::format("Failed to parse module/type from '{}'", target));
|
||||
return;
|
||||
}
|
||||
py::object obj = session_data->import(module_name.c_str())
|
||||
.attr(type_name.c_str())();
|
||||
auto [exe_gw_it, exe_gw_is_new] = execution_gateways.try_emplace(
|
||||
gateway_name,
|
||||
std::make_shared<PythonObjectT<ExecutionGateway>>(std::move(obj)));
|
||||
bts_assert(exe_gw_is_new);
|
||||
LOG_STARTING("execution_gateway", gateway_name);
|
||||
Stopwatch sw = Stopwatch::start_new();
|
||||
shared_ptr<PythonObjectT<ExecutionGateway>> exe_gw = exe_gw_it->second;
|
||||
(*exe_gw)->start(bridge.get());
|
||||
LOG_STARTED("execution_gateway", gateway_name, sw.elapsed());
|
||||
prom->set_value(exe_gw);
|
||||
// Don't let python handle the signals (otherwise it'll hang on ctrl+c)
|
||||
std::signal(SIGINT, SIG_DFL);
|
||||
});
|
||||
return prom->get_future();
|
||||
}
|
||||
|
||||
void stop_execution_gateway(const std::string &provider_name) {
|
||||
auto it = execution_gateways.find(provider_name);
|
||||
if (it != execution_gateways.end()) {
|
||||
(*it->second)->stop();
|
||||
execution_gateways.erase(it);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
PluginManager::PluginManager() : me_(new PluginManager_Impl()) {}
|
||||
|
||||
PluginManager::~PluginManager() {
|
||||
stop_work_thread();
|
||||
delete static_cast<PluginManager_Impl *>(me_);
|
||||
me_ = nullptr;
|
||||
}
|
||||
|
||||
void PluginManager::start_work_thread() {
|
||||
static_cast<PluginManager_Impl *>(me_)->start_work_thread();
|
||||
}
|
||||
|
||||
void PluginManager::stop_work_thread() {
|
||||
static_cast<PluginManager_Impl *>(me_)->stop_work_thread();
|
||||
}
|
||||
|
||||
void PluginManager::invoke(std::function<void()> callback) {
|
||||
auto *me = static_cast<PluginManager_Impl *>(me_);
|
||||
me->work_queue.push(callback);
|
||||
}
|
||||
|
||||
std::future<shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>
|
||||
PluginManager::start_md_provider(
|
||||
string provider_name, string target, shared_ptr<MdBridge> bridge) {
|
||||
return static_cast<PluginManager_Impl *>(me_)->start_md_provider(
|
||||
std::move(provider_name), std::move(target), std::move(bridge));
|
||||
}
|
||||
|
||||
void PluginManager::stop_md_provider(const std::string &provider_name) {
|
||||
static_cast<PluginManager_Impl *>(me_)->stop_md_provider(provider_name);
|
||||
}
|
||||
|
||||
std::future<shared_ptr<PythonObjectT<ExecutionGateway>>>
|
||||
PluginManager::start_execution_gateway(
|
||||
string gateway_name,
|
||||
string target,
|
||||
shared_ptr<ExecutionGatewayBridge> bridge) {
|
||||
return static_cast<PluginManager_Impl *>(me_)->start_execution_gateway(
|
||||
std::move(gateway_name), std::move(target), std::move(bridge));
|
||||
}
|
||||
|
||||
void PluginManager::stop_execution_gateway(const std::string &provider_name) {
|
||||
return static_cast<PluginManager_Impl *>(me_)->stop_execution_gateway(
|
||||
provider_name);
|
||||
}
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -0,0 +1,45 @@
|
||||
#pragma once
|
||||
|
||||
#include "python_interop/execution/execution_gateway.h"
|
||||
#include "python_interop/execution/execution_gateway_bridge.h"
|
||||
#include "python_interop/md/md_bridge.h"
|
||||
#include "python_interop/md/md_provider.h"
|
||||
#include "python_interop/python_object.h"
|
||||
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <memory>
|
||||
#include <pybind11/pytypes.h>
|
||||
#include <string>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
class PluginManager {
|
||||
void *me_;
|
||||
|
||||
public:
|
||||
PluginManager();
|
||||
~PluginManager();
|
||||
|
||||
void start_work_thread();
|
||||
void stop_work_thread();
|
||||
|
||||
std::future<std::shared_ptr<PythonObjectT<bts::md::py_interop::MdProvider>>>
|
||||
start_md_provider(
|
||||
std::string provider_name,
|
||||
std::string target,
|
||||
std::shared_ptr<bts::md::py_interop::MdBridge> bridge);
|
||||
void stop_md_provider(const std::string &provider_name);
|
||||
|
||||
std::future<
|
||||
std::shared_ptr<PythonObjectT<python_execution::ExecutionGateway>>>
|
||||
start_execution_gateway(
|
||||
std::string gateway_name,
|
||||
std::string target,
|
||||
std::shared_ptr<python_execution::ExecutionGatewayBridge> bridge);
|
||||
void stop_execution_gateway(const std::string &gateway_name);
|
||||
|
||||
void invoke(std::function<void()> callback);
|
||||
};
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -0,0 +1,116 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/core/FMT_IMPL.h"
|
||||
#include "spark/core/TimeSpan.h"
|
||||
#include "spark/logging/trace_log.h"
|
||||
#include "spark/os/os.h"
|
||||
#include "spark/utils/thread_id.h"
|
||||
|
||||
#include <fmt/core.h>
|
||||
#include <string>
|
||||
|
||||
namespace python_interop::logs {
|
||||
|
||||
struct StartingLog {
|
||||
std::string instance_name;
|
||||
std::string thread_name;
|
||||
std::int64_t thread_id;
|
||||
|
||||
StartingLog(std::string instance_name)
|
||||
: instance_name(std::move(instance_name)),
|
||||
thread_name(bts::os::get_thread_name()),
|
||||
thread_id(BTS_THIS_THREAD_ID) {}
|
||||
};
|
||||
|
||||
struct StartedLog {
|
||||
std::string instance_name;
|
||||
std::string thread_name;
|
||||
std::int64_t thread_id;
|
||||
bts::core::TimeSpan duration;
|
||||
|
||||
StartedLog(std::string instance_name, bts::core::TimeSpan duration)
|
||||
: instance_name(std::move(instance_name)),
|
||||
thread_name(bts::os::get_thread_name()), thread_id(BTS_THIS_THREAD_ID),
|
||||
duration(duration) {}
|
||||
};
|
||||
|
||||
struct InvokingLog {
|
||||
int count;
|
||||
std::string thread_name;
|
||||
std::int64_t thread_id;
|
||||
|
||||
InvokingLog(int count)
|
||||
: count(count), thread_name(bts::os::get_thread_name()),
|
||||
thread_id(BTS_THIS_THREAD_ID) {}
|
||||
};
|
||||
|
||||
struct InvokedLog {
|
||||
int count;
|
||||
bts::core::TimeSpan duration;
|
||||
std::string thread_name;
|
||||
std::int64_t thread_id;
|
||||
|
||||
InvokedLog(int count, bts::core::TimeSpan dequeue_duration)
|
||||
: count(count), duration(dequeue_duration),
|
||||
thread_name(bts::os::get_thread_name()),
|
||||
thread_id(BTS_THIS_THREAD_ID) {}
|
||||
};
|
||||
|
||||
} // namespace python_interop::logs
|
||||
|
||||
FMT_IMPL(
|
||||
python_interop::logs::StartingLog,
|
||||
l,
|
||||
R"({{ "name": "{}", "ev": "STARTING", "tn": "{}", "tid": "{}" }})",
|
||||
l.instance_name,
|
||||
l.thread_name,
|
||||
l.thread_id);
|
||||
|
||||
FMT_IMPL(
|
||||
python_interop::logs::StartedLog,
|
||||
l,
|
||||
R"({{ "name": "{}", "ev": "STARTED", "dur": "{:.3f}ms", "tn": "{}", "tid": "{}" }})",
|
||||
l.instance_name,
|
||||
l.duration.total_millis(),
|
||||
l.thread_name,
|
||||
l.thread_id);
|
||||
|
||||
FMT_IMPL(
|
||||
python_interop::logs::InvokingLog,
|
||||
l,
|
||||
R"({{ "ev": "INVOKING", "count": {}, "tn": "{}", "tid": "{}" }})",
|
||||
l.count,
|
||||
l.thread_name,
|
||||
l.thread_id);
|
||||
|
||||
FMT_IMPL(
|
||||
python_interop::logs::InvokedLog,
|
||||
l,
|
||||
R"({{ "ev": "INVOKED", "count": {}, "dur": "{:.3f}ms", "tn": "{}", "tid": "{}" }})",
|
||||
l.count,
|
||||
l.duration.total_millis(),
|
||||
l.thread_name,
|
||||
l.thread_id);
|
||||
|
||||
#define LOG_STARTING(type_name, instance_name) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
Gen, bts::Info, string("py_plugin"), string("start_" type_name), \
|
||||
python_interop::logs::StartingLog{instance_name})
|
||||
|
||||
#define LOG_STARTED(type_name, instance_name, dur) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
Gen, bts::Info, string("py_plugin"), string("start_" type_name), \
|
||||
python_interop::logs::StartedLog(instance_name, dur))
|
||||
|
||||
#define LOG_INVOKING(count) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
Gen, bts::Info, string("py_plugin"), string("work"), \
|
||||
python_interop::logs::InvokingLog(count))
|
||||
|
||||
#define LOG_INVOKED(count, duration) \
|
||||
LOG_TRACE_PARAMS( \
|
||||
Gen, bts::Info, string("py_plugin"), string("work"), \
|
||||
python_interop::logs::InvokedLog(count, duration))
|
||||
|
||||
#define LOG_WORK_ERROR(subsystem, params) \
|
||||
LOG_TRACE_PARAMS(subsystem, bts::Error, string("py_plugin"), string("work"), params)
|
||||
@@ -1,93 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "python_interop/python_activator_impl.h"
|
||||
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
class PythonActivator {
|
||||
public:
|
||||
PythonActivator(std::string target, std::vector<std::string> module_paths)
|
||||
: me_(new PythonActivator_Impl(
|
||||
std::move(target), std::move(module_paths))) {}
|
||||
|
||||
PythonActivator() {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
delete me;
|
||||
}
|
||||
|
||||
pybind11::object &get_target_obj() {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
return me->get_target_obj();
|
||||
}
|
||||
|
||||
pybind11::object &get_target_obj(
|
||||
std::function<void(pybind11::object &)> init) {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
return me->get_target_obj(init);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
pybind11::object &get_target_obj(
|
||||
std::function<void(pybind11::object &)> init, Args &&...args) {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
return me->get_target_obj<Args...>(init, std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
void reset() {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
me->reset();
|
||||
}
|
||||
|
||||
private:
|
||||
void *me_;
|
||||
};
|
||||
|
||||
template <typename Target>
|
||||
class PythonActivatorT {
|
||||
public:
|
||||
PythonActivatorT(std::string target, std::vector<std::string> module_paths)
|
||||
: activator_(std::move(target), std::move(module_paths)) {}
|
||||
|
||||
pybind11::object &get_target_obj() { return activator_.get_target_obj(); }
|
||||
|
||||
Target *get_target() {
|
||||
if (target_ == nullptr) {
|
||||
target_ = activator_.get_target_obj().cast<Target *>();
|
||||
}
|
||||
return target_;
|
||||
}
|
||||
|
||||
Target *get_target(std::function<void(Target &)> init) {
|
||||
if (target_ == nullptr) {
|
||||
activator_.get_target_obj([&](pybind11::object &obj) {
|
||||
target_ = obj.cast<Target *>();
|
||||
init(*target_);
|
||||
});
|
||||
}
|
||||
return target_;
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
Target *get_target(std::function<void(Target &)> init, Args &&...args) {
|
||||
if (target_ == nullptr) {
|
||||
activator_.get_target_obj<Args...>(
|
||||
[&](pybind11::object &obj) {
|
||||
target_ = obj.cast<Target *>();
|
||||
init(*target_);
|
||||
},
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
return target_;
|
||||
}
|
||||
|
||||
private:
|
||||
PythonActivator activator_;
|
||||
Target *target_ = nullptr;
|
||||
};
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -1,118 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "python_interop/python_interpreter.h"
|
||||
#include "spark/core/TimeStamp.h"
|
||||
#include "spark/io/json_formatter.h"
|
||||
#include "spark/logging/logging.h"
|
||||
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
struct PythonActivator_Impl {
|
||||
std::string target;
|
||||
std::vector<std::string> module_paths;
|
||||
std::optional<pybind11::object> target_obj;
|
||||
|
||||
PythonActivator_Impl(
|
||||
std::string target, std::vector<std::string> module_paths)
|
||||
: target(std::move(target)), module_paths(std::move(module_paths)) {}
|
||||
|
||||
~PythonActivator_Impl() { reset(); }
|
||||
|
||||
pybind11::object &get_target_obj() {
|
||||
return get_target_obj([](pybind11::object &) {});
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
pybind11::object &get_target_obj(
|
||||
std::function<void(pybind11::object &)> init, Args &&...args) {
|
||||
if (!target_obj) {
|
||||
// Prevent python from overwriting our signal handler (so ctrl+c
|
||||
// actually terminates the process).
|
||||
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
|
||||
target_obj = create_target_with_args<Args...>(
|
||||
init, std::forward<Args>(args)...);
|
||||
std::signal(SIGINT, orig_signal_handler);
|
||||
}
|
||||
return *target_obj;
|
||||
}
|
||||
|
||||
// Undef after use (this is a header)
|
||||
#define LOG_EVENT(ev) \
|
||||
LOG(OrderMan, bts::Info, \
|
||||
R"([py_activator.json]{{ "tgt": "{}", "path": {}, "ev": )" ev \
|
||||
R"(, "ts": {} }})", \
|
||||
target, bts::io::json_formatter(module_paths), \
|
||||
bts::core::TimeStamp::now_ns())
|
||||
#define LOG_ERROR(err) \
|
||||
LOG(OrderMan, bts::Info, \
|
||||
R"([py_activator.json]{{ "tgt": "{}", "path": {}, "ev": "ERROR", "err": "{}", "ts": {} }})", \
|
||||
target, bts::io::json_formatter(module_paths), err, \
|
||||
bts::core::TimeStamp::now_ns())
|
||||
#define LOG_CREATE LOG_EVENT("CREATE")
|
||||
#define LOG_RESET LOG_EVENT("RESET")
|
||||
|
||||
void reset() {
|
||||
LOG_RESET;
|
||||
target_obj.reset();
|
||||
}
|
||||
|
||||
private:
|
||||
template <typename... Args>
|
||||
pybind11::object create_target_with_args(
|
||||
std::function<void(pybind11::object &)> init, Args &&...args) {
|
||||
LOG_CREATE;
|
||||
return PythonInterpreter::exec_t<pybind11::object>(
|
||||
[this, &init,
|
||||
args_tuple =
|
||||
std::make_tuple(std::forward<Args>(args)...)]() mutable {
|
||||
std::string target_module_name;
|
||||
std::string target_type_name;
|
||||
PythonInterpreter::parse_target(
|
||||
target, &target_module_name, &target_type_name);
|
||||
pybind11::object obj;
|
||||
try {
|
||||
// Prepare module path
|
||||
pybind11::module sys = pybind11::module::import("sys");
|
||||
for (const auto &module_path : module_paths) {
|
||||
sys.attr("path").attr("append")(module_path.c_str());
|
||||
}
|
||||
// Import target module
|
||||
pybind11::module target_module =
|
||||
pybind11::module::import(target_module_name.c_str());
|
||||
// Instantiate target using tuple expansion
|
||||
obj = std::apply(
|
||||
[&](auto &&...unpacked_args) {
|
||||
return target_module.attr(target_type_name.c_str())(
|
||||
std::forward<decltype(unpacked_args)>(
|
||||
unpacked_args)...);
|
||||
},
|
||||
std::move(args_tuple));
|
||||
init(obj);
|
||||
} catch (const std::exception &ex) {
|
||||
LOG_ERROR(std::string{ex.what()});
|
||||
}
|
||||
return obj;
|
||||
});
|
||||
}
|
||||
|
||||
pybind11::object create_target() {
|
||||
return create_target_with_args([](pybind11::object &) {});
|
||||
}
|
||||
|
||||
pybind11::object create_target(
|
||||
std::function<void(pybind11::object &)> init) {
|
||||
return create_target_with_args(init);
|
||||
}
|
||||
|
||||
#undef LOG_EVENT
|
||||
#undef LOG_ERROR
|
||||
#undef LOG_CREATE
|
||||
#undef LOG_RESET
|
||||
};
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -1,50 +0,0 @@
|
||||
#include "python_interpreter.h"
|
||||
#include "spark/logging/logging.h"
|
||||
#include "spark/utils/string_utils.h"
|
||||
|
||||
#include <csignal>
|
||||
#include <mutex>
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/gil.h>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
using bts::system::string_utils::snake_to_pascal_case;
|
||||
using bts::system::string_utils::split;
|
||||
|
||||
std::mutex PythonInterpreter::mx;
|
||||
std::optional<pybind11::scoped_interpreter> PythonInterpreter::interpreter;
|
||||
|
||||
void PythonInterpreter::init_interpreter() {
|
||||
if (interpreter.has_value()) {
|
||||
return;
|
||||
}
|
||||
static std::mutex local_mx;
|
||||
std::lock_guard<std::mutex> lock{local_mx};
|
||||
if (interpreter.has_value()) {
|
||||
return;
|
||||
}
|
||||
LOG(Gen, bts::Info, "Creating global py interpreter");
|
||||
// Don't let python handle the signals (otherwise it'll hang on ctrl+c)
|
||||
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
|
||||
interpreter.emplace(false);
|
||||
std::signal(SIGINT, orig_signal_handler);
|
||||
}
|
||||
|
||||
bool PythonInterpreter::parse_target(
|
||||
const std::string &target,
|
||||
std::string *module_name,
|
||||
std::string *type_name) {
|
||||
auto parts = split(target, ".");
|
||||
if (parts.empty()) {
|
||||
return false;
|
||||
}
|
||||
if (parts.size() == 1) {
|
||||
snake_to_pascal_case(parts.emplace_back(parts[0]));
|
||||
}
|
||||
*module_name = parts[0];
|
||||
*type_name = parts[1];
|
||||
return !module_name->empty() && !type_name->empty();
|
||||
}
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -1,125 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/utils/bts_assert.h"
|
||||
#include "spark/utils/string_utils.h"
|
||||
|
||||
#include <mutex>
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/gil.h>
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <string>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
class PythonInterpreter {
|
||||
static std::mutex mx;
|
||||
static std::optional<pybind11::scoped_interpreter> interpreter;
|
||||
static void init_interpreter();
|
||||
|
||||
public:
|
||||
/**
|
||||
* Parses module and type name from a \p target arg. The following formats are supported:
|
||||
* 1) <module_name>.<type_name>
|
||||
* 2) <module_name>
|
||||
* In this case, the type name defaults to the PascalCase equivalent of the module name.
|
||||
* For example, the module "foo_bar" would have type name "FooBar".
|
||||
*/
|
||||
static bool parse_target(
|
||||
const std::string &target,
|
||||
std::string *module_name,
|
||||
std::string *type_name);
|
||||
|
||||
static void exec(std::function<void()> fn) {
|
||||
init_interpreter();
|
||||
std::lock_guard<std::mutex> lock{mx};
|
||||
fn();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static T exec_t(std::function<T()> fn) {
|
||||
init_interpreter();
|
||||
std::lock_guard<std::mutex> lock{mx};
|
||||
pybind11::gil_scoped_acquire gil;
|
||||
T rv = fn();
|
||||
pybind11::gil_scoped_release gil_rel;
|
||||
return rv;
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static pybind11::object create_object(
|
||||
const std::vector<std::string> &module_paths,
|
||||
const std::string &target_name,
|
||||
Args &&...args) {
|
||||
std::string target_module_name;
|
||||
std::string target_type_name;
|
||||
bts_assert(
|
||||
parse_target(target_name, &target_module_name, &target_type_name));
|
||||
return create_object(
|
||||
module_paths, target_module_name, target_type_name,
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static pybind11::object create_object(
|
||||
const std::vector<std::string> &module_paths,
|
||||
const std::string &target_type_name,
|
||||
const std::string &target_module_name,
|
||||
Args &&...args) {
|
||||
namespace py = pybind11;
|
||||
return exec_t<py::object>([&]() {
|
||||
// Prepare module path
|
||||
py::module sys = py::module::import("sys");
|
||||
for (const auto &module_path : module_paths) {
|
||||
sys.attr("path").attr("append")(module_path.c_str());
|
||||
}
|
||||
// Import target module
|
||||
py::module target_module =
|
||||
py::module::import(target_module_name.c_str());
|
||||
// Instantiate target
|
||||
return target_module.attr(target_type_name.c_str())(
|
||||
std::forward<Args>(args)...);
|
||||
});
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static pybind11::object create_object_with_init(
|
||||
const std::vector<std::string> &module_paths,
|
||||
const std::string &target_name,
|
||||
std::function<void(pybind11::object &)> init_fn,
|
||||
Args &&...args) {
|
||||
std::string target_module_name;
|
||||
std::string target_type_name;
|
||||
bts_assert(
|
||||
parse_target(target_name, &target_module_name, &target_type_name));
|
||||
return create_object_with_init(
|
||||
module_paths, target_module_name, target_type_name, init_fn,
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static pybind11::object create_object_with_init(
|
||||
const std::vector<std::string> &module_paths,
|
||||
const std::string &target_type_name,
|
||||
const std::string &target_module_name,
|
||||
std::function<void(pybind11::object &)> init_fn,
|
||||
Args &&...args) {
|
||||
namespace py = pybind11;
|
||||
return exec_t<py::object>([&]() {
|
||||
// Prepare module path
|
||||
py::module sys = py::module::import("sys");
|
||||
for (const auto &module_path : module_paths) {
|
||||
sys.attr("path").attr("append")(module_path.c_str());
|
||||
}
|
||||
// Import target module
|
||||
py::module target_module =
|
||||
py::module::import(target_module_name.c_str());
|
||||
// Instantiate target
|
||||
py::object obj = target_module.attr(target_type_name.c_str())(
|
||||
std::forward<Args>(args)...);
|
||||
init_fn(obj);
|
||||
return obj;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -0,0 +1,45 @@
|
||||
#include "python_object.h"
|
||||
#include "python_interop/execution/execution_gateway.h"
|
||||
#include "python_interop/md/md_provider.h"
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
template <typename T>
|
||||
struct __attribute__((visibility("hidden"))) PythonObjectT_Impl {
|
||||
pybind11::object obj_;
|
||||
T *tp_ = nullptr;
|
||||
|
||||
PythonObjectT_Impl(pybind11::object &&obj)
|
||||
: obj_(std::move(obj)), tp_(obj_.cast<T *>()) {}
|
||||
|
||||
T &operator*();
|
||||
T *operator->();
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
PythonObjectT<T>::PythonObjectT(pybind11::object &&obj)
|
||||
: me(new PythonObjectT_Impl<T>(std::move(obj))) {}
|
||||
|
||||
template <typename T>
|
||||
PythonObjectT<T>::~PythonObjectT() {
|
||||
delete static_cast<PythonObjectT_Impl<T> *>(me);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T &PythonObjectT<T>::operator*() {
|
||||
return *static_cast<PythonObjectT_Impl<T> *>(me)->tp_;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T *PythonObjectT<T>::operator->() {
|
||||
return static_cast<PythonObjectT_Impl<T> *>(me)->tp_;
|
||||
}
|
||||
|
||||
#define PYTHON_OBJECT_DEF(T) \
|
||||
template struct PythonObjectT_Impl<T>; \
|
||||
template class PythonObjectT<T>;
|
||||
|
||||
PYTHON_OBJECT_DEF(bts::md::py_interop::MdProvider);
|
||||
PYTHON_OBJECT_DEF(python_execution::ExecutionGateway);
|
||||
|
||||
} // namespace python_interop
|
||||
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
#include <pybind11/pytypes.h>
|
||||
|
||||
namespace python_interop {
|
||||
|
||||
template <typename T>
|
||||
class PythonObjectT {
|
||||
void *me;
|
||||
|
||||
public:
|
||||
__attribute__((visibility("hidden"))) PythonObjectT(pybind11::object &&obj);
|
||||
~PythonObjectT();
|
||||
|
||||
T &operator*();
|
||||
T *operator->();
|
||||
};
|
||||
|
||||
} // namespace python_interop
|
||||
Reference in new issue
Block a user