refactor: Redesign py md so it doesn't lock the global interpreter

We can only embed one interpreter in the sparkd process, so all py
plugins need to share it.

* Move core py interop code to new lib_python_core. We're about to add a
  new lib for py execution, so it needs to be accesible from both
  execution and md libs
* Fix typo for type hint in py_md_utils.py
* Run py md providers on background thread
* Rename MdSink -> MdBridge
This commit is contained in:
John Colvin committed 2025-06-09 05:54:10 +00:00
1 parent de8edca010
commit e7ee1e02dd
25 files changed
+840 -435

No files matched your search

+30
View File
@@ -0,0 +1,30 @@
cmake_minimum_required(VERSION 3.23)
project(python_core)
include(${CMAKE_SOURCE_DIR}/cmake/InitLibrary.cmake)
init_library()
target_link_libraries(
${PROJECT_NAME}
PUBLIC
spark
)
# Find Python from the virtual environment
find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
find_package(pybind11 REQUIRED)
target_include_directories(
${PROJECT_NAME}
PUBLIC
${Python3_INCLUDE_DIRS}
)
target_link_directories(
${PROJECT_NAME}
PUBLIC
${Python3_LIBRARY_DIRS}
)
target_link_libraries(
${PROJECT_NAME}
PUBLIC
pybind11::embed
${Python3_LIBRARIES}
)
View File
Whitespace-only changes.
@@ -0,0 +1,93 @@
#pragma once
#include "python_core/python_activator_impl.h"
#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
#include <string>
#include <vector>
namespace python_core {
class PythonActivator {
public:
PythonActivator(std::string target, std::vector<std::string> module_paths)
: me_(new PythonActivator_Impl(
std::move(target), std::move(module_paths))) {}
PythonActivator() {
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
delete me;
}
pybind11::object &get_target_obj() {
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
return me->get_target_obj();
}
pybind11::object &get_target_obj(
std::function<void(pybind11::object &)> init) {
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
return me->get_target_obj(init);
}
template <typename... Args>
pybind11::object &get_target_obj(
std::function<void(pybind11::object &)> init, Args &&...args) {
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
return me->get_target_obj<Args...>(init, std::forward<Args>(args)...);
}
void reset() {
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
me->reset();
}
private:
void *me_;
};
template <typename Target>
class PythonActivatorT {
public:
PythonActivatorT(std::string target, std::vector<std::string> module_paths)
: activator_(std::move(target), std::move(module_paths)) {}
pybind11::object &get_target_obj() { return activator_.get_target_obj(); }
Target *get_target() {
if (target_ == nullptr) {
target_ = activator_.get_target_obj().cast<Target *>();
}
return target_;
}
Target *get_target(std::function<void(Target &)> init) {
if (target_ == nullptr) {
activator_.get_target_obj([&](pybind11::object &obj) {
target_ = obj.cast<Target *>();
init(*target_);
});
}
return target_;
}
template <typename... Args>
Target *get_target(std::function<void(Target &)> init, Args &&...args) {
if (target_ == nullptr) {
activator_.get_target_obj<Args...>(
[&](pybind11::object &obj) {
target_ = obj.cast<Target *>();
init(*target_);
},
std::forward<Args>(args)...);
}
return target_;
}
private:
PythonActivator activator_;
Target *target_ = nullptr;
};
} // namespace python_core
@@ -0,0 +1,140 @@
#pragma once
#include "python_core/python_interpreter.h"
#include "spark/logging/logging.h"
#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
#include <string>
#include <vector>
namespace python_core {
struct PythonActivator_Impl {
std::string target;
std::vector<std::string> module_paths;
std::optional<pybind11::object> target_obj;
PythonActivator_Impl(
std::string target, std::vector<std::string> module_paths)
: target(std::move(target)), module_paths(std::move(module_paths)) {}
~PythonActivator_Impl() { reset(); }
template <typename... Args>
pybind11::object create_target_with_args(
std::function<void(pybind11::object &)> init, Args &&...args) {
LOG(OrderMan, bts::Info, "PythonActivator::create_target. {}", target);
return PythonInterpreter::exec_t<pybind11::object>(
[this, &init,
args_tuple =
std::make_tuple(std::forward<Args>(args)...)]() mutable {
std::string target_module_name;
std::string target_type_name;
PythonInterpreter::parse_target(
target, &target_module_name, &target_type_name);
// Prepare module path
pybind11::module sys = pybind11::module::import("sys");
for (const auto &module_path : module_paths) {
sys.attr("path").attr("append")(module_path.c_str());
}
// Import target module
pybind11::module target_module =
pybind11::module::import(target_module_name.c_str());
// Instantiate target using tuple expansion
pybind11::object obj = std::apply(
[&](auto &&...unpacked_args) {
return target_module.attr(target_type_name.c_str())(
std::forward<decltype(unpacked_args)>(
unpacked_args)...);
},
std::move(args_tuple));
init(obj);
return obj;
});
}
pybind11::object &get_target_obj() {
if (!target_obj) {
// Prevent python from overwriting our signal handler (so ctrl+c
// actually terminates the process).
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
target_obj = create_target();
std::signal(SIGINT, orig_signal_handler);
}
return *target_obj;
}
pybind11::object &get_target_obj(
std::function<void(pybind11::object &)> init) {
if (!target_obj) {
// Prevent python from overwriting our signal handler (so ctrl+c
// actually terminates the process).
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
target_obj = create_target(init);
std::signal(SIGINT, orig_signal_handler);
}
return *target_obj;
}
template <typename... Args>
pybind11::object &get_target_obj(
std::function<void(pybind11::object &)> init, Args &&...args) {
if (!target_obj) {
// Prevent python from overwriting our signal handler (so ctrl+c
// actually terminates the process).
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
target_obj = create_target_with_args<Args...>(
init, std::forward<Args>(args)...);
std::signal(SIGINT, orig_signal_handler);
}
return *target_obj;
}
pybind11::object create_target() {
LOG(OrderMan, bts::Info, "PythonActivator::create_target. {}", target);
return PythonInterpreter::exec_t<pybind11::object>([&]() {
std::string target_module_name;
std::string target_type_name;
PythonInterpreter::parse_target(
target, &target_module_name, &target_type_name);
// Prepare module path
pybind11::module sys = pybind11::module::import("sys");
for (const auto &module_path : module_paths) {
sys.attr("path").attr("append")(module_path.c_str());
}
// Import target module
pybind11::module target_module =
pybind11::module::import(target_module_name.c_str());
// Instantiate target
return target_module.attr(target_type_name.c_str())();
});
}
pybind11::object create_target(
std::function<void(pybind11::object &)> init) {
LOG(OrderMan, bts::Info, "PythonActivator::create_target. {}", target);
return PythonInterpreter::exec_t<pybind11::object>([&]() {
std::string target_module_name;
std::string target_type_name;
PythonInterpreter::parse_target(
target, &target_module_name, &target_type_name);
// Prepare module path
pybind11::module sys = pybind11::module::import("sys");
for (const auto &module_path : module_paths) {
sys.attr("path").attr("append")(module_path.c_str());
}
// Import target module
pybind11::module target_module =
pybind11::module::import(target_module_name.c_str());
// Instantiate target
pybind11::object obj = target_module.attr(target_type_name.c_str())();
init(obj);
return obj;
});
}
void reset() { target_obj.reset(); }
};
} // namespace python_core
@@ -0,0 +1,50 @@
#include "python_interpreter.h"
#include "spark/logging/logging.h"
#include "spark/utils/string_utils.h"
#include <csignal>
#include <mutex>
#include <pybind11/embed.h>
#include <pybind11/gil.h>
namespace python_core {
using bts::system::string_utils::snake_to_pascal_case;
using bts::system::string_utils::split;
std::mutex PythonInterpreter::mx;
std::optional<pybind11::scoped_interpreter> PythonInterpreter::interpreter;
void PythonInterpreter::init_interpreter() {
if (interpreter.has_value()) {
return;
}
static std::mutex local_mx;
std::lock_guard<std::mutex> lock{local_mx};
if (interpreter.has_value()) {
return;
}
LOG(Gen, bts::Info, "Creating global py interpreter");
// Don't let python handle the signals (otherwise it'll hang on ctrl+c)
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
interpreter.emplace(false);
std::signal(SIGINT, orig_signal_handler);
}
bool PythonInterpreter::parse_target(
const std::string &target,
std::string *module_name,
std::string *type_name) {
auto parts = split(target, ".");
if (parts.empty()) {
return false;
}
if (parts.size() == 1) {
snake_to_pascal_case(parts.emplace_back(parts[0]));
}
*module_name = parts[0];
*type_name = parts[1];
return !module_name->empty() && !type_name->empty();
}
} // namespace python_core
@@ -0,0 +1,125 @@
#pragma once
#include "spark/utils/bts_assert.h"
#include "spark/utils/string_utils.h"
#include <mutex>
#include <pybind11/embed.h>
#include <pybind11/gil.h>
#include <pybind11/pybind11.h>
#include <string>
namespace python_core {
class PythonInterpreter {
static std::mutex mx;
static std::optional<pybind11::scoped_interpreter> interpreter;
static void init_interpreter();
public:
/**
* Parses module and type name from a \p target arg. The following formats are supported:
* 1) <module_name>.<type_name>
* 2) <module_name>
* In this case, the type name defaults to the PascalCase equivalent of the module name.
* For example, the module "foo_bar" would have type name "FooBar".
*/
static bool parse_target(
const std::string &target,
std::string *module_name,
std::string *type_name);
static void exec(std::function<void()> fn) {
init_interpreter();
std::lock_guard<std::mutex> lock{mx};
fn();
}
template <typename T>
static T exec_t(std::function<T()> fn) {
init_interpreter();
std::lock_guard<std::mutex> lock{mx};
pybind11::gil_scoped_acquire gil;
T rv = fn();
pybind11::gil_scoped_release gil_rel;
return rv;
}
template <typename... Args>
static pybind11::object create_object(
const std::vector<std::string> &module_paths,
const std::string &target_name,
Args &&...args) {
std::string target_module_name;
std::string target_type_name;
bts_assert(
parse_target(target_name, &target_module_name, &target_type_name));
return create_object(
module_paths, target_module_name, target_type_name,
std::forward<Args>(args)...);
}
template <typename... Args>
static pybind11::object create_object(
const std::vector<std::string> &module_paths,
const std::string &target_type_name,
const std::string &target_module_name,
Args &&...args) {
namespace py = pybind11;
return exec_t<py::object>([&]() {
// Prepare module path
py::module sys = py::module::import("sys");
for (const auto &module_path : module_paths) {
sys.attr("path").attr("append")(module_path.c_str());
}
// Import target module
py::module target_module =
py::module::import(target_module_name.c_str());
// Instantiate target
return target_module.attr(target_type_name.c_str())(
std::forward<Args>(args)...);
});
}
template <typename... Args>
static pybind11::object create_object_with_init(
const std::vector<std::string> &module_paths,
const std::string &target_name,
std::function<void(pybind11::object &)> init_fn,
Args &&...args) {
std::string target_module_name;
std::string target_type_name;
bts_assert(
parse_target(target_name, &target_module_name, &target_type_name));
return create_object_with_init(
module_paths, target_module_name, target_type_name, init_fn,
std::forward<Args>(args)...);
}
template <typename... Args>
static pybind11::object create_object_with_init(
const std::vector<std::string> &module_paths,
const std::string &target_type_name,
const std::string &target_module_name,
std::function<void(pybind11::object &)> init_fn,
Args &&...args) {
namespace py = pybind11;
return exec_t<py::object>([&]() {
// Prepare module path
py::module sys = py::module::import("sys");
for (const auto &module_path : module_paths) {
sys.attr("path").attr("append")(module_path.c_str());
}
// Import target module
py::module target_module =
py::module::import(target_module_name.c_str());
// Instantiate target
py::object obj = target_module.attr(target_type_name.c_str())(
std::forward<Args>(args)...);
init_fn(obj);
return obj;
});
}
};
} // namespace python_core