refactor: Redesign py md so it doesn't lock the global interpreter
We can only embed one interpreter in the sparkd process, so all py plugins need to share it. * Move core py interop code to new lib_python_core. We're about to add a new lib for py execution, so it needs to be accesible from both execution and md libs * Fix typo for type hint in py_md_utils.py * Run py md providers on background thread * Rename MdSink -> MdBridge
This commit is contained in:
1 parent
de8edca010
commit
e7ee1e02dd
25 files changed
+840
-435
No files matched your search
@@ -0,0 +1,30 @@
|
||||
cmake_minimum_required(VERSION 3.23)
|
||||
project(python_core)
|
||||
include(${CMAKE_SOURCE_DIR}/cmake/InitLibrary.cmake)
|
||||
init_library()
|
||||
target_link_libraries(
|
||||
${PROJECT_NAME}
|
||||
PUBLIC
|
||||
spark
|
||||
)
|
||||
|
||||
# Find Python from the virtual environment
|
||||
find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
|
||||
find_package(pybind11 REQUIRED)
|
||||
|
||||
target_include_directories(
|
||||
${PROJECT_NAME}
|
||||
PUBLIC
|
||||
${Python3_INCLUDE_DIRS}
|
||||
)
|
||||
target_link_directories(
|
||||
${PROJECT_NAME}
|
||||
PUBLIC
|
||||
${Python3_LIBRARY_DIRS}
|
||||
)
|
||||
target_link_libraries(
|
||||
${PROJECT_NAME}
|
||||
PUBLIC
|
||||
pybind11::embed
|
||||
${Python3_LIBRARIES}
|
||||
)
|
||||
Whitespace-only changes.
@@ -0,0 +1,93 @@
|
||||
#pragma once
|
||||
|
||||
#include "python_core/python_activator_impl.h"
|
||||
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace python_core {
|
||||
|
||||
class PythonActivator {
|
||||
public:
|
||||
PythonActivator(std::string target, std::vector<std::string> module_paths)
|
||||
: me_(new PythonActivator_Impl(
|
||||
std::move(target), std::move(module_paths))) {}
|
||||
|
||||
PythonActivator() {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
delete me;
|
||||
}
|
||||
|
||||
pybind11::object &get_target_obj() {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
return me->get_target_obj();
|
||||
}
|
||||
|
||||
pybind11::object &get_target_obj(
|
||||
std::function<void(pybind11::object &)> init) {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
return me->get_target_obj(init);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
pybind11::object &get_target_obj(
|
||||
std::function<void(pybind11::object &)> init, Args &&...args) {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
return me->get_target_obj<Args...>(init, std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
void reset() {
|
||||
auto *me = static_cast<PythonActivator_Impl *>(this->me_);
|
||||
me->reset();
|
||||
}
|
||||
|
||||
private:
|
||||
void *me_;
|
||||
};
|
||||
|
||||
template <typename Target>
|
||||
class PythonActivatorT {
|
||||
public:
|
||||
PythonActivatorT(std::string target, std::vector<std::string> module_paths)
|
||||
: activator_(std::move(target), std::move(module_paths)) {}
|
||||
|
||||
pybind11::object &get_target_obj() { return activator_.get_target_obj(); }
|
||||
|
||||
Target *get_target() {
|
||||
if (target_ == nullptr) {
|
||||
target_ = activator_.get_target_obj().cast<Target *>();
|
||||
}
|
||||
return target_;
|
||||
}
|
||||
|
||||
Target *get_target(std::function<void(Target &)> init) {
|
||||
if (target_ == nullptr) {
|
||||
activator_.get_target_obj([&](pybind11::object &obj) {
|
||||
target_ = obj.cast<Target *>();
|
||||
init(*target_);
|
||||
});
|
||||
}
|
||||
return target_;
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
Target *get_target(std::function<void(Target &)> init, Args &&...args) {
|
||||
if (target_ == nullptr) {
|
||||
activator_.get_target_obj<Args...>(
|
||||
[&](pybind11::object &obj) {
|
||||
target_ = obj.cast<Target *>();
|
||||
init(*target_);
|
||||
},
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
return target_;
|
||||
}
|
||||
|
||||
private:
|
||||
PythonActivator activator_;
|
||||
Target *target_ = nullptr;
|
||||
};
|
||||
|
||||
} // namespace python_core
|
||||
@@ -0,0 +1,140 @@
|
||||
#pragma once
|
||||
|
||||
#include "python_core/python_interpreter.h"
|
||||
#include "spark/logging/logging.h"
|
||||
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace python_core {
|
||||
|
||||
struct PythonActivator_Impl {
|
||||
std::string target;
|
||||
std::vector<std::string> module_paths;
|
||||
std::optional<pybind11::object> target_obj;
|
||||
|
||||
PythonActivator_Impl(
|
||||
std::string target, std::vector<std::string> module_paths)
|
||||
: target(std::move(target)), module_paths(std::move(module_paths)) {}
|
||||
|
||||
~PythonActivator_Impl() { reset(); }
|
||||
|
||||
template <typename... Args>
|
||||
pybind11::object create_target_with_args(
|
||||
std::function<void(pybind11::object &)> init, Args &&...args) {
|
||||
LOG(OrderMan, bts::Info, "PythonActivator::create_target. {}", target);
|
||||
return PythonInterpreter::exec_t<pybind11::object>(
|
||||
[this, &init,
|
||||
args_tuple =
|
||||
std::make_tuple(std::forward<Args>(args)...)]() mutable {
|
||||
std::string target_module_name;
|
||||
std::string target_type_name;
|
||||
PythonInterpreter::parse_target(
|
||||
target, &target_module_name, &target_type_name);
|
||||
// Prepare module path
|
||||
pybind11::module sys = pybind11::module::import("sys");
|
||||
for (const auto &module_path : module_paths) {
|
||||
sys.attr("path").attr("append")(module_path.c_str());
|
||||
}
|
||||
// Import target module
|
||||
pybind11::module target_module =
|
||||
pybind11::module::import(target_module_name.c_str());
|
||||
// Instantiate target using tuple expansion
|
||||
pybind11::object obj = std::apply(
|
||||
[&](auto &&...unpacked_args) {
|
||||
return target_module.attr(target_type_name.c_str())(
|
||||
std::forward<decltype(unpacked_args)>(
|
||||
unpacked_args)...);
|
||||
},
|
||||
std::move(args_tuple));
|
||||
init(obj);
|
||||
return obj;
|
||||
});
|
||||
}
|
||||
|
||||
pybind11::object &get_target_obj() {
|
||||
if (!target_obj) {
|
||||
// Prevent python from overwriting our signal handler (so ctrl+c
|
||||
// actually terminates the process).
|
||||
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
|
||||
target_obj = create_target();
|
||||
std::signal(SIGINT, orig_signal_handler);
|
||||
}
|
||||
return *target_obj;
|
||||
}
|
||||
|
||||
pybind11::object &get_target_obj(
|
||||
std::function<void(pybind11::object &)> init) {
|
||||
if (!target_obj) {
|
||||
// Prevent python from overwriting our signal handler (so ctrl+c
|
||||
// actually terminates the process).
|
||||
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
|
||||
target_obj = create_target(init);
|
||||
std::signal(SIGINT, orig_signal_handler);
|
||||
}
|
||||
return *target_obj;
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
pybind11::object &get_target_obj(
|
||||
std::function<void(pybind11::object &)> init, Args &&...args) {
|
||||
if (!target_obj) {
|
||||
// Prevent python from overwriting our signal handler (so ctrl+c
|
||||
// actually terminates the process).
|
||||
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
|
||||
target_obj = create_target_with_args<Args...>(
|
||||
init, std::forward<Args>(args)...);
|
||||
std::signal(SIGINT, orig_signal_handler);
|
||||
}
|
||||
return *target_obj;
|
||||
}
|
||||
|
||||
pybind11::object create_target() {
|
||||
LOG(OrderMan, bts::Info, "PythonActivator::create_target. {}", target);
|
||||
return PythonInterpreter::exec_t<pybind11::object>([&]() {
|
||||
std::string target_module_name;
|
||||
std::string target_type_name;
|
||||
PythonInterpreter::parse_target(
|
||||
target, &target_module_name, &target_type_name);
|
||||
// Prepare module path
|
||||
pybind11::module sys = pybind11::module::import("sys");
|
||||
for (const auto &module_path : module_paths) {
|
||||
sys.attr("path").attr("append")(module_path.c_str());
|
||||
}
|
||||
// Import target module
|
||||
pybind11::module target_module =
|
||||
pybind11::module::import(target_module_name.c_str());
|
||||
// Instantiate target
|
||||
return target_module.attr(target_type_name.c_str())();
|
||||
});
|
||||
}
|
||||
|
||||
pybind11::object create_target(
|
||||
std::function<void(pybind11::object &)> init) {
|
||||
LOG(OrderMan, bts::Info, "PythonActivator::create_target. {}", target);
|
||||
return PythonInterpreter::exec_t<pybind11::object>([&]() {
|
||||
std::string target_module_name;
|
||||
std::string target_type_name;
|
||||
PythonInterpreter::parse_target(
|
||||
target, &target_module_name, &target_type_name);
|
||||
// Prepare module path
|
||||
pybind11::module sys = pybind11::module::import("sys");
|
||||
for (const auto &module_path : module_paths) {
|
||||
sys.attr("path").attr("append")(module_path.c_str());
|
||||
}
|
||||
// Import target module
|
||||
pybind11::module target_module =
|
||||
pybind11::module::import(target_module_name.c_str());
|
||||
// Instantiate target
|
||||
pybind11::object obj = target_module.attr(target_type_name.c_str())();
|
||||
init(obj);
|
||||
return obj;
|
||||
});
|
||||
}
|
||||
|
||||
void reset() { target_obj.reset(); }
|
||||
};
|
||||
|
||||
} // namespace python_core
|
||||
@@ -0,0 +1,50 @@
|
||||
#include "python_interpreter.h"
|
||||
#include "spark/logging/logging.h"
|
||||
#include "spark/utils/string_utils.h"
|
||||
|
||||
#include <csignal>
|
||||
#include <mutex>
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/gil.h>
|
||||
|
||||
namespace python_core {
|
||||
|
||||
using bts::system::string_utils::snake_to_pascal_case;
|
||||
using bts::system::string_utils::split;
|
||||
|
||||
std::mutex PythonInterpreter::mx;
|
||||
std::optional<pybind11::scoped_interpreter> PythonInterpreter::interpreter;
|
||||
|
||||
void PythonInterpreter::init_interpreter() {
|
||||
if (interpreter.has_value()) {
|
||||
return;
|
||||
}
|
||||
static std::mutex local_mx;
|
||||
std::lock_guard<std::mutex> lock{local_mx};
|
||||
if (interpreter.has_value()) {
|
||||
return;
|
||||
}
|
||||
LOG(Gen, bts::Info, "Creating global py interpreter");
|
||||
// Don't let python handle the signals (otherwise it'll hang on ctrl+c)
|
||||
auto orig_signal_handler = std::signal(SIGINT, SIG_DFL);
|
||||
interpreter.emplace(false);
|
||||
std::signal(SIGINT, orig_signal_handler);
|
||||
}
|
||||
|
||||
bool PythonInterpreter::parse_target(
|
||||
const std::string &target,
|
||||
std::string *module_name,
|
||||
std::string *type_name) {
|
||||
auto parts = split(target, ".");
|
||||
if (parts.empty()) {
|
||||
return false;
|
||||
}
|
||||
if (parts.size() == 1) {
|
||||
snake_to_pascal_case(parts.emplace_back(parts[0]));
|
||||
}
|
||||
*module_name = parts[0];
|
||||
*type_name = parts[1];
|
||||
return !module_name->empty() && !type_name->empty();
|
||||
}
|
||||
|
||||
} // namespace python_core
|
||||
@@ -0,0 +1,125 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/utils/bts_assert.h"
|
||||
#include "spark/utils/string_utils.h"
|
||||
|
||||
#include <mutex>
|
||||
#include <pybind11/embed.h>
|
||||
#include <pybind11/gil.h>
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <string>
|
||||
|
||||
namespace python_core {
|
||||
|
||||
class PythonInterpreter {
|
||||
static std::mutex mx;
|
||||
static std::optional<pybind11::scoped_interpreter> interpreter;
|
||||
static void init_interpreter();
|
||||
|
||||
public:
|
||||
/**
|
||||
* Parses module and type name from a \p target arg. The following formats are supported:
|
||||
* 1) <module_name>.<type_name>
|
||||
* 2) <module_name>
|
||||
* In this case, the type name defaults to the PascalCase equivalent of the module name.
|
||||
* For example, the module "foo_bar" would have type name "FooBar".
|
||||
*/
|
||||
static bool parse_target(
|
||||
const std::string &target,
|
||||
std::string *module_name,
|
||||
std::string *type_name);
|
||||
|
||||
static void exec(std::function<void()> fn) {
|
||||
init_interpreter();
|
||||
std::lock_guard<std::mutex> lock{mx};
|
||||
fn();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static T exec_t(std::function<T()> fn) {
|
||||
init_interpreter();
|
||||
std::lock_guard<std::mutex> lock{mx};
|
||||
pybind11::gil_scoped_acquire gil;
|
||||
T rv = fn();
|
||||
pybind11::gil_scoped_release gil_rel;
|
||||
return rv;
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static pybind11::object create_object(
|
||||
const std::vector<std::string> &module_paths,
|
||||
const std::string &target_name,
|
||||
Args &&...args) {
|
||||
std::string target_module_name;
|
||||
std::string target_type_name;
|
||||
bts_assert(
|
||||
parse_target(target_name, &target_module_name, &target_type_name));
|
||||
return create_object(
|
||||
module_paths, target_module_name, target_type_name,
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static pybind11::object create_object(
|
||||
const std::vector<std::string> &module_paths,
|
||||
const std::string &target_type_name,
|
||||
const std::string &target_module_name,
|
||||
Args &&...args) {
|
||||
namespace py = pybind11;
|
||||
return exec_t<py::object>([&]() {
|
||||
// Prepare module path
|
||||
py::module sys = py::module::import("sys");
|
||||
for (const auto &module_path : module_paths) {
|
||||
sys.attr("path").attr("append")(module_path.c_str());
|
||||
}
|
||||
// Import target module
|
||||
py::module target_module =
|
||||
py::module::import(target_module_name.c_str());
|
||||
// Instantiate target
|
||||
return target_module.attr(target_type_name.c_str())(
|
||||
std::forward<Args>(args)...);
|
||||
});
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static pybind11::object create_object_with_init(
|
||||
const std::vector<std::string> &module_paths,
|
||||
const std::string &target_name,
|
||||
std::function<void(pybind11::object &)> init_fn,
|
||||
Args &&...args) {
|
||||
std::string target_module_name;
|
||||
std::string target_type_name;
|
||||
bts_assert(
|
||||
parse_target(target_name, &target_module_name, &target_type_name));
|
||||
return create_object_with_init(
|
||||
module_paths, target_module_name, target_type_name, init_fn,
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
static pybind11::object create_object_with_init(
|
||||
const std::vector<std::string> &module_paths,
|
||||
const std::string &target_type_name,
|
||||
const std::string &target_module_name,
|
||||
std::function<void(pybind11::object &)> init_fn,
|
||||
Args &&...args) {
|
||||
namespace py = pybind11;
|
||||
return exec_t<py::object>([&]() {
|
||||
// Prepare module path
|
||||
py::module sys = py::module::import("sys");
|
||||
for (const auto &module_path : module_paths) {
|
||||
sys.attr("path").attr("append")(module_path.c_str());
|
||||
}
|
||||
// Import target module
|
||||
py::module target_module =
|
||||
py::module::import(target_module_name.c_str());
|
||||
// Instantiate target
|
||||
py::object obj = target_module.attr(target_type_name.c_str())(
|
||||
std::forward<Args>(args)...);
|
||||
init_fn(obj);
|
||||
return obj;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace python_core
|
||||
Reference in new issue
Block a user