feat: Add support for python md plugins

This commit is contained in:
John Colvin committed 2024-09-18 14:02:39 +00:00
1 parent 2b62668df8
commit 2e9b4c78f3
46 files changed
+2265 -11

No files matched your search

+137
View File
@@ -0,0 +1,137 @@
cmake_minimum_required(VERSION 3.23)
# This lists file configures 3 primary targets:
# 1) python_md -> libpython_md.so
# * Normal c++ libs/exes (e.g., libspark, sparkd) can link this
# 2) python_md_module -> python_md_module.cpython-310-x86_64-linux-gnu.so
# * Exports a py module with the MdProvider base class
# * Py md providers must import this and define a class that derives from 'python_md_module.MdProvider'
# * Normal c++ libs/exes shouldn't link this
# * For more info on the cpython api:
# - https://docs.python.org/3/extending/building.html
# - https://docs.python.org/3/c-api/module.html#c.PyModuleDef
# 3) python_md_providers
# * Copies py md provider plugins to a runtime dir where sparkd can find them
# load them as plugins.
# * Doesn't output a c++ lib or exe
project(python_md)
include(GNUInstallDirs)
include(${CMAKE_SOURCE_DIR}/cmake/InitSharedLibrary.cmake)
include(${CMAKE_SOURCE_DIR}/cmake/InstallBasicConfig.cmake)
if (NOT DEFINED PROJECT_VERSION)
set(PROJECT_VERSION 1.0.0)
endif()
# Find Python from the virtual environment
find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
find_package(pybind11 REQUIRED)
# 1) python_md
init_shared_library()
target_include_directories(
${PROJECT_NAME}
PUBLIC
${Python3_INCLUDE_DIRS}
)
target_link_directories(
${PROJECT_NAME}
PUBLIC
${Python3_LIBRARY_DIRS}
)
target_link_libraries(
${PROJECT_NAME}
PUBLIC
pybind11::embed
${Python3_LIBRARIES}
system
)
install_basic_config()
# 2) python_md_module
set(module_proj ${PROJECT_NAME}_module)
set(module_proj_SOURCES ${${PROJECT_NAME}_SOURCES})
pybind11_add_module(${module_proj} SHARED ${module_proj_SOURCES})
target_link_libraries(${module_proj} PUBLIC pybind11::embed system)
foreach(dir ${pybind11_INCLUDE_DIRS})
install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
endforeach()
target_include_directories(
${module_proj}
PRIVATE
$<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
)
target_include_directories(
${module_proj}
PUBLIC
${Python3_INCLUDE_DIRS}
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
)
install(
TARGETS ${module_proj}
EXPORT ${module_proj}Targets
RUNTIME
DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY
DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE
DESTINATION ${CMAKE_INSTALL_LIBDIR}
)
install(
EXPORT ${module_proj}Targets
FILE ${module_proj}Targets.cmake
NAMESPACE ${PROJECT_NAME}::
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
)
# 3) python_md_providers
set(providers_proj ${PROJECT_NAME}_providers)
file(
GLOB_RECURSE providers
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
${providers_proj}/*.py)
set(provider_outs)
foreach(_provider ${providers})
set(_provider_in ${CMAKE_CURRENT_SOURCE_DIR}/${_provider})
set(_provider_out ${CMAKE_CURRENT_BINARY_DIR}/${_provider})
add_custom_command(
OUTPUT ${_provider_out}
COMMAND ${CMAKE_COMMAND} -E copy ${_provider_in} ${_provider_out}
DEPENDS ${_provider_in}
COMMENT "Copying md provider plugin ${_provider} to ${_provider_out}"
)
list(APPEND provider_outs ${_provider_out})
endforeach()
add_custom_target(copy_${providers_proj} ALL DEPENDS ${provider_outs})
add_library(${providers_proj} INTERFACE)
add_library(${PROJECT_NAME}::${providers_proj} ALIAS ${providers_proj})
set_target_properties(
${providers_proj}
PROPERTIES
INTERFACE_PYTHON_SCRIPTS_DIR
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${providers_proj}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${providers_proj}>
)
set_target_properties(
${providers_proj}
PROPERTIES
INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${providers}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${providers}>
)
install(
TARGETS ${providers_proj}
EXPORT ${providers_proj}Targets
DESTINATION ${CMAKE_INSTALL_BINDIR}
)
install(
EXPORT ${providers_proj}Targets
FILE ${providers_proj}Targets.cmake
NAMESPACE ${PROJECT_NAME}::
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
)
install(DIRECTORY ${providers_proj} DESTINATION ${CMAKE_INSTALL_BINDIR})
+12
View File
@@ -0,0 +1,12 @@
@PACKAGE_INIT@
include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@_moduleTargets.cmake")
include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@_providersTargets.cmake")
find_dependency(Python3 REQUIRED COMPONENTS Interpreter Development)
find_dependency(pybind11)
check_required_components(@PROJECT_NAME@)
check_required_components(@PROJECT_NAME@_module)
check_required_components(@PROJECT_NAME@_providers)
+54
View File
@@ -0,0 +1,54 @@
<!-- Links -->
[pybind11]: https://github.com/pybind/pybind11
[lib_spark]: ../lib_spark/README.md
# lib_python_md
This library binds c++ market data providers to python modules
which can be loaded at runtime.
## **Dependencies**
* **[Pybind11][pybind11]**
* **[lib_spark][lib_spark]**
## Example
```python
import python_md_module as pymd
class MdProviderExample(pymd.MdProvider):
def __init__(self):
super(MdProviderExample, self).__init__()
pass
def start(self, sink: pymd.MdSink):
print("Starting MdProviderExample")
# An actual implementation would subscribe to market data here.
# This is just an example to show you how to pass market data
# back to the c++ Spark api.
def add_dummy_security():
# This example populates a dummy security for demonstrative purposes.
# Of course, an actual implementation would populate the security list
# with data received from the exchange (or some other provider).
sec_list = pymd.PySecurityList()
sec = pymd.PySecurity()
sec.set_venue("GLBX")
sec.set_name("ESH5")
sec.set_asset("ES")
sec.set_expiration("2025-03-28")
sec_list.append(sec)
# Note: the example only adds 1 sec, but batch adds are supported.
sink.on_securities(sec_list)
add_dummy_security()
def stop(self):
print("Stopping MdProviderExample")
```
## Devs
Python files in `python_md_providers/` (child dir of this dir) are copied to<br/>
the plugins directory at build time, and should automatically be loaded at runtime.
@@ -0,0 +1,7 @@
#include "python_md/md_provider.h"
namespace bts::md::py_interop {
MdProvider::~MdProvider() = default;
} // namespace bts::md::py_interop
+14
View File
@@ -0,0 +1,14 @@
#pragma once
#include "python_md/md_sink.h"
namespace bts::md::py_interop {
class MdProvider {
public:
virtual ~MdProvider();
virtual void start(MdSink *sink) = 0;
virtual void stop() = 0;
};
} // namespace bts::md::py_interop
+7
View File
@@ -0,0 +1,7 @@
#include "md_sink.h"
namespace bts::md::py_interop {
MdSink::~MdSink() = default;
} // namespace bts::md::py_interop
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include "python_md/py_book_list.h"
#include "python_md/py_security_list.h"
#include "python_md/py_trade_list.h"
namespace bts::md::py_interop {
class MdSink {
public:
virtual ~MdSink();
virtual void on_securities(const py_interop::PySecurityList &securities) = 0;
virtual void on_books(const py_interop::PyBookList &books) = 0;
virtual void on_trades(const py_interop::PyTradeList &trades) = 0;
};
} // namespace bts::md::py_interop
@@ -0,0 +1,73 @@
#include "python_md/md_sink_impl.h"
#include "python_md/py_security_list.h"
#include "python_md/py_trade_list.h"
#include <exception>
#include <iostream>
namespace bts::md::py_interop {
struct MdSinkImpl::PImpl {
std::function<void(const py_interop::PySecurityList &)> securities_callback;
std::function<void(const py_interop::PyBookList &)> books_callback;
std::function<void(const py_interop::PyTradeList &)> trades_callback;
};
MdSinkImpl::MdSinkImpl() : me_(std::make_unique<PImpl>()) {
// TODO
}
MdSinkImpl::MdSinkImpl(MdSinkImpl &&) = default;
MdSinkImpl &MdSinkImpl::operator=(MdSinkImpl &&) = default;
MdSinkImpl::~MdSinkImpl() = default;
void MdSinkImpl::on_books(const py_interop::PyBookList &books) {
if (books.items().empty()) {
std::cerr << "'on_books' called with empty list" << std::endl;
return;
}
if (me_->books_callback) {
me_->books_callback(books);
}
}
void MdSinkImpl::on_securities(const py_interop::PySecurityList &securities) {
try {
if (securities.items().empty()) {
std::cerr << "'on_securities' called with empty list" << std::endl;
return;
}
if (me_->securities_callback) {
me_->securities_callback(securities);
}
} catch (const std::exception &ex) {
std::cerr << "MdSinkImpl.on_securities threw: " << ex.what() << std::endl;
}
}
void MdSinkImpl::on_trades(const py_interop::PyTradeList &trades) {
if (trades.items().empty()) {
std::cout << "'on_trades' called with empty list" << std::endl;
return;
}
if (me_->trades_callback) {
me_->trades_callback(trades);
}
}
void MdSinkImpl::set_books_callback(
std::function<void(const py_interop::PyBookList &)> callback) {
me_->books_callback = callback;
}
void MdSinkImpl::set_trades_callback(
std::function<void(const py_interop::PyTradeList &)> callback) {
me_->trades_callback = callback;
}
void MdSinkImpl::set_securities_callback(
std::function<void(const py_interop::PySecurityList &)> callback) {
me_->securities_callback = callback;
}
} // namespace bts::md::py_interop
@@ -0,0 +1,37 @@
#pragma once
#include "python_md/md_sink.h"
#include "python_md/py_book_list.h"
#include "python_md/py_security_list.h"
#include "python_md/py_trade_list.h"
#include <functional>
#include <memory>
namespace bts::md::py_interop {
class MdSinkImpl : public MdSink {
public:
MdSinkImpl();
MdSinkImpl(MdSinkImpl &&);
~MdSinkImpl() override;
MdSinkImpl &operator=(MdSinkImpl &&);
void on_books(const py_interop::PyBookList &books) override;
void on_trades(const py_interop::PyTradeList &trades) override;
void on_securities(const py_interop::PySecurityList &securities) override;
void set_books_callback(
std::function<void(const py_interop::PyBookList &)> callback);
void set_trades_callback(
std::function<void(const py_interop::PyTradeList &)> callback);
void set_securities_callback(
std::function<void(const py_interop::PySecurityList &)> callback);
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md::py_interop
+97
View File
@@ -0,0 +1,97 @@
#pragma once
#include "python_md/py_book_side.h"
#include <sstream>
#include <string>
namespace bts::md::py_interop {
/**
* @brief Represents the L2 Order Book for a particular security, at a particular venue.
*/
class PyBook {
public:
/**
* @brief Gets the name of the venue (i.e., exchange).
*
* @return const std::string& The name of the venue.
*/
const std::string &venue() const;
void set_venue(std::string value);
/**
* @brief Gets the exchange security id. This is typically the security's name
* (e.g., "BTS-PERPETUAL", or "ESH5").
*
* @note Must be unique for each security at a particular venue (i.e., a single
* venue cannot have 2 securities with the same name).
*
* @note There is no "security_id" member because that is an internal id that
* we generate elsewhere in c++ code. The python plugins just need to set the
* venue and exchange_security_id.
*/
const std::string &exchange_security_id() const;
/**
* @brief Sets the exchange security id.
*
* @param value The new exchange security id.
*/
void set_exchange_security_id(std::string value);
PyBookSide &bids();
const PyBookSide &bids() const;
void set_bids(PyBookSide bids);
PyBookSide &asks();
const PyBookSide &asks() const;
void set_asks(PyBookSide asks);
/**
* @brief Gets the number of levels in the book (i.e., the number of prices
* with resting orders).
*/
constexpr int size() const;
std::string to_string() const;
private:
std::string venue_;
std::string exchange_security_id_;
PyBookSide bids_;
PyBookSide asks_;
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline const std::string &PyBook::venue() const { return venue_; }
inline void PyBook::set_venue(std::string value) { venue_ = std::move(value); }
inline const std::string &PyBook::exchange_security_id() const {
return exchange_security_id_;
}
inline void PyBook::set_exchange_security_id(std::string value) {
exchange_security_id_ = std::move(value);
}
inline PyBookSide &PyBook::bids() { return bids_; }
inline const PyBookSide &PyBook::bids() const { return bids_; }
inline void PyBook::set_bids(PyBookSide bids) { bids_ = std::move(bids); }
inline PyBookSide &PyBook::asks() { return asks_; }
inline const PyBookSide &PyBook::asks() const { return asks_; }
inline void PyBook::set_asks(PyBookSide asks) { asks_ = std::move(asks); }
constexpr int PyBook::size() const { return bids_.size() + asks_.size(); }
inline std::string PyBook::to_string() const {
std::stringstream str;
// pseudo json format
str << "bids: [" << bids_.to_string() << "], asks: [" << asks_.to_string()
<< "]";
return str.str();
}
} // namespace bts::md::py_interop
@@ -0,0 +1,42 @@
#pragma once
#include <limits>
#include <sstream>
#include <string>
namespace bts::md::py_interop {
class PyBookLevel {
public:
constexpr PyBookLevel() = default;
constexpr double price() const;
void set_price(double value);
constexpr double quantity() const;
void set_quantity(double value);
std::string to_string() const;
private:
double price_{std::numeric_limits<double>::quiet_NaN()};
double quantity_{0};
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
constexpr double PyBookLevel::price() const { return price_; }
inline void PyBookLevel::set_price(double value) { price_ = value; }
constexpr double PyBookLevel::quantity() const { return quantity_; }
inline void PyBookLevel::set_quantity(double value) { quantity_ = value; }
inline std::string PyBookLevel::to_string() const {
std::stringstream str;
str << price_ << " (" << quantity_ << ")";
return str.str();
}
} // namespace bts::md::py_interop
@@ -0,0 +1,54 @@
#pragma once
#include "python_md/py_book.h"
#include <sstream>
#include <vector>
namespace bts::md::py_interop {
class PyBookList {
public:
void append(const py_interop::PyBook &book);
void clear();
std::vector<py_interop::PyBook> &items();
const std::vector<py_interop::PyBook> &items() const;
constexpr std::size_t size() const;
std::string to_string() const;
private:
std::vector<py_interop::PyBook> items_;
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline void PyBookList::append(const py_interop::PyBook &book) {
items_.push_back(book);
}
inline void PyBookList::clear() { items_.clear(); }
constexpr std::size_t PyBookList::size() const { return items_.size(); }
inline std::string PyBookList::to_string() const {
std::stringstream str;
bool is_first = true;
for (const auto &book : items_) {
if (is_first) {
is_first = false;
} else {
str << "\n";
}
str << book.to_string();
}
return str.str();
}
inline std::vector<py_interop::PyBook> &PyBookList::items() { return items_; }
inline const std::vector<py_interop::PyBook> &PyBookList::items() const {
return items_;
}
} // namespace bts::md::py_interop
+123
View File
@@ -0,0 +1,123 @@
#pragma once
#include "python_md/py_book_level.h"
#include <algorithm>
#include <cmath>
#include <iostream>
#include <vector>
namespace bts::md::py_interop {
/**
* @brief Represents an L2 Order Boo for a particular side of the market (i.e.,
* bids or offers).
*
* @details Bid market levels should be in DESCENDING order by price, while Ask
* levels (offers) should be in ASCENDING order. Another way to think about it
* is the price levels are sorted by their distance from the opposite market.
*/
class PyBookSide {
public:
constexpr bool is_ascending();
void set_is_ascending(bool is_ascending);
constexpr PyBookLevel get_level(int index);
constexpr PyBookLevel get_level(int index) const;
void set_level(PyBookLevel level);
constexpr int size() const;
std::string to_string() const;
private:
std::vector<PyBookLevel> levels_;
bool is_ascending_;
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
constexpr bool PyBookSide::is_ascending() { return is_ascending_; }
inline void PyBookSide::set_is_ascending(bool is_ascending) {
is_ascending_ = is_ascending;
}
inline void PyBookSide::set_level(PyBookLevel level) {
if (std::isnan(level.price())) {
std::cerr << "PyBookLevel invalid price (nan)" << std::endl;
return;
}
if (level.quantity() < 0) {
std::cerr << "PyBookLevel invalid quantity (less than 0)" << std::endl;
return;
}
auto it = std::lower_bound(
levels_.begin(), levels_.end(), level,
[](const auto &lhs, const auto &rhs) {
return lhs.price() < rhs.price();
});
if (it == levels_.end()) {
if (level.quantity() == 0) {
std::cerr << "PyBookLevel " << to_string()
<< " indicates remove op, but the price level does not exist"
<< std::endl;
return;
}
levels_.push_back(std::move(level));
} else if (it->price() == level.price()) {
if (level.quantity() == 0) {
levels_.erase(it);
} else {
*it = std::move(level);
}
} else {
levels_.insert(it, std::move(level));
}
}
constexpr PyBookLevel PyBookSide::get_level(int index) {
if (!is_ascending_) {
index = levels_.size() - 1 - index;
}
return levels_.at(index);
}
constexpr PyBookLevel PyBookSide::get_level(int index) const {
if (!is_ascending_) {
index = levels_.size() - 1 - index;
}
return levels_.at(index);
}
constexpr int PyBookSide::size() const { return levels_.size(); }
inline std::string PyBookSide::to_string() const {
std::string rv;
if (is_ascending_) {
bool is_first = true;
for (const auto &level : levels_) {
if (is_first) {
is_first = false;
} else {
rv.append(", ");
}
rv.append(level.to_string());
}
} else {
bool is_first = true;
for (auto it = levels_.rbegin(); it != levels_.rend(); it++) {
if (is_first) {
is_first = false;
} else {
rv.append(", ");
}
rv.append(it->to_string());
}
}
return rv;
}
} // namespace bts::md::py_interop
+45
View File
@@ -0,0 +1,45 @@
#pragma once
#include <sstream>
#include <string>
namespace bts::md::py_interop {
/**
* @brief Defines the expiration for a Future contract.
*/
class PyFuture {
public:
const std::string &expiration() const;
void set_expiration(std::string value);
std::string to_string() const;
private:
std::string expiration_;
};
std::ostream &operator<<(std::ostream &out, const PyFuture &fut) noexcept;
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline const std::string &PyFuture::expiration() const { return expiration_; }
inline void PyFuture::set_expiration(std::string value) {
expiration_ = std::move(value);
}
inline std::string PyFuture::to_string() const {
std::stringstream str;
str << *this;
return str.str();
}
inline std::ostream &operator<<(
std::ostream &out, const PyFuture &fut) noexcept {
out << fut.expiration();
return out;
}
} // namespace bts::md::py_interop
@@ -0,0 +1,141 @@
#include "py_md_executor.h"
#include "python_md/md_provider.h"
#include "python_md/md_sink.h"
#include "system/string_utils.h"
#include "system/thread_id.h"
#include <mutex>
#include <pybind11/embed.h>
#include <pybind11/pybind11.h>
#include <stdexcept>
#include <string>
#include <thread>
namespace py = pybind11;
using bts::system::string_utils::snake_to_pascal_case;
using bts::system::string_utils::split;
using std::int64_t;
using std::size_t;
using std::string;
namespace bts::md::py_interop {
class Foo {
public:
void dostuff();
void dostuff() const;
};
// Typedefs for member function pointers
typedef void (Foo::*NonConstDostuff)(); // Non-const member function
typedef void (Foo::*ConstDostuff)() const; // Const member function
// Alternatively, using the `using` keyword (modern C++):
using NonConstDostuffPtr = void (Foo::*)();
using ConstDostuffPtr = void (Foo::*)() const;
/**
* Parses module and type name from a \p py_md_provider arg. The following formats are supported:
* 1) <module_name>.<type_name>
* 2) <module_name>
* In this case, the type name defaults to the PascalCase equivalent of the module name.
* For example, the module "foo_bar" would have type name "FooBar".
*/
static void parse_py_md_provider(
const std::string &py_md_provider,
std::string *module_name,
std::string *type_name) {
auto parts = split(py_md_provider, ".");
if (parts.empty()) {
return;
}
if (parts.size() == 1) {
snake_to_pascal_case(parts.emplace_back(parts[0]));
}
*module_name = parts[0];
*type_name = parts[1];
}
struct PyMdExecutor::PImpl {
std::string py_md_provider;
std::vector<std::string> module_paths;
std::unique_ptr<MdProvider> provider;
MdSink *sink;
std::optional<py::object> provider_obj;
bool is_running_ = false;
size_t thread_id = BTS_THIS_THREAD_ID;
std::mutex _instance_mx;
std::optional<py::scoped_interpreter> interpreter{};
std::function<void(const py_interop::PySecurityList &)> securities_callback;
std::function<void(const py_interop::PyBookList &)> books_callback;
std::function<void(const py_interop::PyTradeList &)> trades_callback;
void start() {
check_access();
{
std::lock_guard<std::mutex> lock{_instance_mx};
if (is_running_) {
return;
}
interpreter.emplace();
is_running_ = true;
}
provider_obj = create_provider();
MdProvider *provider = provider_obj->cast<MdProvider *>();
provider->start(sink);
}
void stop() {
check_access();
std::lock_guard<std::mutex> lock{_instance_mx};
if (!provider_obj) {
return;
}
MdProvider *providerx = provider_obj->cast<MdProvider *>();
providerx->stop();
provider_obj = std::nullopt;
interpreter.reset();
}
py::object create_provider() {
string provider_module_name;
string provider_type_name;
parse_py_md_provider(
py_md_provider, &provider_module_name, &provider_type_name);
// Prepare module path
py::module sys = py::module::import("sys");
for (const auto &module_path : module_paths) {
sys.attr("path").attr("append")(module_path.c_str());
}
// Import provider module
py::module provider_module =
py::module::import(provider_module_name.c_str());
// Instantiate provider
return provider_module.attr(provider_type_name.c_str())();
}
void check_access() {
if (BTS_THIS_THREAD_ID != thread_id) {
throw std::runtime_error{
"All calls to PyMdExecutor must be on the same thread."};
}
}
};
PyMdExecutor::PyMdExecutor(
std::string py_md_provider,
std::vector<std::string> module_paths,
MdSink *sink)
: me_(std::make_unique<PImpl>()) {
me_->py_md_provider = std::move(py_md_provider);
me_->module_paths = std::move(module_paths);
me_->sink = sink;
}
PyMdExecutor::~PyMdExecutor() = default;
void PyMdExecutor::start() { me_->start(); }
void PyMdExecutor::stop() { me_->stop(); }
} // namespace bts::md::py_interop
@@ -0,0 +1,25 @@
#pragma once
#include "python_md/md_sink.h"
#include <memory>
namespace bts::md::py_interop {
class PyMdExecutor {
public:
PyMdExecutor(
std::string py_md_provider,
std::vector<std::string> module_paths,
MdSink *sink);
~PyMdExecutor();
void start();
void stop();
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md::py_interop
@@ -0,0 +1,7 @@
#include "py_md_provider.h"
namespace bts::md::py_interop {
PyMdProvider::~PyMdProvider() = default;
} // namespace bts::md::py_interop
@@ -0,0 +1,31 @@
#pragma once
#include "python_md/md_provider.h"
#include "python_md/md_sink.h"
#include <pybind11/pybind11.h>
namespace bts::md::py_interop {
/**
* An implementation of `bts::md::py_interop::MdProvider` that binds to a python object at runtime.
*
* @note The python object must derive from 'python_md_module.MdProvider', and define methods:
* 1. 'start(md_sink: python_md_module.MdSink)'
* 2. 'stop'
* @note 'python_md_module' is a python module defined by us in c++ (see 'py_module.cpp').
*/
class PyMdProvider : public MdProvider {
public:
using MdProvider::MdProvider;
~PyMdProvider() override;
void start(MdSink *sink) override {
PYBIND11_OVERLOAD_PURE(void, MdProvider, start, sink);
}
void stop() override { PYBIND11_OVERLOAD_PURE(void, MdProvider, stop); }
};
} // namespace bts::md::py_interop
+146
View File
@@ -0,0 +1,146 @@
#include "md_provider.h"
#include "python_md/md_sink.h"
#include "python_md/md_sink_impl.h"
#include "python_md/py_book_level.h"
#include "python_md/py_book_list.h"
#include "python_md/py_book_side.h"
#include "python_md/py_md_provider.h"
#include "python_md/py_security.h"
#include "python_md/py_security_list.h"
#include "python_md/py_trade.h"
#include <pybind11/pybind11.h>
using bts::md::py_interop::MdSinkImpl;
using bts::md::py_interop::PyBook;
using bts::md::py_interop::PyBookLevel;
using bts::md::py_interop::PyBookList;
using bts::md::py_interop::PyBookSide;
using bts::md::py_interop::PyFuture;
using bts::md::py_interop::PyOption;
using bts::md::py_interop::PySecurity;
using bts::md::py_interop::PySecurityList;
using bts::md::py_interop::PySpot;
using bts::md::py_interop::PyTrade;
using bts::md::py_interop::PyTradeList;
namespace py = pybind11;
#define _STR(x) #x
// In general, we bind c++ methods to python methods with the same name verbatim.
// The DEF macro avoids typos in the python method name for this common case.
#define DEF(type, member) def(_STR(member), &type::member)
#define EXPLICIT_DEF(type, member, member_type) \
def(_STR(member), (member_type) & type::member)
// The PROP macro defines getter and setter methods according to <name>
// Binds a pseudo property with getter/setter methods.
// Binds getter -> <name>, and setter -> set_<name>.
#define PROP(type, name) \
def(_STR(name), &type::name).def(_STR(set_##name), &type::set_##name)
// namespace blah {
// typedef bts::md::py_interop::PyBookSide & (bts::md::py_interop::PyBook::*bids());
// }
// Define python module with c++ classes.
PYBIND11_MODULE(python_md_module, m) {
using BookSideGetter = PyBookSide &(PyBook::*)();
using BookLevelGetter = PyBookLevel (PyBookSide::*)(int) const;
// security
py::class_<bts::md::py_interop::PySpot>(m, "PySpot")
.def(py::init<>())
.def("__str__", &PySpot::to_string);
py::class_<bts::md::py_interop::PyFuture>(m, "PyFuture")
.def(py::init<>())
.PROP(PyFuture, expiration)
.def("__str__", &PyFuture::to_string);
py::class_<bts::md::py_interop::PyOption>(m, "PyOption")
.def(py::init<>())
.PROP(PyOption, expiration)
.PROP(PyOption, is_call)
.PROP(PyOption, strike)
.def("__str__", &PyOption::to_string);
py::class_<bts::md::py_interop::PySecurity>(m, "PySecurity")
.def(py::init<>())
.PROP(PySecurity, id)
.PROP(PySecurity, exchange_id)
.PROP(PySecurity, venue)
.PROP(PySecurity, name)
.PROP(PySecurity, currency)
.PROP(PySecurity, group)
.PROP(PySecurity, asset)
.PROP(PySecurity, spot)
.PROP(PySecurity, future)
.PROP(PySecurity, option)
.def("__str__", &PySecurity::to_string);
py::class_<PySecurityList>(m, "PySecurityList")
.def(py::init<>())
.DEF(PySecurityList, append)
.DEF(PySecurityList, clear)
.def("__len__", &PySecurityList::size)
.def("__str__", &PySecurityList::to_string);
// book
py::class_<PyBookLevel>(m, "PyBookLevel")
.def(py::init<>())
.PROP(PyBookLevel, price)
.PROP(PyBookLevel, quantity);
py::class_<PyBookSide>(m, "PyBookSide")
.def(py::init<>())
.EXPLICIT_DEF(PyBookSide, get_level, BookLevelGetter)
.DEF(PyBookSide, set_level)
.PROP(PyBookSide, is_ascending)
.def("__len__", &PyBookSide::size)
.def("__str__", &PyBookSide::to_string);
py::class_<bts::md::py_interop::PyBook>(m, "PyBook")
.def(py::init<>())
.PROP(PyBook, venue)
.PROP(PyBook, exchange_security_id)
.EXPLICIT_DEF(PyBook, bids, BookSideGetter)
.DEF(PyBook, set_bids)
.EXPLICIT_DEF(PyBook, asks, BookSideGetter)
.DEF(PyBook, set_asks)
.def("__len__", &PyBook::size)
.def("__str__", &PyBook::to_string);
py::class_<PyBookList>(m, "PyBookList")
.def(py::init<>())
.DEF(PyBookList, append)
.DEF(PyBookList, clear)
.def("__len__", &PyBookList::size)
.def("__str__", &PyBookList::to_string);
// trade
py::class_<PyTrade>(m, "PyTrade")
.def(py::init<>())
.PROP(PyTrade, venue)
.PROP(PyTrade, exchange_security_id)
.PROP(PyTrade, price)
.PROP(PyTrade, quantity)
.PROP(PyTrade, aggressor_side)
.PROP(PyTrade, timestamp);
py::class_<PyTradeList>(m, "PyTradeList")
.def(py::init<>())
.DEF(PyTradeList, append)
.DEF(PyTradeList, clear)
.def("__len__", &PyTradeList::size)
.def("__str__", &PyTradeList::to_string);
// md sink
py::class_<bts::md::py_interop::MdSink>(m, "MdSink")
.DEF(bts::md::py_interop::MdSink, on_securities)
.DEF(bts::md::py_interop::MdSink, on_books)
.DEF(bts::md::py_interop::MdSink, on_trades);
py::class_<bts::md::py_interop::MdSinkImpl, bts::md::py_interop::MdSink>(
m, "MdSinkImpl")
.def(py::init<>())
.DEF(bts::md::py_interop::MdSinkImpl, on_securities)
.DEF(bts::md::py_interop::MdSinkImpl, on_books)
.DEF(bts::md::py_interop::MdSinkImpl, on_trades);
// md provider
py::class_<
bts::md::py_interop::MdProvider, bts::md::py_interop::PyMdProvider>(
m, "MdProvider")
.def(py::init<>())
.DEF(bts::md::py_interop::MdProvider, start)
.DEF(bts::md::py_interop::MdProvider, stop);
}
#undef PROP
#undef DEF
#undef _STR
+68
View File
@@ -0,0 +1,68 @@
#pragma once
#include <limits>
#include <sstream>
#include <string>
namespace bts::md::py_interop {
/**
* @brief Defines the expiration, strike, and call/put for an Option contract.
*/
class PyOption {
public:
constexpr PyOption() = default;
const std::string &expiration() const;
void set_expiration(std::string value);
constexpr bool is_call() const;
void set_is_call(bool value);
constexpr double strike() const;
void set_strike(double value);
std::string to_string() const;
private:
std::string expiration_;
bool is_call_{false};
double strike_{std::numeric_limits<double>::quiet_NaN()};
};
std::ostream &operator<<(std::ostream &out, const PyOption &opt) noexcept;
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline const std::string &PyOption::expiration() const { return expiration_; }
inline void PyOption::set_expiration(std::string value) {
expiration_ = std::move(value);
}
constexpr bool PyOption::is_call() const { return is_call_; }
inline void PyOption::set_is_call(bool value) { is_call_ = value; }
constexpr double PyOption::strike() const { return strike_; }
inline void PyOption::set_strike(double value) { strike_ = value; }
inline std::string PyOption::to_string() const {
std::stringstream str;
str << *this;
return str.str();
}
inline std::ostream &operator<<(
std::ostream &out, const PyOption &opt) noexcept {
out << opt.expiration() << " ";
if (opt.is_call()) {
out << "CALL ";
} else {
out << "PUT ";
}
out << opt.strike();
return out;
}
} // namespace bts::md::py_interop
+136
View File
@@ -0,0 +1,136 @@
#pragma once
#include "python_md/py_future.h"
#include "python_md/py_option.h"
#include "python_md/py_spot.h"
#include <cstdint>
#include <stdexcept>
#include <string>
#include <variant>
namespace bts::md::py_interop {
class PySecurity {
public:
constexpr std::uint32_t id() const;
void set_id(std::uint32_t value);
const std::string &exchange_id() const;
void set_exchange_id(std::string value);
const std::string &venue() const;
void set_venue(std::string value);
const std::string &currency() const;
void set_currency(std::string value);
const std::string &group() const;
void set_group(std::string value);
const std::string &asset() const;
void set_asset(std::string value);
const std::string &name() const;
void set_name(std::string value);
const PySpot &spot() const;
void set_spot(PySpot);
constexpr bool is_spot() const;
const PyFuture &future() const;
void set_future(PyFuture);
constexpr bool is_future() const;
const PyOption &option() const;
void set_option(PyOption);
constexpr bool is_option() const;
std::string to_string() const;
private:
std::uint32_t id_;
std::string exchange_id_;
std::string venue_;
std::string group_;
std::string asset_;
std::string name_;
std::string currency_;
std::variant<PySpot, PyFuture, PyOption> sec_;
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
constexpr std::uint32_t PySecurity::id() const { return id_; }
inline void PySecurity::set_id(std::uint32_t value) { id_ = value; }
inline const std::string &PySecurity::exchange_id() const {
return exchange_id_;
}
inline void PySecurity::set_exchange_id(std::string value) {
exchange_id_ = std::move(value);
}
inline const std::string &PySecurity::venue() const { return venue_; }
inline void PySecurity::set_venue(std::string value) {
venue_ = std::move(value);
}
inline const std::string &PySecurity::currency() const { return currency_; }
inline void PySecurity::set_currency(std::string value) {
currency_ = std::move(value);
}
inline const std::string &PySecurity::group() const { return group_; }
inline void PySecurity::set_group(std::string value) {
group_ = std::move(value);
}
inline const std::string &PySecurity::asset() const { return asset_; }
inline void PySecurity::set_asset(std::string value) {
asset_ = std::move(value);
}
inline const std::string &PySecurity::name() const { return name_; }
inline void PySecurity::set_name(std::string value) {
name_ = std::move(value);
}
inline const PySpot &PySecurity::spot() const {
if (std::holds_alternative<PySpot>(sec_)) {
return std::get<PySpot>(sec_);
}
throw std::runtime_error{"PySecurity is not a spot"};
}
inline void PySecurity::set_spot(PySpot value) { sec_ = std::move(value); }
constexpr bool PySecurity::is_spot() const {
return std::holds_alternative<PySpot>(sec_);
}
inline const PyFuture &PySecurity::future() const {
if (std::holds_alternative<PyFuture>(sec_)) {
return std::get<PyFuture>(sec_);
}
throw std::runtime_error{"PySecurity is not a future"};
}
inline void PySecurity::set_future(PyFuture value) { sec_ = std::move(value); }
constexpr bool PySecurity::is_future() const {
return std::holds_alternative<PyFuture>(sec_);
}
inline const PyOption &PySecurity::option() const {
if (std::holds_alternative<PyOption>(sec_)) {
return std::get<PyOption>(sec_);
}
throw std::runtime_error{"PySecurity is not an option"};
}
inline void PySecurity::set_option(PyOption value) { sec_ = std::move(value); }
constexpr bool PySecurity::is_option() const {
return std::holds_alternative<PyOption>(sec_);
}
inline std::string PySecurity::to_string() const { return name_; }
} // namespace bts::md::py_interop
@@ -0,0 +1,56 @@
#pragma once
#include "python_md/py_security.h"
#include <sstream>
#include <vector>
namespace bts::md::py_interop {
class PySecurityList {
public:
void append(const py_interop::PySecurity &security);
void clear();
const std::vector<py_interop::PySecurity> &items() const;
constexpr std::size_t size() const;
std::string to_string() const;
private:
std::vector<py_interop::PySecurity> items_;
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline void PySecurityList::append(const py_interop::PySecurity &security) {
items_.push_back(security);
}
inline void PySecurityList::clear() { items_.clear(); }
constexpr std::size_t PySecurityList::size() const {
return items_.size();
}
inline const std::vector<py_interop::PySecurity> &
PySecurityList::items() const {
return items_;
}
inline std::string PySecurityList::to_string() const {
std::stringstream str;
bool is_first = true;
for (const auto &sec : items_) {
if (is_first) {
is_first = false;
} else {
str << "\n";
}
str << sec.to_string();
}
return str.str();
}
} // namespace bts::md::py_interop
+32
View File
@@ -0,0 +1,32 @@
#pragma once
#include <string>
namespace bts::md::py_interop {
/**
* @brief A placeholder object that represents a Spot contract.
*
* @details Unlike the similarly named types `PyFuture` and `PyOption`, this class has no members.
* A Spot contract's properties are universal to all security types in our system, and are thus all
* defined in `PySecurity`. It exists so we can follow the "one of" paradigm per security type.
*/
class PySpot {
public:
std::string to_string() const;
};
std::ostream &operator<<(std::ostream &out, const PySpot &spot) noexcept;
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline std::string PySpot::to_string() const { return ""; }
inline std::ostream &operator<<(
std::ostream &out, const PySpot & /*spot*/) noexcept {
return out;
}
} // namespace bts::md::py_interop
+86
View File
@@ -0,0 +1,86 @@
#pragma once
#include <cstdint>
#include <iostream>
#include <sstream>
#include <string>
namespace bts::md::py_interop {
class PyTrade {
public:
const std::string &venue() const;
void set_venue(std::string value);
const std::string &exchange_security_id() const;
void set_exchange_security_id(std::string value);
constexpr double price() const;
void set_price(double value);
constexpr double quantity() const;
void set_quantity(double value);
const std::string &aggressor_side() const;
void set_aggressor_side(std::string value);
/**
* @brief The time at which the trade was executed. Expressed as the number of nanoseconds since
* the unix epoch: Jan 1, 1970 00:00:00 UTC.
*
* @return constexpr std::int64_t The time at which the trade was executed.
*/
constexpr std::int64_t timestamp() const;
void set_timestamp(std::int64_t value);
std::string to_string() const;
private:
std::string venue_;
std::string exchange_security_id_;
double price_;
double quantity_;
std::string aggressor_side_;
std::int64_t timestamp_;
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline const std::string &PyTrade::venue() const { return venue_; }
inline void PyTrade::set_venue(std::string value) { venue_ = std::move(value); }
inline const std::string &PyTrade::exchange_security_id() const {
return exchange_security_id_;
}
inline void PyTrade::set_exchange_security_id(std::string value) {
exchange_security_id_ = std::move(value);
}
constexpr double PyTrade::price() const { return price_; }
inline void PyTrade::set_price(double value) { price_ = value; }
constexpr double PyTrade::quantity() const { return quantity_; }
inline void PyTrade::set_quantity(double value) { quantity_ = value; }
inline const std::string &PyTrade::aggressor_side() const {
return aggressor_side_;
}
inline void PyTrade::set_aggressor_side(std::string value) {
aggressor_side_ = std::move(value);
}
constexpr std::int64_t PyTrade::timestamp() const { return timestamp_; }
inline void PyTrade::set_timestamp(std::int64_t value) { timestamp_ = value; }
inline std::string PyTrade::to_string() const {
std::stringstream str;
if (!aggressor_side_.empty()) {
str << aggressor_side_ << " ";
}
str << quantity_ << " @ " << price_ << " (ts " << timestamp_ << ")";
return str.str();
}
} // namespace bts::md::py_interop
@@ -0,0 +1,52 @@
#pragma once
#include "python_md/py_trade.h"
#include <sstream>
#include <vector>
namespace bts::md::py_interop {
class PyTradeList {
public:
void append(const py_interop::PyTrade &trade);
void clear();
const std::vector<py_interop::PyTrade> &items() const;
constexpr std::size_t size() const;
std::string to_string() const;
private:
std::vector<py_interop::PyTrade> items_;
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline void PyTradeList::append(const py_interop::PyTrade &trade) {
items_.push_back(trade);
}
inline void PyTradeList::clear() { items_.clear(); }
constexpr std::size_t PyTradeList::size() const { return items_.size(); }
inline const std::vector<py_interop::PyTrade> &PyTradeList::items() const {
return items_;
}
inline std::string PyTradeList::to_string() const {
std::stringstream str;
bool is_first = true;
for (const auto &sec : items_) {
if (is_first) {
is_first = false;
} else {
str << "\n";
}
str << sec.to_string();
}
return str.str();
}
} // namespace bts::md::py_interop
@@ -0,0 +1,122 @@
import python_md_module as pymd
from typing import Optional
def _get_security(
venue: str,
name: str,
asset: str,
group: str,
currency: str,
expiration: Optional[str] = None,
option_kind: Optional[str] = None,
strike: Optional[float] = None,
):
sec = pymd.PySecurity()
sec.set_venue(venue)
sec.set_exchange_id(name)
sec.set_name(name)
sec.set_currency(currency)
sec.set_asset(asset)
sec.set_group(group)
is_spot = expiration is None and option_kind is None and strike is None
is_future = expiration is not None and option_kind is None and strike is None
is_option = (
expiration is not None and option_kind is not None and strike is not None
)
if is_spot:
sec.set_spot(pymd.PySpot())
elif is_future:
fut = pymd.PyFuture()
fut.set_expiration(expiration)
sec.set_future(fut)
elif is_option:
opt = pymd.PyOption()
opt.set_expiration(expiration)
opt.set_is_call(option_kind.lower().startswith("c"))
opt.set_strike(strike)
sec.set_option(opt)
else:
raise ValueError("Invalid parameters (unable to deduce security type)")
return sec
def _get_book_side(levels: list[(float, float)], is_bid_side: bool):
book_side = pymd.PyBookSide()
book_side.set_is_ascending(not is_bid_side)
for price, qty in levels:
book_level = pymd.PyBookLevel()
book_level.set_price(price)
book_level.set_quantity(qty)
book_side.set_level(book_level)
return book_side
def _get_trade(
venue: str,
exchange_security_id: str,
price: float,
quantity: float,
aggressor_side: str,
):
trade = pymd.PyTrade()
trade.set_venue(venue)
trade.set_exchange_security_id(exchange_security_id)
trade.set_price(price)
trade.set_quantity(quantity)
trade.set_aggressor_side(aggressor_side)
return trade
class DummyMdProvider(pymd.MdProvider):
def __init__(self):
super(DummyMdProvider, self).__init__()
pass
def start(self, sink: pymd.MdSink):
# start subscriptions, bind callbacks etc.
def add_fake_securities():
sec_list = pymd.PySecurityList()
sec_list.append(
_get_security(
venue="Glbx",
name="ESH5",
asset="ES",
group="ES",
currency="USD",
expiration="2025-03-28",
)
)
sink.on_securities(sec_list)
def add_fake_books():
bids = _get_book_side([(99.75, 8), (99.50, 11)], is_bid_side=True)
asks = _get_book_side([(100.25, 3)], is_bid_side=False)
book = pymd.PyBook()
book.set_venue("Glbx")
book.set_exchange_security_id("ESH5")
book.set_bids(bids)
book.set_asks(asks)
# pass book to sink
book_list = pymd.PyBookList()
book_list.append(book)
sink.on_books(book_list)
def add_fake_trades():
trade = _get_trade(
venue="Glbx",
exchange_security_id="ESH5",
price=100,
quantity=2,
aggressor_side="Buy",
)
trade_list = pymd.PyTradeList()
trade_list.append(trade)
sink.on_trades(trade_list)
add_fake_securities()
add_fake_books()
add_fake_trades()
def stop(self):
print("Stopping python MdProviderImpl")
@@ -0,0 +1,31 @@
from typing import Optional
import python_md_module as pymd
def get_book_side(levels: list[(float, float)], is_bid_side: bool):
book_side = pymd.PyBookSide()
book_side.set_is_ascending(not is_bid_side)
for price, qty in levels:
book_level = pymd.PyBookLevel()
book_level.set_price(price)
book_level.set_quantity(qty)
book_side.set_level(book_level)
return book_side
def get_trade(
price: Optional[float] = None,
quantity: Optional[float] = None,
aggressor: Optional[str] = None,
security_id: Optional[int] = None,
):
trade = pymd.PyTrade()
if price is not None:
trade.set_price(100)
if quantity is not None:
trade.set_quantity(2)
if aggressor is not None:
trade.set_aggressor(aggressor)
if security_id is not None:
trade.set_security_id(security_id)
return trade