From 8a5fcfbe04fa5881365a07cdbd84b245f0c59078 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sat, 31 May 2025 15:58:38 +0000 Subject: [PATCH] improvement: Set PyBook data in single call to reduce interop overhead --- src/lib_python_md/CMakeLists.txt | 2 +- src/lib_python_md/python_md/py_book.cpp | 51 +++++++++++++++++++ src/lib_python_md/python_md/py_book.h | 22 ++++++++ src/lib_python_md/python_md/py_module.cpp | 4 ++ .../python_md_providers/bybit_md_provider.py | 18 ++++--- .../python_md_providers/gateio_md_provider.py | 18 ++++--- 6 files changed, 98 insertions(+), 17 deletions(-) create mode 100644 src/lib_python_md/python_md/py_book.cpp diff --git a/src/lib_python_md/CMakeLists.txt b/src/lib_python_md/CMakeLists.txt index 9cb509bf5..37081be21 100644 --- a/src/lib_python_md/CMakeLists.txt +++ b/src/lib_python_md/CMakeLists.txt @@ -77,7 +77,7 @@ target_include_directories( add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${PROJECT_NAME}.pyi DEPENDS ${PROJECT_NAME} ${module_proj} - COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen ${PROJECT_NAME}_module + COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var ${PROJECT_NAME}_module COMMENT "Generating py stubs" ) add_custom_target( diff --git a/src/lib_python_md/python_md/py_book.cpp b/src/lib_python_md/python_md/py_book.cpp new file mode 100644 index 000000000..180ce1ff5 --- /dev/null +++ b/src/lib_python_md/python_md/py_book.cpp @@ -0,0 +1,51 @@ +#include "py_book.h" +#include "python_md/py_book_level.h" + +namespace bts::md::py_interop { + +static const void validate_depth_array(const pybind11::buffer_info &info) { + if (info.ndim != 2 || info.shape[0] < 0 || info.shape[1] != 2) { + throw std::runtime_error(fmt::format( + "Expected array of shape (10, 2). Actually got ({}, {}, {})", + info.ndim, info.shape[0], info.shape[1])); + } +} + +static void to_book_side( + PyBookSide &dest, const PyBook::PyDepthArray &src, std::size_t count) { + auto r = src.unchecked<2>(); + PyBookLevel lvl; + for (std::size_t i = 0; i < count; i++) { + double px = r(i, 0); + double qty = r(i, 1); + if (std::isnan(px) || !(qty > 0)) { + break; + } + lvl.set_price(r(i, 0)); + lvl.set_quantity(r(i, 1)); + dest.set_level(lvl); + } +} + +void PyBook::set_data( + std::string exchange_security_id, + std::string venue, + std::string security_type, + PyDepthArray bids, + PyDepthArray asks) { + const auto bids_info = bids.request(); + const auto asks_info = asks.request(); + validate_depth_array(bids_info); + validate_depth_array(asks_info); + exchange_security_id_ = std::move(exchange_security_id); + venue_ = std::move(venue); + security_type_ = std::move(security_type); + bids_.set_is_ascending(false); + asks_.set_is_ascending(true); + const size_t n_bids = bids_info.size / 2; + const size_t n_asks = asks_info.size / 2; + to_book_side(bids_, bids, n_bids); + to_book_side(asks_, asks, n_asks); +} + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_book.h b/src/lib_python_md/python_md/py_book.h index 8c1f1a0cf..a78c1fe4e 100644 --- a/src/lib_python_md/python_md/py_book.h +++ b/src/lib_python_md/python_md/py_book.h @@ -1,9 +1,15 @@ #pragma once +#include "python_md/py_book_level.h" #include "python_md/py_book_side.h" #include "spark/core/FMT_IMPL.h" #include "spark/io/json_formatter.h" +#include "spark/utils/bts_assert.h" +#include +#include +#include +#include #include #include @@ -14,6 +20,11 @@ namespace bts::md::py_interop { */ class PyBook { public: + // A more performant alternative to PyBookSide. Populate bid/ask numpy arrays + // in python, then init everything in a single call (set_data). + using PyDepthArray = pybind11:: + array_t; + /** * Gets the exchange security id. This is typically the security's name * (e.g., "BTS-PERPETUAL", or "ESH5"). @@ -63,6 +74,17 @@ class PyBook { const PyBookSide &asks() const; void set_asks(PyBookSide asks); + /** + * Sets all data at once (more performant than setting props one at a time + * when calling from python). + */ + void set_data( + std::string exchange_security_id, + std::string venue, + std::string security_type, + PyDepthArray bids, + PyDepthArray asks); + /** * @brief Gets the number of levels in the book (i.e., the number of prices * with resting orders). diff --git a/src/lib_python_md/python_md/py_module.cpp b/src/lib_python_md/python_md/py_module.cpp index b139ecff0..5bf07cf48 100644 --- a/src/lib_python_md/python_md/py_module.cpp +++ b/src/lib_python_md/python_md/py_module.cpp @@ -73,6 +73,10 @@ PYBIND11_MODULE(python_md_module, m) { .DEF(PyBook, set_bids) .EXPLICIT_DEF(PyBook, asks, BookSideGetter) .DEF(PyBook, set_asks) + .def( + "set_data", &PyBook::set_data, py::arg("exch_sec_id"), + py::arg("venue"), py::arg("sec_type"), py::arg("bids"), + py::arg("asks")) .def("__len__", &PyBook::size) .def("__str__", &PyBook::to_string); py::bind_vector>>(m, "PyBookList"); diff --git a/src/lib_python_md/python_md_providers/bybit_md_provider.py b/src/lib_python_md/python_md_providers/bybit_md_provider.py index b57d987b6..b3ca44f04 100644 --- a/src/lib_python_md/python_md_providers/bybit_md_provider.py +++ b/src/lib_python_md/python_md_providers/bybit_md_provider.py @@ -1,4 +1,5 @@ import json +import numpy as np import time import logging import sys @@ -195,19 +196,20 @@ class BybitMdProvider(pymd.MdProvider): # Sort and take top 5, padding with [float('nan'), float('nan')] if needed sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5] - sorted_bids += [[float('nan'), float('nan')]] * (5 - len(sorted_bids)) + sorted_bids += [(np.float64('nan'), np.float64('nan'))] sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5] - sorted_asks += [[float('nan'), float('nan')]] * (5 - len(sorted_asks)) + sorted_asks += [(np.float64('nan'), np.float64('nan'))] logging.debug(f"📘 BOOK DATA for {contract}: bids={sorted_bids} asks={sorted_asks}") book = pymd.PyBook() - book.set_venue("Bybit") - book.set_security_type("SPOT") - book.set_exchange_security_id(contract) - book.set_bids(get_book_side(sorted_bids, True)) - book.set_asks(get_book_side(sorted_asks, False)) - + book.set_data( + contract, + "Bybit", + "SPOT", + np.array(sorted_bids, dtype=np.float64), + np.array(sorted_asks, dtype=np.float64), + ) book_list = pymd.PyBookList() book_list.append(book) if self.bridge: diff --git a/src/lib_python_md/python_md_providers/gateio_md_provider.py b/src/lib_python_md/python_md_providers/gateio_md_provider.py index 4aac682a4..958f20e71 100644 --- a/src/lib_python_md/python_md_providers/gateio_md_provider.py +++ b/src/lib_python_md/python_md_providers/gateio_md_provider.py @@ -1,4 +1,5 @@ import json +import numpy as np import time import logging import sys @@ -173,17 +174,18 @@ class GateIoMdProvider(pymd.MdProvider): payload = data.get("result", {}) if isinstance(payload, dict) and ("bids" in payload or "asks" in payload): contract = payload.get("contract", "") - bids = [(float(l["p"]), float(l["s"])) for l in payload.get("bids", [])] - asks = [(float(l["p"]), float(l["s"])) for l in payload.get("asks", [])] + bids = [(np.float64(l["p"]), np.float64(l["s"])) for l in payload.get("bids", [])] + asks = [(np.float64(l["p"]), np.float64(l["s"])) for l in payload.get("asks", [])] logging.debug(f"📘 BOOK DATA for {contract}: bids={bids} asks={asks}") book = pymd.PyBook() - book.set_venue("GateIO") - book.set_security_type("SPOT") - book.set_exchange_security_id(contract) - book.set_bids(get_book_side(bids, True)) - book.set_asks(get_book_side(asks, False)) - + book.set_data( + contract, + "GateIO", + "SPOT", + np.array(bids, dtype=np.float64), + np.array(asks, dtype=np.float64), + ) book_list = pymd.PyBookList() book_list.append(book) self.bridge.on_books(book_list)