improvement: Set PyBook data in single call to reduce interop overhead

This commit is contained in:
John Colvin committed 2025-06-09 05:58:37 +00:00
1 parent ccc8faf353
commit 8a5fcfbe04
6 files changed
+98 -17

No files matched your search

+1 -1
View File
@@ -77,7 +77,7 @@ target_include_directories(
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${PROJECT_NAME}.pyi
DEPENDS ${PROJECT_NAME} ${module_proj}
COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen ${PROJECT_NAME}_module
COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var ${PROJECT_NAME}_module
COMMENT "Generating py stubs"
)
add_custom_target(
+51
View File
@@ -0,0 +1,51 @@
#include "py_book.h"
#include "python_md/py_book_level.h"
namespace bts::md::py_interop {
static const void validate_depth_array(const pybind11::buffer_info &info) {
if (info.ndim != 2 || info.shape[0] < 0 || info.shape[1] != 2) {
throw std::runtime_error(fmt::format(
"Expected array of shape (10, 2). Actually got ({}, {}, {})",
info.ndim, info.shape[0], info.shape[1]));
}
}
static void to_book_side(
PyBookSide &dest, const PyBook::PyDepthArray &src, std::size_t count) {
auto r = src.unchecked<2>();
PyBookLevel lvl;
for (std::size_t i = 0; i < count; i++) {
double px = r(i, 0);
double qty = r(i, 1);
if (std::isnan(px) || !(qty > 0)) {
break;
}
lvl.set_price(r(i, 0));
lvl.set_quantity(r(i, 1));
dest.set_level(lvl);
}
}
void PyBook::set_data(
std::string exchange_security_id,
std::string venue,
std::string security_type,
PyDepthArray bids,
PyDepthArray asks) {
const auto bids_info = bids.request();
const auto asks_info = asks.request();
validate_depth_array(bids_info);
validate_depth_array(asks_info);
exchange_security_id_ = std::move(exchange_security_id);
venue_ = std::move(venue);
security_type_ = std::move(security_type);
bids_.set_is_ascending(false);
asks_.set_is_ascending(true);
const size_t n_bids = bids_info.size / 2;
const size_t n_asks = asks_info.size / 2;
to_book_side(bids_, bids, n_bids);
to_book_side(asks_, asks, n_asks);
}
} // namespace bts::md::py_interop
+22
View File
@@ -1,9 +1,15 @@
#pragma once
#include "python_md/py_book_level.h"
#include "python_md/py_book_side.h"
#include "spark/core/FMT_IMPL.h"
#include "spark/io/json_formatter.h"
#include "spark/utils/bts_assert.h"
#include <cmath>
#include <limits>
#include <pybind11/numpy.h>
#include <pybind11/pybind11.h>
#include <sstream>
#include <string>
@@ -14,6 +20,11 @@ namespace bts::md::py_interop {
*/
class PyBook {
public:
// A more performant alternative to PyBookSide. Populate bid/ask numpy arrays
// in python, then init everything in a single call (set_data).
using PyDepthArray = pybind11::
array_t<double, pybind11::array::c_style | pybind11::array::forcecast>;
/**
* Gets the exchange security id. This is typically the security's name
* (e.g., "BTS-PERPETUAL", or "ESH5").
@@ -63,6 +74,17 @@ class PyBook {
const PyBookSide &asks() const;
void set_asks(PyBookSide asks);
/**
* Sets all data at once (more performant than setting props one at a time
* when calling from python).
*/
void set_data(
std::string exchange_security_id,
std::string venue,
std::string security_type,
PyDepthArray bids,
PyDepthArray asks);
/**
* @brief Gets the number of levels in the book (i.e., the number of prices
* with resting orders).
@@ -73,6 +73,10 @@ PYBIND11_MODULE(python_md_module, m) {
.DEF(PyBook, set_bids)
.EXPLICIT_DEF(PyBook, asks, BookSideGetter)
.DEF(PyBook, set_asks)
.def(
"set_data", &PyBook::set_data, py::arg("exch_sec_id"),
py::arg("venue"), py::arg("sec_type"), py::arg("bids"),
py::arg("asks"))
.def("__len__", &PyBook::size)
.def("__str__", &PyBook::to_string);
py::bind_vector<std::vector<shared_ptr<PyBook>>>(m, "PyBookList");
@@ -1,4 +1,5 @@
import json
import numpy as np
import time
import logging
import sys
@@ -195,19 +196,20 @@ class BybitMdProvider(pymd.MdProvider):
# Sort and take top 5, padding with [float('nan'), float('nan')] if needed
sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5]
sorted_bids += [[float('nan'), float('nan')]] * (5 - len(sorted_bids))
sorted_bids += [(np.float64('nan'), np.float64('nan'))]
sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5]
sorted_asks += [[float('nan'), float('nan')]] * (5 - len(sorted_asks))
sorted_asks += [(np.float64('nan'), np.float64('nan'))]
logging.debug(f"📘 BOOK DATA for {contract}: bids={sorted_bids} asks={sorted_asks}")
book = pymd.PyBook()
book.set_venue("Bybit")
book.set_security_type("SPOT")
book.set_exchange_security_id(contract)
book.set_bids(get_book_side(sorted_bids, True))
book.set_asks(get_book_side(sorted_asks, False))
book.set_data(
contract,
"Bybit",
"SPOT",
np.array(sorted_bids, dtype=np.float64),
np.array(sorted_asks, dtype=np.float64),
)
book_list = pymd.PyBookList()
book_list.append(book)
if self.bridge:
@@ -1,4 +1,5 @@
import json
import numpy as np
import time
import logging
import sys
@@ -173,17 +174,18 @@ class GateIoMdProvider(pymd.MdProvider):
payload = data.get("result", {})
if isinstance(payload, dict) and ("bids" in payload or "asks" in payload):
contract = payload.get("contract", "<no-contract>")
bids = [(float(l["p"]), float(l["s"])) for l in payload.get("bids", [])]
asks = [(float(l["p"]), float(l["s"])) for l in payload.get("asks", [])]
bids = [(np.float64(l["p"]), np.float64(l["s"])) for l in payload.get("bids", [])]
asks = [(np.float64(l["p"]), np.float64(l["s"])) for l in payload.get("asks", [])]
logging.debug(f"📘 BOOK DATA for {contract}: bids={bids} asks={asks}")
book = pymd.PyBook()
book.set_venue("GateIO")
book.set_security_type("SPOT")
book.set_exchange_security_id(contract)
book.set_bids(get_book_side(bids, True))
book.set_asks(get_book_side(asks, False))
book.set_data(
contract,
"GateIO",
"SPOT",
np.array(bids, dtype=np.float64),
np.array(asks, dtype=np.float64),
)
book_list = pymd.PyBookList()
book_list.append(book)
self.bridge.on_books(book_list)