diff --git a/CMakeLists.txt b/CMakeLists.txt index 272049335..079941617 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,7 @@ else() # subdirs if (BUILD_SPARKD) add_subdirectory(src/lib_system) + add_subdirectory(src/lib_py_market_data) add_subdirectory(src/app_sparkd) add_subdirectory(src/data_files) endif() diff --git a/CMakePresets.json b/CMakePresets.json index 77c83151b..77e5e237e 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -55,7 +55,7 @@ }, "CMAKE_CXX_STANDARD": "20", "CMAKE_INSTALL_RPATH": ".", - "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd", + "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11", "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef", "EXTERNALS_ROOT": "${sourceDir}/externals/linux", "BUILD_SDK": { diff --git a/src/app_sparkd/CMakeLists.txt b/src/app_sparkd/CMakeLists.txt index 23a2a8aa6..260c71c62 100644 --- a/src/app_sparkd/CMakeLists.txt +++ b/src/app_sparkd/CMakeLists.txt @@ -106,12 +106,38 @@ target_link_directories( # The dirs are relative to sparkd's path, represented by "$ORIGIN". set_target_properties( ${PROJECT_NAME} PROPERTIES - BUILD_RPATH "$ORIGIN/../lib_spark" + BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_md" INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}" ) +# Python interop (for python md providers) +add_dependencies(${PROJECT_NAME} + python_md + python_md_module + python_md_providers + copy_python_md_providers +) +include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake) + +sym_link_local_lib(python_md) +sym_link_file( + TARGET_NAME link_python_md_module + SOURCE ${CMAKE_BINARY_DIR}/src/lib_python_md/python_md_module.cpython-310-x86_64-linux-gnu.so + DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_module.cpython-310-x86_64-linux-gnu.so + INSTALL_DEST ${CMAKE_INSTALL_BINDIR}/ +) +get_target_property(providers python_md_providers INTERFACE_PYTHON_SCRIPTS_DIR) +sym_link_dir( + TARGET_NAME link_python_md_providers + SOURCE ${providers} + DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_providers +) + target_link_libraries( sparkd + system + python_md + python_md_module spark databento::databento ${emumba-deribit-connector-library_LIBRARIES} diff --git a/src/app_sparkd/Config.cmake.in b/src/app_sparkd/Config.cmake.in index 68b8825e2..5fac74e3b 100644 --- a/src/app_sparkd/Config.cmake.in +++ b/src/app_sparkd/Config.cmake.in @@ -2,4 +2,4 @@ include("${CMAKE_CURRENT_LIST_DIR}/sparkdTargets.cmake") -check_required_components(libspark) +check_required_components(sparkd) diff --git a/src/app_sparkd/sparkd/cmds/TestPythonMdProviderCmd.cpp b/src/app_sparkd/sparkd/cmds/TestPythonMdProviderCmd.cpp new file mode 100644 index 000000000..51395c885 --- /dev/null +++ b/src/app_sparkd/sparkd/cmds/TestPythonMdProviderCmd.cpp @@ -0,0 +1,73 @@ +#include "TestPythonMdProviderCmd.h" +#include "python_md/md_sink_impl.h" +#include "python_md/py_md_executor.h" +#include "spark/logging/logging.h" + +#include +#include +#include +#include + +using bts::md::py_interop::MdSinkImpl; +using std::string; + +namespace bts::cmds { + +struct TestPythonMdProviderCmd::Args { + std::string py_md_provider; + std::vector module_paths; +}; + +static void execute( + std::string py_md_provider, std::vector module_paths) { + using namespace std::chrono_literals; + MdSinkImpl sink; + sink.set_securities_callback([](const auto &securities) { + LOG(md, Info, "Received securities: {}", securities.to_string()); + }); + sink.set_books_callback([](const auto &books) { + LOG(md, Info, "Received books: {}", books.to_string()); + }); + sink.set_trades_callback([](const auto &trades) { + LOG(md, Info, "Received trades: {}", trades.to_string()); + }); + try { + md::py_interop::PyMdExecutor py_md_exec{ + std::move(py_md_provider), std::move(module_paths), &sink}; + LOG(md, Info, "Starting py md executor."); + py_md_exec.start(); + LOG(md, Info, "Stopping py md executor."); + py_md_exec.stop(); + int i = 0; + while (true) { + std::cout << "Sleep loop count " << ++i << std::endl; + std::this_thread::sleep_for(3s); + } + } catch (const std::exception &ex) { + LOG(md, Error, "Py md executor threw: {}", string{ex.what()}); + } +} + +TestPythonMdProviderCmd::TestPythonMdProviderCmd() + : CLI::App("Test python md provider plugin", "test-py-md-provider"), + args_(std::make_unique()) { + add_option( + "--provider", args_->py_md_provider, + "Python md provider plugin (py file)"); + add_option( + "--module-path", args_->module_paths, "Module path for python plugin"); + callback([this]() { + if (args_->py_md_provider.empty()) { + args_->py_md_provider = "md_provider_impl.MdProviderImpl"; + } + if (args_->module_paths.empty()) { + args_->module_paths.push_back("."); + args_->module_paths.push_back("./python_md_providers"); + } + execute(std::move(args_->py_md_provider), std::move(args_->module_paths)); + }); +} + +TestPythonMdProviderCmd::~TestPythonMdProviderCmd() = default; + +} // namespace bts::cmds diff --git a/src/app_sparkd/sparkd/cmds/TestPythonMdProviderCmd.h b/src/app_sparkd/sparkd/cmds/TestPythonMdProviderCmd.h new file mode 100644 index 000000000..c50d146dc --- /dev/null +++ b/src/app_sparkd/sparkd/cmds/TestPythonMdProviderCmd.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +namespace bts::cmds { + +/** + * This command is designed to run a quick sanity check on Python md provider C++ interop. + * It instantiates a python md provider (in C++) and calls .start() with a dummy + * sink that prints updates to console, but does not write md to shared memory. + */ +struct TestPythonMdProviderCmd : public CLI::App { + TestPythonMdProviderCmd(); + ~TestPythonMdProviderCmd(); + + private: + struct Args; + std::unique_ptr args_; +}; + +} // namespace bts::cmds diff --git a/src/app_sparkd/sparkd/main.cpp b/src/app_sparkd/sparkd/main.cpp index be88dbbf9..bd2ace949 100644 --- a/src/app_sparkd/sparkd/main.cpp +++ b/src/app_sparkd/sparkd/main.cpp @@ -6,9 +6,10 @@ #include "sparkd/cmds/Md.h" #include "sparkd/cmds/QuickTest.h" #include "sparkd/cmds/Run.h" +#include "sparkd/cmds/TestPythonMdProviderCmd.h" #include "sparkd/cmds/Web.h" #include "sparkd/cmds/crypto/CryptoCmd.h" -#include "sparkd/md/MarketDataGateway.h" + #include using namespace bts; @@ -31,6 +32,7 @@ int main(int argc, char **argv) { app.add_subcommand(CLI::App_p(new cmds::Run())); app.add_subcommand(CLI::App_p(new cmds::KeyCmd())); app.add_subcommand(CLI::App_p(new cmds::Web())); + app.add_subcommand(CLI::App_p(new cmds::TestPythonMdProviderCmd())); app.add_subcommand( "test", "Run unit test(s)"); // so this shows up in help CLI11_PARSE(app, argc, argv); diff --git a/src/app_sparkd/sparkd/md/MarketDataGateway.cpp b/src/app_sparkd/sparkd/md/MarketDataGateway.cpp index 506a65d60..1d8049706 100644 --- a/src/app_sparkd/sparkd/md/MarketDataGateway.cpp +++ b/src/app_sparkd/sparkd/md/MarketDataGateway.cpp @@ -1,4 +1,6 @@ #include "MarketDataGateway.h" +#include "py_market_data/md_provider.h" +#include "py_market_data/md_sink_impl.h" #include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h" #include "spark/api/md/MarketDataConfig.pb.h" #include "spark/api/md/MarketDataConfigUtils.h" @@ -23,6 +25,7 @@ #include "sparkd/md/MarketDataRegistrationError.h" #include "sparkd/md/MarketDataRegistrationFactory.h" #include "sparkd/md/MarketDataSubscription.h" +#include "sparkd/md/PyMarketDataSubscription.h" #include "sparkd/md/binance/BinanceSubscription.h" #include "sparkd/md/binance_us/BinanceUsSubscription.h" #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h" @@ -37,13 +40,18 @@ #include #include +#include #include #include #include #include +#include +#include #include #include +namespace py = pybind11; + namespace bts::md { using bts::api::md::MarketDataConfig; @@ -52,6 +60,7 @@ using bts::config::SystemConfig; using bts::domain::SecurityID; using bts::io::ProtoPrinter; using bts::persistence::IDGenerator; +using os::sparkd_wd; using std::string; using std::unique_ptr; using std::unordered_map; @@ -115,6 +124,7 @@ struct MarketDataGateway::PImpl { std::vector> replays = {}; // we create these and pass them to the cc md sources std::vector> venue_listeners = {}; + std::vector> py_md_subs = {}; void start_glbx(); void start_cboe(); @@ -144,6 +154,7 @@ void MarketDataGateway::run() { me_->start_taifex(); me_->start_cboe(); _start_replays(me_->md_config); + start_py_(); while (true) { std::this_thread::sleep_for(std::chrono::seconds(1)); } @@ -151,6 +162,24 @@ void MarketDataGateway::run() { os::get_thread_id(), os::get_thread_name()); } +void MarketDataGateway::start_py_() { + const Path wd{sparkd_wd()}; + vector module_paths{ + wd.string(), + (wd / "py_market_data_providers").string(), + }; + for (const auto &py_plugin : me_->md_config.py_market_data_providers()) { + try { + me_->py_md_subs.push_back(std::make_shared( + py_plugin, module_paths, &me_->sec_id_store)); + } catch (const std::exception &ex) { + bts_assert_msgf( + false, "PyMarketDataSubscription threw: {}", + std::string{ex.what()}); + } + } +} + // CCAPI ////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// MarketDataRegistration MarketDataGateway::_start_cc( diff --git a/src/app_sparkd/sparkd/md/MarketDataGateway.h b/src/app_sparkd/sparkd/md/MarketDataGateway.h index e7a3d161a..0eaeef9e9 100644 --- a/src/app_sparkd/sparkd/md/MarketDataGateway.h +++ b/src/app_sparkd/sparkd/md/MarketDataGateway.h @@ -26,15 +26,16 @@ class MarketDataGateway { */ void run(); void unsubscribe(MarketDataListenerID id); + static void test_py(); private: struct PImpl; std::unique_ptr me_; MarketDataRegistration _start_cc(const api::md::MarketDataConfig &) noexcept; + void start_py_(); static void _start_emumba_deribit( bts::api::md::EmumbaDeribitMarketDataConfig cfg); - static void _start_onix_glbx( - bts::api::md::GlbxMarketDataConfig cfg); + static void _start_onix_glbx(bts::api::md::GlbxMarketDataConfig cfg); void _start_replays(const bts::api::md::MarketDataConfig &cfg); }; diff --git a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp new file mode 100644 index 000000000..48d8d3d00 --- /dev/null +++ b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp @@ -0,0 +1,283 @@ +#include "PyMarketDataSubscription.h" +#include "python_md/md_sink.h" +#include "python_md/py_book.h" +#include "python_md/py_md_executor.h" +#include "python_md/py_security.h" +#include "python_md/py_security_list.h" +#include "python_md/py_trade.h" +#include "spark/api/domain/CallPut.pb.h" +#include "spark/api/domain/Venue.h" +#include "spark/core/TimeStamp.h" +#include "spark/domain/AggressorSide.h" +#include "spark/domain/DomainSecurityUtils.h" +#include "spark/domain/FutureSecurity.h" +#include "spark/domain/Security.h" +#include "spark/domain/SecurityID.h" +#include "spark/domain/SpotSecurity.h" +#include "spark/domain/TradeRecord.h" +#include "spark/domain_model/Security.pb.h" +#include "spark/logging/logging.h" +#include "spark/md/GeneralMarketDataListener.h" +#include "spark/md/GeneralSecurityIndex.h" +#include "spark/md/MDProducer.h" +#include "spark/os/os.h" +#include "spark/persistence/IDGenerator.h" +#include "spark/utils/bts_assert.h" + +#include +#include +#include +#include +#include +#include + +namespace bts::md { + +using bts::domain::SecurityID; +using bts::md::py_interop::PyBook; +using bts::md::py_interop::PyBookList; +using bts::md::py_interop::PyBookSide; +using bts::md::py_interop::PyMdExecutor; +using bts::md::py_interop::PySecurityList; +using bts::md::py_interop::PyTrade; +using bts::md::py_interop::PyTradeList; +using bts::persistence::IDGenerator; +using std::pair; +using std::shared_ptr; +using std::string; +using std::unique_ptr; +using std::unordered_map; +using std::vector; + +using Path = boost::filesystem::path; + +struct VenueSink { + VenueSink(api::Venue venue, GeneralSecurityIndex *index) + : producer(), listener(*index, producer, venue) {} + + MDProducer producer; + GeneralMarketDataListener listener; +}; + +struct TradeTuple { + py_interop::PyTrade py_trade; + domain::TradeRecord trade; + api::domain::Venue venue; +}; + +struct PyMarketDataSubscription::PImpl : private py_interop::MdSink { + PImpl( + string py_plugin, vector module_paths, IDGenerator *sec_id_store) + : py_plugin_(std::move(py_plugin)), + module_paths_(std::move(module_paths)), + py_md_exec_(py_plugin_, module_paths_, this), + sec_id_store_(sec_id_store), + index_(std::make_unique()) {} + + void start() { py_md_exec_.start(); } + void stop() { py_md_exec_.stop(); } + + private: + string py_plugin_; + vector module_paths_; + PyMdExecutor py_md_exec_; + unordered_map> sinks_; + + IDGenerator *sec_id_store_; + unique_ptr index_; + // The main store for securities. + vector securities_; + // We also maintain Security pointers per venue (the GeneralMarketDataListener does not support + // writing arbitrary groups of securities, they must share a common venue when batched together). + unordered_map> + securities_by_venue_; + domain::Book book_; + // Since the python md provider isn't tethered to a single venue, we + // must know the venue for each individual trade. + vector trades_; + unordered_map< + SecurityID, + pair>> + trades_by_security_; + + void on_securities(const PySecurityList &py_securities) override { + LOG(md, Info, "Received securities from python plugin. count: {}", + py_securities.size()); + // `securities_` and `securities_by_venue_` should already be empty, but no harm in clearing + // them again. We keep them as member variables to avoid (re)allocating memory in each callback. + securities_.clear(); + securities_by_venue_.clear(); + for (const auto &py_sec : py_securities.items()) { + LOG(md, Info, "PyMarketDataSubscription adding security {}", + py_sec.name()); + auto sec = to_bts_security_(py_sec); + if (!sec) { + LOG(md, Error, + "Failed to convert security {} to bts::domain::Security", + py_sec.name()); + continue; + } + securities_.emplace_back(std::move(*sec)); + } + // Populate vectors of security ptrs per each venue. + for (auto &sec : securities_) { + securities_by_venue_[sec.venue()].push_back(&sec); + } + // Pass securities to the sinks (per venue). + for (const auto &[venue, secs] : securities_by_venue_) { + get_listener_(venue).on_securities(secs); + } + securities_.clear(); + securities_by_venue_.clear(); + } + + void on_books(const PyBookList &py_books) override { + for (auto &py_book : py_books.items()) { + api::domain::Venue venue; + bts_assert(api::domain::venue_parse_smart(py_book.venue(), &venue)); + bts_assert(book_.count_populated() == 0); + to_book_(py_book, book_); + get_listener_(venue).on_book(book_); + book_.clear(); + } + } + + void on_trades(const PyTradeList &py_trades) override { + LOG(md, Info, "Received trades from python plugin. count: {}", + py_trades.size()); + // `trades_` and `trades_by_security_` should already be empty, but no harm in clearing them + // again. We keep them as member variables to avoid (re)allocating memory in each callback. + trades_.clear(); + trades_by_security_.clear(); + for (const auto &py_trade : py_trades.items()) { + LOG(md, Info, "PyMarketDataSubscription received trade {}", + py_trade.to_string()); + auto &trade = trades_.emplace_back(); + bts_assert( + api::domain::venue_parse_smart(py_trade.venue(), &trade.venue)); + to_bts_trade_(py_trade, trade.trade); + } + // Populate vectors of trade ptrs per each security. + for (const auto &trade : trades_) { + auto &sec_trades = trades_by_security_[trade.trade.sec_id]; + sec_trades.first = trade.venue; + sec_trades.second.push_back(&trade.trade); + } + // Pass trades to the sinks (per venue) + for (const auto &[sec_id, p] : trades_by_security_) { + get_listener_(p.first).on_trades(p.second); + } + trades_.clear(); + trades_by_security_.clear(); + } + + GeneralMarketDataListener &get_listener_(api::domain::Venue venue) { + const auto api_venue = domain::DomainSecurityUtils::convert(venue); + return sinks_ + .try_emplace( + api_venue, std::make_shared(api_venue, index_.get())) + .first->second->listener; + } + + SecurityID get_sec_id_(const string &venue_name, const string &sec_name) { + api::domain::Venue venue; + // Although an overload of 'get_id' accepts a string, we parse `venue_name` to the enum to + // normalize different spellings of the same venue. For example, "deribt", "DERIBIT", and + // "VENUE_deribit" all represent the VENUE_DERIBIT enum (see `venue_parse_smart` for details). + bts_assert(api::domain::venue_parse_smart(venue_name, &venue)); + auto sec_id = sec_id_store_->get_id(venue, sec_name, true); + bts_assert_msgf( + sec_id, "Failed to get security id for py_book. venue: {}; name: {}", + venue, sec_name); + return SecurityID{*sec_id}; + } + + std::optional to_bts_security_( + const py_interop::PySecurity &py_sec) { + // We can't let python provider assign the id, because it is unaware of other + // md providers, so can't possibly guarentee a unique id. + bts_assert(py_sec.id() == 0); + domain::Security sec; + auto &sec_comm = sec.security_common; + if (!api::domain::venue_parse_smart(py_sec.venue(), &sec_comm.venue)) { + LOG(md, Error, "Failed to parse venue from '{}'", py_sec.venue()); + return std::nullopt; + } + sec_comm.id = get_sec_id_(py_sec.venue(), py_sec.exchange_id()); + sec_comm.exchange_sec_id = py_sec.exchange_id(); + sec_comm.currency = py_sec.currency(); + sec_comm.asset = py_sec.asset(); + sec_comm.group = py_sec.group(); + sec_comm.symbol = py_sec.asset(); + if (sec_comm.symbol.empty()) { + sec_comm.symbol = py_sec.group(); + } + sec_comm.desc = py_sec.name(); + if (py_sec.is_spot()) { + sec.security = domain::SpotSecurity{}; + } else if (py_sec.is_future()) { + domain::FutureSecurity bts_fut; + bts_fut.expiration = + core::TimeStamp::from_iso_string(py_sec.future().expiration()); + sec.security = std::move(bts_fut); + } else if (py_sec.is_option()) { + domain::OptionSecurity opt; + const auto &py_opt = py_sec.option(); + opt.expiration = core::TimeStamp::from_iso_string(py_opt.expiration()); + opt.call_put = py_opt.is_call() ? api::domain::CallPut::CALL + : api::domain::CallPut::PUT; + opt.strike = domain::Price::from_double(py_opt.strike()); + sec.security = std::move(opt); + } else { + LOG(md, Error, + "PySecurity has incomplete definition (not spot, future, or " + "option)"); + return std::nullopt; + } + return sec; + } + + void to_bts_trade_(const PyTrade &source, domain::TradeRecord &dest) { + dest.sec_id = get_sec_id_(source.venue(), source.exchange_security_id()); + dest.time_stamp = core::TimeStamp::from_nanos(source.timestamp()); + dest.agg_side = source.aggressor_side().starts_with('B') || + source.aggressor_side().starts_with('b') + ? domain::AggressorSide::Buy + : domain::AggressorSide::Sell; + dest.trade_price = domain::Price::from_double(source.price()); + *dest.trade_quantity = source.quantity(); + } + + void to_book_side_(const PyBookSide &source, domain::BookSide &dest) { + for (int i = 0; i < source.size(); i++) { + auto py_level = source.get_level(i); + const auto price = py_level.price(); + const auto quantity = py_level.quantity(); + bts_assert(!std::isnan(price)); + bts_assert(quantity > 0); + *dest[i].total_quantity = quantity; + *dest[i].direct_quantity = quantity; + dest[i].price = domain::Price::from_double(price); + } + } + + void to_book_(const PyBook &source, domain::Book &dest) { + to_book_side_(source.bids(), dest.bids()); + to_book_side_(source.asks(), dest.asks()); + dest.set_security_id( + get_sec_id_(source.venue(), source.exchange_security_id())); + } +}; + +PyMarketDataSubscription::PyMarketDataSubscription( + string py_plugin, vector module_paths, IDGenerator *sec_id_store) + : me_(std::make_unique( + std::move(py_plugin), std::move(module_paths), sec_id_store)) { + LOG(md, Info, "PyMarketDataSubscription::start t_id:{} t_name:{}", + os::get_thread_id(), os::get_thread_name()); + me_->start(); +} + +PyMarketDataSubscription::~PyMarketDataSubscription() { me_->stop(); } + +} // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.h b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.h new file mode 100644 index 000000000..770dd8fc5 --- /dev/null +++ b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +#include + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::md { + +class PyMarketDataSubscription { + public: + explicit PyMarketDataSubscription( + std::string py_plugin, + std::vector module_paths, + persistence::IDGenerator *sec_id_store); + ~PyMarketDataSubscription(); + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md diff --git a/src/lib_python_md/CMakeLists.txt b/src/lib_python_md/CMakeLists.txt new file mode 100644 index 000000000..572ec6b03 --- /dev/null +++ b/src/lib_python_md/CMakeLists.txt @@ -0,0 +1,137 @@ +cmake_minimum_required(VERSION 3.23) +# This lists file configures 3 primary targets: +# 1) python_md -> libpython_md.so +# * Normal c++ libs/exes (e.g., libspark, sparkd) can link this +# 2) python_md_module -> python_md_module.cpython-310-x86_64-linux-gnu.so +# * Exports a py module with the MdProvider base class +# * Py md providers must import this and define a class that derives from 'python_md_module.MdProvider' +# * Normal c++ libs/exes shouldn't link this +# * For more info on the cpython api: +# - https://docs.python.org/3/extending/building.html +# - https://docs.python.org/3/c-api/module.html#c.PyModuleDef +# 3) python_md_providers +# * Copies py md provider plugins to a runtime dir where sparkd can find them +# load them as plugins. +# * Doesn't output a c++ lib or exe + +project(python_md) +include(GNUInstallDirs) +include(${CMAKE_SOURCE_DIR}/cmake/InitSharedLibrary.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/InstallBasicConfig.cmake) + +if (NOT DEFINED PROJECT_VERSION) + set(PROJECT_VERSION 1.0.0) +endif() + +# Find Python from the virtual environment +find_package(Python3 REQUIRED COMPONENTS Interpreter Development) +find_package(pybind11 REQUIRED) + +# 1) python_md +init_shared_library() +target_include_directories( + ${PROJECT_NAME} + PUBLIC + ${Python3_INCLUDE_DIRS} +) +target_link_directories( + ${PROJECT_NAME} + PUBLIC + ${Python3_LIBRARY_DIRS} +) +target_link_libraries( + ${PROJECT_NAME} + PUBLIC + pybind11::embed + ${Python3_LIBRARIES} + system +) +install_basic_config() + +# 2) python_md_module +set(module_proj ${PROJECT_NAME}_module) +set(module_proj_SOURCES ${${PROJECT_NAME}_SOURCES}) +pybind11_add_module(${module_proj} SHARED ${module_proj_SOURCES}) +target_link_libraries(${module_proj} PUBLIC pybind11::embed system) +foreach(dir ${pybind11_INCLUDE_DIRS}) + install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +endforeach() +target_include_directories( + ${module_proj} + PRIVATE + $ +) +target_include_directories( + ${module_proj} + PUBLIC + ${Python3_INCLUDE_DIRS} + $ + $ +) +install( + TARGETS ${module_proj} + EXPORT ${module_proj}Targets + RUNTIME + DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY + DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE + DESTINATION ${CMAKE_INSTALL_LIBDIR} +) +install( + EXPORT ${module_proj}Targets + FILE ${module_proj}Targets.cmake + NAMESPACE ${PROJECT_NAME}:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} +) + +# 3) python_md_providers +set(providers_proj ${PROJECT_NAME}_providers) +file( + GLOB_RECURSE providers + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + ${providers_proj}/*.py) +set(provider_outs) +foreach(_provider ${providers}) + set(_provider_in ${CMAKE_CURRENT_SOURCE_DIR}/${_provider}) + set(_provider_out ${CMAKE_CURRENT_BINARY_DIR}/${_provider}) + add_custom_command( + OUTPUT ${_provider_out} + COMMAND ${CMAKE_COMMAND} -E copy ${_provider_in} ${_provider_out} + DEPENDS ${_provider_in} + COMMENT "Copying md provider plugin ${_provider} to ${_provider_out}" + ) + list(APPEND provider_outs ${_provider_out}) +endforeach() + +add_custom_target(copy_${providers_proj} ALL DEPENDS ${provider_outs}) + +add_library(${providers_proj} INTERFACE) +add_library(${PROJECT_NAME}::${providers_proj} ALIAS ${providers_proj}) + +set_target_properties( + ${providers_proj} + PROPERTIES + INTERFACE_PYTHON_SCRIPTS_DIR + $$ +) + +set_target_properties( + ${providers_proj} + PROPERTIES + INTERFACE_PYTHON_SCRIPTS $$ +) + +install( + TARGETS ${providers_proj} + EXPORT ${providers_proj}Targets + DESTINATION ${CMAKE_INSTALL_BINDIR} +) + +install( + EXPORT ${providers_proj}Targets + FILE ${providers_proj}Targets.cmake + NAMESPACE ${PROJECT_NAME}:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} +) +install(DIRECTORY ${providers_proj} DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/lib_python_md/Config.cmake.in b/src/lib_python_md/Config.cmake.in new file mode 100644 index 000000000..ea8dbc30c --- /dev/null +++ b/src/lib_python_md/Config.cmake.in @@ -0,0 +1,12 @@ +@PACKAGE_INIT@ + +include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@_moduleTargets.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@_providersTargets.cmake") + +find_dependency(Python3 REQUIRED COMPONENTS Interpreter Development) +find_dependency(pybind11) + +check_required_components(@PROJECT_NAME@) +check_required_components(@PROJECT_NAME@_module) +check_required_components(@PROJECT_NAME@_providers) diff --git a/src/lib_python_md/README.md b/src/lib_python_md/README.md new file mode 100644 index 000000000..af0f08ad3 --- /dev/null +++ b/src/lib_python_md/README.md @@ -0,0 +1,54 @@ + + +[pybind11]: https://github.com/pybind/pybind11 +[lib_spark]: ../lib_spark/README.md + +# lib_python_md + +This library binds c++ market data providers to python modules +which can be loaded at runtime. + +## **Dependencies** + +* **[Pybind11][pybind11]** +* **[lib_spark][lib_spark]** + +## Example +```python +import python_md_module as pymd + + +class MdProviderExample(pymd.MdProvider): + def __init__(self): + super(MdProviderExample, self).__init__() + pass + + def start(self, sink: pymd.MdSink): + print("Starting MdProviderExample") + # An actual implementation would subscribe to market data here. + # This is just an example to show you how to pass market data + # back to the c++ Spark api. + + def add_dummy_security(): + # This example populates a dummy security for demonstrative purposes. + # Of course, an actual implementation would populate the security list + # with data received from the exchange (or some other provider). + sec_list = pymd.PySecurityList() + sec = pymd.PySecurity() + sec.set_venue("GLBX") + sec.set_name("ESH5") + sec.set_asset("ES") + sec.set_expiration("2025-03-28") + sec_list.append(sec) + # Note: the example only adds 1 sec, but batch adds are supported. + sink.on_securities(sec_list) + + add_dummy_security() + + def stop(self): + print("Stopping MdProviderExample") +``` + +## Devs +Python files in `python_md_providers/` (child dir of this dir) are copied to
+the plugins directory at build time, and should automatically be loaded at runtime. diff --git a/src/lib_python_md/python_md/md_provider.cpp b/src/lib_python_md/python_md/md_provider.cpp new file mode 100644 index 000000000..4e9de1a48 --- /dev/null +++ b/src/lib_python_md/python_md/md_provider.cpp @@ -0,0 +1,7 @@ +#include "python_md/md_provider.h" + +namespace bts::md::py_interop { + +MdProvider::~MdProvider() = default; + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/md_provider.h b/src/lib_python_md/python_md/md_provider.h new file mode 100644 index 000000000..74ad68109 --- /dev/null +++ b/src/lib_python_md/python_md/md_provider.h @@ -0,0 +1,14 @@ +#pragma once + +#include "python_md/md_sink.h" + +namespace bts::md::py_interop { + +class MdProvider { + public: + virtual ~MdProvider(); + virtual void start(MdSink *sink) = 0; + virtual void stop() = 0; +}; + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/md_sink.cpp b/src/lib_python_md/python_md/md_sink.cpp new file mode 100644 index 000000000..c09561296 --- /dev/null +++ b/src/lib_python_md/python_md/md_sink.cpp @@ -0,0 +1,7 @@ +#include "md_sink.h" + +namespace bts::md::py_interop { + +MdSink::~MdSink() = default; + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/md_sink.h b/src/lib_python_md/python_md/md_sink.h new file mode 100644 index 000000000..19c4e35e2 --- /dev/null +++ b/src/lib_python_md/python_md/md_sink.h @@ -0,0 +1,18 @@ +#pragma once + +#include "python_md/py_book_list.h" +#include "python_md/py_security_list.h" +#include "python_md/py_trade_list.h" + +namespace bts::md::py_interop { + +class MdSink { + public: + virtual ~MdSink(); + + virtual void on_securities(const py_interop::PySecurityList &securities) = 0; + virtual void on_books(const py_interop::PyBookList &books) = 0; + virtual void on_trades(const py_interop::PyTradeList &trades) = 0; +}; + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/md_sink_impl.cpp b/src/lib_python_md/python_md/md_sink_impl.cpp new file mode 100644 index 000000000..9c2b63938 --- /dev/null +++ b/src/lib_python_md/python_md/md_sink_impl.cpp @@ -0,0 +1,73 @@ +#include "python_md/md_sink_impl.h" +#include "python_md/py_security_list.h" +#include "python_md/py_trade_list.h" + +#include +#include + +namespace bts::md::py_interop { + +struct MdSinkImpl::PImpl { + std::function securities_callback; + std::function books_callback; + std::function trades_callback; +}; + +MdSinkImpl::MdSinkImpl() : me_(std::make_unique()) { + // TODO +} + +MdSinkImpl::MdSinkImpl(MdSinkImpl &&) = default; +MdSinkImpl &MdSinkImpl::operator=(MdSinkImpl &&) = default; +MdSinkImpl::~MdSinkImpl() = default; + +void MdSinkImpl::on_books(const py_interop::PyBookList &books) { + if (books.items().empty()) { + std::cerr << "'on_books' called with empty list" << std::endl; + return; + } + if (me_->books_callback) { + me_->books_callback(books); + } +} + +void MdSinkImpl::on_securities(const py_interop::PySecurityList &securities) { + try { + if (securities.items().empty()) { + std::cerr << "'on_securities' called with empty list" << std::endl; + return; + } + if (me_->securities_callback) { + me_->securities_callback(securities); + } + } catch (const std::exception &ex) { + std::cerr << "MdSinkImpl.on_securities threw: " << ex.what() << std::endl; + } +} + +void MdSinkImpl::on_trades(const py_interop::PyTradeList &trades) { + if (trades.items().empty()) { + std::cout << "'on_trades' called with empty list" << std::endl; + return; + } + if (me_->trades_callback) { + me_->trades_callback(trades); + } +} + +void MdSinkImpl::set_books_callback( + std::function callback) { + me_->books_callback = callback; +} + +void MdSinkImpl::set_trades_callback( + std::function callback) { + me_->trades_callback = callback; +} + +void MdSinkImpl::set_securities_callback( + std::function callback) { + me_->securities_callback = callback; +} + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/md_sink_impl.h b/src/lib_python_md/python_md/md_sink_impl.h new file mode 100644 index 000000000..868313814 --- /dev/null +++ b/src/lib_python_md/python_md/md_sink_impl.h @@ -0,0 +1,37 @@ +#pragma once + +#include "python_md/md_sink.h" +#include "python_md/py_book_list.h" +#include "python_md/py_security_list.h" +#include "python_md/py_trade_list.h" + +#include +#include + +namespace bts::md::py_interop { + +class MdSinkImpl : public MdSink { + public: + MdSinkImpl(); + MdSinkImpl(MdSinkImpl &&); + ~MdSinkImpl() override; + + MdSinkImpl &operator=(MdSinkImpl &&); + + void on_books(const py_interop::PyBookList &books) override; + void on_trades(const py_interop::PyTradeList &trades) override; + void on_securities(const py_interop::PySecurityList &securities) override; + + void set_books_callback( + std::function callback); + void set_trades_callback( + std::function callback); + void set_securities_callback( + std::function callback); + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_book.h b/src/lib_python_md/python_md/py_book.h new file mode 100644 index 000000000..592568ce2 --- /dev/null +++ b/src/lib_python_md/python_md/py_book.h @@ -0,0 +1,97 @@ +#pragma once + +#include "python_md/py_book_side.h" + +#include +#include + +namespace bts::md::py_interop { + +/** + * @brief Represents the L2 Order Book for a particular security, at a particular venue. + */ +class PyBook { + public: + /** + * @brief Gets the name of the venue (i.e., exchange). + * + * @return const std::string& The name of the venue. + */ + const std::string &venue() const; + void set_venue(std::string value); + + /** + * @brief Gets the exchange security id. This is typically the security's name + * (e.g., "BTS-PERPETUAL", or "ESH5"). + * + * @note Must be unique for each security at a particular venue (i.e., a single + * venue cannot have 2 securities with the same name). + * + * @note There is no "security_id" member because that is an internal id that + * we generate elsewhere in c++ code. The python plugins just need to set the + * venue and exchange_security_id. + */ + const std::string &exchange_security_id() const; + + /** + * @brief Sets the exchange security id. + * + * @param value The new exchange security id. + */ + void set_exchange_security_id(std::string value); + + PyBookSide &bids(); + const PyBookSide &bids() const; + void set_bids(PyBookSide bids); + + PyBookSide &asks(); + const PyBookSide &asks() const; + void set_asks(PyBookSide asks); + + /** + * @brief Gets the number of levels in the book (i.e., the number of prices + * with resting orders). + */ + constexpr int size() const; + std::string to_string() const; + + private: + std::string venue_; + std::string exchange_security_id_; + PyBookSide bids_; + PyBookSide asks_; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline const std::string &PyBook::venue() const { return venue_; } +inline void PyBook::set_venue(std::string value) { venue_ = std::move(value); } + +inline const std::string &PyBook::exchange_security_id() const { + return exchange_security_id_; +} +inline void PyBook::set_exchange_security_id(std::string value) { + exchange_security_id_ = std::move(value); +} + +inline PyBookSide &PyBook::bids() { return bids_; } +inline const PyBookSide &PyBook::bids() const { return bids_; } +inline void PyBook::set_bids(PyBookSide bids) { bids_ = std::move(bids); } + +inline PyBookSide &PyBook::asks() { return asks_; } +inline const PyBookSide &PyBook::asks() const { return asks_; } +inline void PyBook::set_asks(PyBookSide asks) { asks_ = std::move(asks); } + +constexpr int PyBook::size() const { return bids_.size() + asks_.size(); } + +inline std::string PyBook::to_string() const { + std::stringstream str; + // pseudo json format + str << "bids: [" << bids_.to_string() << "], asks: [" << asks_.to_string() + << "]"; + return str.str(); +} + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_book_level.h b/src/lib_python_md/python_md/py_book_level.h new file mode 100644 index 000000000..d75b09bba --- /dev/null +++ b/src/lib_python_md/python_md/py_book_level.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + +namespace bts::md::py_interop { + +class PyBookLevel { + public: + constexpr PyBookLevel() = default; + + constexpr double price() const; + void set_price(double value); + + constexpr double quantity() const; + void set_quantity(double value); + + std::string to_string() const; + + private: + double price_{std::numeric_limits::quiet_NaN()}; + double quantity_{0}; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +constexpr double PyBookLevel::price() const { return price_; } +inline void PyBookLevel::set_price(double value) { price_ = value; } + +constexpr double PyBookLevel::quantity() const { return quantity_; } +inline void PyBookLevel::set_quantity(double value) { quantity_ = value; } + +inline std::string PyBookLevel::to_string() const { + std::stringstream str; + str << price_ << " (" << quantity_ << ")"; + return str.str(); +} + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_book_list.h b/src/lib_python_md/python_md/py_book_list.h new file mode 100644 index 000000000..aba8cdb9f --- /dev/null +++ b/src/lib_python_md/python_md/py_book_list.h @@ -0,0 +1,54 @@ +#pragma once + +#include "python_md/py_book.h" + +#include +#include + +namespace bts::md::py_interop { + +class PyBookList { + public: + void append(const py_interop::PyBook &book); + void clear(); + std::vector &items(); + const std::vector &items() const; + constexpr std::size_t size() const; + std::string to_string() const; + + private: + std::vector items_; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline void PyBookList::append(const py_interop::PyBook &book) { + items_.push_back(book); +} + +inline void PyBookList::clear() { items_.clear(); } + +constexpr std::size_t PyBookList::size() const { return items_.size(); } + +inline std::string PyBookList::to_string() const { + std::stringstream str; + bool is_first = true; + for (const auto &book : items_) { + if (is_first) { + is_first = false; + } else { + str << "\n"; + } + str << book.to_string(); + } + return str.str(); +} + +inline std::vector &PyBookList::items() { return items_; } +inline const std::vector &PyBookList::items() const { + return items_; +} + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_book_side.h b/src/lib_python_md/python_md/py_book_side.h new file mode 100644 index 000000000..f16b68d4a --- /dev/null +++ b/src/lib_python_md/python_md/py_book_side.h @@ -0,0 +1,123 @@ +#pragma once + +#include "python_md/py_book_level.h" + +#include +#include +#include +#include + +namespace bts::md::py_interop { + +/** + * @brief Represents an L2 Order Boo for a particular side of the market (i.e., + * bids or offers). + * + * @details Bid market levels should be in DESCENDING order by price, while Ask + * levels (offers) should be in ASCENDING order. Another way to think about it + * is the price levels are sorted by their distance from the opposite market. + */ +class PyBookSide { + public: + constexpr bool is_ascending(); + void set_is_ascending(bool is_ascending); + + constexpr PyBookLevel get_level(int index); + constexpr PyBookLevel get_level(int index) const; + void set_level(PyBookLevel level); + + constexpr int size() const; + + std::string to_string() const; + + private: + std::vector levels_; + bool is_ascending_; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +constexpr bool PyBookSide::is_ascending() { return is_ascending_; } + +inline void PyBookSide::set_is_ascending(bool is_ascending) { + is_ascending_ = is_ascending; +} + +inline void PyBookSide::set_level(PyBookLevel level) { + if (std::isnan(level.price())) { + std::cerr << "PyBookLevel invalid price (nan)" << std::endl; + return; + } + if (level.quantity() < 0) { + std::cerr << "PyBookLevel invalid quantity (less than 0)" << std::endl; + return; + } + auto it = std::lower_bound( + levels_.begin(), levels_.end(), level, + [](const auto &lhs, const auto &rhs) { + return lhs.price() < rhs.price(); + }); + if (it == levels_.end()) { + if (level.quantity() == 0) { + std::cerr << "PyBookLevel " << to_string() + << " indicates remove op, but the price level does not exist" + << std::endl; + return; + } + levels_.push_back(std::move(level)); + } else if (it->price() == level.price()) { + if (level.quantity() == 0) { + levels_.erase(it); + } else { + *it = std::move(level); + } + } else { + levels_.insert(it, std::move(level)); + } +} + +constexpr PyBookLevel PyBookSide::get_level(int index) { + if (!is_ascending_) { + index = levels_.size() - 1 - index; + } + return levels_.at(index); +} + +constexpr PyBookLevel PyBookSide::get_level(int index) const { + if (!is_ascending_) { + index = levels_.size() - 1 - index; + } + return levels_.at(index); +} + +constexpr int PyBookSide::size() const { return levels_.size(); } + +inline std::string PyBookSide::to_string() const { + std::string rv; + if (is_ascending_) { + bool is_first = true; + for (const auto &level : levels_) { + if (is_first) { + is_first = false; + } else { + rv.append(", "); + } + rv.append(level.to_string()); + } + } else { + bool is_first = true; + for (auto it = levels_.rbegin(); it != levels_.rend(); it++) { + if (is_first) { + is_first = false; + } else { + rv.append(", "); + } + rv.append(it->to_string()); + } + } + return rv; +} + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_future.h b/src/lib_python_md/python_md/py_future.h new file mode 100644 index 000000000..b0d8670f2 --- /dev/null +++ b/src/lib_python_md/python_md/py_future.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include + +namespace bts::md::py_interop { + +/** + * @brief Defines the expiration for a Future contract. + */ +class PyFuture { + public: + const std::string &expiration() const; + void set_expiration(std::string value); + + std::string to_string() const; + + private: + std::string expiration_; +}; + +std::ostream &operator<<(std::ostream &out, const PyFuture &fut) noexcept; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline const std::string &PyFuture::expiration() const { return expiration_; } +inline void PyFuture::set_expiration(std::string value) { + expiration_ = std::move(value); +} + +inline std::string PyFuture::to_string() const { + std::stringstream str; + str << *this; + return str.str(); +} + +inline std::ostream &operator<<( + std::ostream &out, const PyFuture &fut) noexcept { + out << fut.expiration(); + return out; +} + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_md_executor.cpp b/src/lib_python_md/python_md/py_md_executor.cpp new file mode 100644 index 000000000..c080f1ede --- /dev/null +++ b/src/lib_python_md/python_md/py_md_executor.cpp @@ -0,0 +1,141 @@ +#include "py_md_executor.h" +#include "python_md/md_provider.h" +#include "python_md/md_sink.h" +#include "system/string_utils.h" +#include "system/thread_id.h" + +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; + +using bts::system::string_utils::snake_to_pascal_case; +using bts::system::string_utils::split; +using std::int64_t; +using std::size_t; +using std::string; + +namespace bts::md::py_interop { +class Foo { + public: + void dostuff(); + void dostuff() const; +}; + +// Typedefs for member function pointers +typedef void (Foo::*NonConstDostuff)(); // Non-const member function +typedef void (Foo::*ConstDostuff)() const; // Const member function + +// Alternatively, using the `using` keyword (modern C++): +using NonConstDostuffPtr = void (Foo::*)(); +using ConstDostuffPtr = void (Foo::*)() const; + +/** + * Parses module and type name from a \p py_md_provider arg. The following formats are supported: + * 1) . + * 2) + * In this case, the type name defaults to the PascalCase equivalent of the module name. + * For example, the module "foo_bar" would have type name "FooBar". + */ +static void parse_py_md_provider( + const std::string &py_md_provider, + std::string *module_name, + std::string *type_name) { + auto parts = split(py_md_provider, "."); + if (parts.empty()) { + return; + } + if (parts.size() == 1) { + snake_to_pascal_case(parts.emplace_back(parts[0])); + } + *module_name = parts[0]; + *type_name = parts[1]; +} + +struct PyMdExecutor::PImpl { + std::string py_md_provider; + std::vector module_paths; + std::unique_ptr provider; + MdSink *sink; + std::optional provider_obj; + bool is_running_ = false; + size_t thread_id = BTS_THIS_THREAD_ID; + std::mutex _instance_mx; + std::optional interpreter{}; + std::function securities_callback; + std::function books_callback; + std::function trades_callback; + + void start() { + check_access(); + { + std::lock_guard lock{_instance_mx}; + if (is_running_) { + return; + } + interpreter.emplace(); + is_running_ = true; + } + provider_obj = create_provider(); + MdProvider *provider = provider_obj->cast(); + provider->start(sink); + } + + void stop() { + check_access(); + std::lock_guard lock{_instance_mx}; + if (!provider_obj) { + return; + } + MdProvider *providerx = provider_obj->cast(); + providerx->stop(); + provider_obj = std::nullopt; + interpreter.reset(); + } + + py::object create_provider() { + string provider_module_name; + string provider_type_name; + parse_py_md_provider( + py_md_provider, &provider_module_name, &provider_type_name); + + // Prepare module path + py::module sys = py::module::import("sys"); + for (const auto &module_path : module_paths) { + sys.attr("path").attr("append")(module_path.c_str()); + } + // Import provider module + py::module provider_module = + py::module::import(provider_module_name.c_str()); + // Instantiate provider + return provider_module.attr(provider_type_name.c_str())(); + } + + void check_access() { + if (BTS_THIS_THREAD_ID != thread_id) { + throw std::runtime_error{ + "All calls to PyMdExecutor must be on the same thread."}; + } + } +}; + +PyMdExecutor::PyMdExecutor( + std::string py_md_provider, + std::vector module_paths, + MdSink *sink) + : me_(std::make_unique()) { + me_->py_md_provider = std::move(py_md_provider); + me_->module_paths = std::move(module_paths); + me_->sink = sink; +} + +PyMdExecutor::~PyMdExecutor() = default; + +void PyMdExecutor::start() { me_->start(); } +void PyMdExecutor::stop() { me_->stop(); } + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_md_executor.h b/src/lib_python_md/python_md/py_md_executor.h new file mode 100644 index 000000000..5edcabcdf --- /dev/null +++ b/src/lib_python_md/python_md/py_md_executor.h @@ -0,0 +1,25 @@ +#pragma once + +#include "python_md/md_sink.h" + +#include + +namespace bts::md::py_interop { + +class PyMdExecutor { + public: + PyMdExecutor( + std::string py_md_provider, + std::vector module_paths, + MdSink *sink); + ~PyMdExecutor(); + + void start(); + void stop(); + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_md_provider.cpp b/src/lib_python_md/python_md/py_md_provider.cpp new file mode 100644 index 000000000..f4a386d81 --- /dev/null +++ b/src/lib_python_md/python_md/py_md_provider.cpp @@ -0,0 +1,7 @@ +#include "py_md_provider.h" + +namespace bts::md::py_interop { + +PyMdProvider::~PyMdProvider() = default; + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_md_provider.h b/src/lib_python_md/python_md/py_md_provider.h new file mode 100644 index 000000000..d1aff32cc --- /dev/null +++ b/src/lib_python_md/python_md/py_md_provider.h @@ -0,0 +1,31 @@ +#pragma once + +#include "python_md/md_provider.h" +#include "python_md/md_sink.h" + +#include + +namespace bts::md::py_interop { + +/** + * An implementation of `bts::md::py_interop::MdProvider` that binds to a python object at runtime. + * + * @note The python object must derive from 'python_md_module.MdProvider', and define methods: + * 1. 'start(md_sink: python_md_module.MdSink)' + * 2. 'stop' + * @note 'python_md_module' is a python module defined by us in c++ (see 'py_module.cpp'). + */ +class PyMdProvider : public MdProvider { + public: + using MdProvider::MdProvider; + + ~PyMdProvider() override; + + void start(MdSink *sink) override { + PYBIND11_OVERLOAD_PURE(void, MdProvider, start, sink); + } + + void stop() override { PYBIND11_OVERLOAD_PURE(void, MdProvider, stop); } +}; + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_module.cpp b/src/lib_python_md/python_md/py_module.cpp new file mode 100644 index 000000000..1156aa3b0 --- /dev/null +++ b/src/lib_python_md/python_md/py_module.cpp @@ -0,0 +1,146 @@ +#include "md_provider.h" +#include "python_md/md_sink.h" +#include "python_md/md_sink_impl.h" +#include "python_md/py_book_level.h" +#include "python_md/py_book_list.h" +#include "python_md/py_book_side.h" +#include "python_md/py_md_provider.h" +#include "python_md/py_security.h" +#include "python_md/py_security_list.h" +#include "python_md/py_trade.h" + +#include + +using bts::md::py_interop::MdSinkImpl; +using bts::md::py_interop::PyBook; +using bts::md::py_interop::PyBookLevel; +using bts::md::py_interop::PyBookList; +using bts::md::py_interop::PyBookSide; +using bts::md::py_interop::PyFuture; +using bts::md::py_interop::PyOption; +using bts::md::py_interop::PySecurity; +using bts::md::py_interop::PySecurityList; +using bts::md::py_interop::PySpot; +using bts::md::py_interop::PyTrade; +using bts::md::py_interop::PyTradeList; + +namespace py = pybind11; + +#define _STR(x) #x +// In general, we bind c++ methods to python methods with the same name verbatim. +// The DEF macro avoids typos in the python method name for this common case. +#define DEF(type, member) def(_STR(member), &type::member) +#define EXPLICIT_DEF(type, member, member_type) \ + def(_STR(member), (member_type) & type::member) +// The PROP macro defines getter and setter methods according to +// Binds a pseudo property with getter/setter methods. +// Binds getter -> , and setter -> set_. +#define PROP(type, name) \ + def(_STR(name), &type::name).def(_STR(set_##name), &type::set_##name) +// namespace blah { +// typedef bts::md::py_interop::PyBookSide & (bts::md::py_interop::PyBook::*bids()); +// } + +// Define python module with c++ classes. +PYBIND11_MODULE(python_md_module, m) { + using BookSideGetter = PyBookSide &(PyBook::*)(); + using BookLevelGetter = PyBookLevel (PyBookSide::*)(int) const; + // security + py::class_(m, "PySpot") + .def(py::init<>()) + .def("__str__", &PySpot::to_string); + py::class_(m, "PyFuture") + .def(py::init<>()) + .PROP(PyFuture, expiration) + .def("__str__", &PyFuture::to_string); + py::class_(m, "PyOption") + .def(py::init<>()) + .PROP(PyOption, expiration) + .PROP(PyOption, is_call) + .PROP(PyOption, strike) + .def("__str__", &PyOption::to_string); + py::class_(m, "PySecurity") + .def(py::init<>()) + .PROP(PySecurity, id) + .PROP(PySecurity, exchange_id) + .PROP(PySecurity, venue) + .PROP(PySecurity, name) + .PROP(PySecurity, currency) + .PROP(PySecurity, group) + .PROP(PySecurity, asset) + .PROP(PySecurity, spot) + .PROP(PySecurity, future) + .PROP(PySecurity, option) + .def("__str__", &PySecurity::to_string); + py::class_(m, "PySecurityList") + .def(py::init<>()) + .DEF(PySecurityList, append) + .DEF(PySecurityList, clear) + .def("__len__", &PySecurityList::size) + .def("__str__", &PySecurityList::to_string); + // book + py::class_(m, "PyBookLevel") + .def(py::init<>()) + .PROP(PyBookLevel, price) + .PROP(PyBookLevel, quantity); + py::class_(m, "PyBookSide") + .def(py::init<>()) + .EXPLICIT_DEF(PyBookSide, get_level, BookLevelGetter) + .DEF(PyBookSide, set_level) + .PROP(PyBookSide, is_ascending) + .def("__len__", &PyBookSide::size) + .def("__str__", &PyBookSide::to_string); + py::class_(m, "PyBook") + .def(py::init<>()) + .PROP(PyBook, venue) + .PROP(PyBook, exchange_security_id) + .EXPLICIT_DEF(PyBook, bids, BookSideGetter) + .DEF(PyBook, set_bids) + .EXPLICIT_DEF(PyBook, asks, BookSideGetter) + .DEF(PyBook, set_asks) + .def("__len__", &PyBook::size) + .def("__str__", &PyBook::to_string); + py::class_(m, "PyBookList") + .def(py::init<>()) + .DEF(PyBookList, append) + .DEF(PyBookList, clear) + .def("__len__", &PyBookList::size) + .def("__str__", &PyBookList::to_string); + // trade + py::class_(m, "PyTrade") + .def(py::init<>()) + .PROP(PyTrade, venue) + .PROP(PyTrade, exchange_security_id) + .PROP(PyTrade, price) + .PROP(PyTrade, quantity) + .PROP(PyTrade, aggressor_side) + .PROP(PyTrade, timestamp); + py::class_(m, "PyTradeList") + .def(py::init<>()) + .DEF(PyTradeList, append) + .DEF(PyTradeList, clear) + .def("__len__", &PyTradeList::size) + .def("__str__", &PyTradeList::to_string); + // md sink + py::class_(m, "MdSink") + .DEF(bts::md::py_interop::MdSink, on_securities) + .DEF(bts::md::py_interop::MdSink, on_books) + .DEF(bts::md::py_interop::MdSink, on_trades); + py::class_( + m, "MdSinkImpl") + .def(py::init<>()) + .DEF(bts::md::py_interop::MdSinkImpl, on_securities) + .DEF(bts::md::py_interop::MdSinkImpl, on_books) + .DEF(bts::md::py_interop::MdSinkImpl, on_trades); + // md provider + py::class_< + bts::md::py_interop::MdProvider, bts::md::py_interop::PyMdProvider>( + m, "MdProvider") + .def(py::init<>()) + .DEF(bts::md::py_interop::MdProvider, start) + .DEF(bts::md::py_interop::MdProvider, stop); +} + +#undef PROP +#undef DEF +#undef _STR diff --git a/src/lib_python_md/python_md/py_option.h b/src/lib_python_md/python_md/py_option.h new file mode 100644 index 000000000..e6ecaaecd --- /dev/null +++ b/src/lib_python_md/python_md/py_option.h @@ -0,0 +1,68 @@ +#pragma once + +#include +#include +#include + +namespace bts::md::py_interop { + +/** + * @brief Defines the expiration, strike, and call/put for an Option contract. + */ +class PyOption { + public: + constexpr PyOption() = default; + + const std::string &expiration() const; + void set_expiration(std::string value); + + constexpr bool is_call() const; + void set_is_call(bool value); + + constexpr double strike() const; + void set_strike(double value); + + std::string to_string() const; + + private: + std::string expiration_; + bool is_call_{false}; + double strike_{std::numeric_limits::quiet_NaN()}; +}; + +std::ostream &operator<<(std::ostream &out, const PyOption &opt) noexcept; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline const std::string &PyOption::expiration() const { return expiration_; } +inline void PyOption::set_expiration(std::string value) { + expiration_ = std::move(value); +} + +constexpr bool PyOption::is_call() const { return is_call_; } +inline void PyOption::set_is_call(bool value) { is_call_ = value; } + +constexpr double PyOption::strike() const { return strike_; } +inline void PyOption::set_strike(double value) { strike_ = value; } + +inline std::string PyOption::to_string() const { + std::stringstream str; + str << *this; + return str.str(); +} + +inline std::ostream &operator<<( + std::ostream &out, const PyOption &opt) noexcept { + out << opt.expiration() << " "; + if (opt.is_call()) { + out << "CALL "; + } else { + out << "PUT "; + } + out << opt.strike(); + return out; +} + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_security.h b/src/lib_python_md/python_md/py_security.h new file mode 100644 index 000000000..4bac2c541 --- /dev/null +++ b/src/lib_python_md/python_md/py_security.h @@ -0,0 +1,136 @@ +#pragma once + +#include "python_md/py_future.h" +#include "python_md/py_option.h" +#include "python_md/py_spot.h" + +#include +#include +#include +#include + +namespace bts::md::py_interop { + +class PySecurity { + public: + constexpr std::uint32_t id() const; + void set_id(std::uint32_t value); + + const std::string &exchange_id() const; + void set_exchange_id(std::string value); + + const std::string &venue() const; + void set_venue(std::string value); + + const std::string ¤cy() const; + void set_currency(std::string value); + + const std::string &group() const; + void set_group(std::string value); + + const std::string &asset() const; + void set_asset(std::string value); + + const std::string &name() const; + void set_name(std::string value); + + const PySpot &spot() const; + void set_spot(PySpot); + constexpr bool is_spot() const; + + const PyFuture &future() const; + void set_future(PyFuture); + constexpr bool is_future() const; + + const PyOption &option() const; + void set_option(PyOption); + constexpr bool is_option() const; + + std::string to_string() const; + + private: + std::uint32_t id_; + std::string exchange_id_; + std::string venue_; + std::string group_; + std::string asset_; + std::string name_; + std::string currency_; + std::variant sec_; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +constexpr std::uint32_t PySecurity::id() const { return id_; } +inline void PySecurity::set_id(std::uint32_t value) { id_ = value; } + +inline const std::string &PySecurity::exchange_id() const { + return exchange_id_; +} +inline void PySecurity::set_exchange_id(std::string value) { + exchange_id_ = std::move(value); +} + +inline const std::string &PySecurity::venue() const { return venue_; } +inline void PySecurity::set_venue(std::string value) { + venue_ = std::move(value); +} + +inline const std::string &PySecurity::currency() const { return currency_; } +inline void PySecurity::set_currency(std::string value) { + currency_ = std::move(value); +} + +inline const std::string &PySecurity::group() const { return group_; } +inline void PySecurity::set_group(std::string value) { + group_ = std::move(value); +} + +inline const std::string &PySecurity::asset() const { return asset_; } +inline void PySecurity::set_asset(std::string value) { + asset_ = std::move(value); +} + +inline const std::string &PySecurity::name() const { return name_; } +inline void PySecurity::set_name(std::string value) { + name_ = std::move(value); +} + +inline const PySpot &PySecurity::spot() const { + if (std::holds_alternative(sec_)) { + return std::get(sec_); + } + throw std::runtime_error{"PySecurity is not a spot"}; +} +inline void PySecurity::set_spot(PySpot value) { sec_ = std::move(value); } +constexpr bool PySecurity::is_spot() const { + return std::holds_alternative(sec_); +} + +inline const PyFuture &PySecurity::future() const { + if (std::holds_alternative(sec_)) { + return std::get(sec_); + } + throw std::runtime_error{"PySecurity is not a future"}; +} +inline void PySecurity::set_future(PyFuture value) { sec_ = std::move(value); } +constexpr bool PySecurity::is_future() const { + return std::holds_alternative(sec_); +} + +inline const PyOption &PySecurity::option() const { + if (std::holds_alternative(sec_)) { + return std::get(sec_); + } + throw std::runtime_error{"PySecurity is not an option"}; +} +inline void PySecurity::set_option(PyOption value) { sec_ = std::move(value); } +constexpr bool PySecurity::is_option() const { + return std::holds_alternative(sec_); +} + +inline std::string PySecurity::to_string() const { return name_; } + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_security_list.h b/src/lib_python_md/python_md/py_security_list.h new file mode 100644 index 000000000..e864e454c --- /dev/null +++ b/src/lib_python_md/python_md/py_security_list.h @@ -0,0 +1,56 @@ +#pragma once + +#include "python_md/py_security.h" + +#include +#include + +namespace bts::md::py_interop { + +class PySecurityList { + public: + void append(const py_interop::PySecurity &security); + void clear(); + const std::vector &items() const; + constexpr std::size_t size() const; + std::string to_string() const; + + private: + std::vector items_; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline void PySecurityList::append(const py_interop::PySecurity &security) { + items_.push_back(security); +} + +inline void PySecurityList::clear() { items_.clear(); } + +constexpr std::size_t PySecurityList::size() const { + return items_.size(); +} + +inline const std::vector & +PySecurityList::items() const { + return items_; +} + +inline std::string PySecurityList::to_string() const { + std::stringstream str; + bool is_first = true; + for (const auto &sec : items_) { + if (is_first) { + is_first = false; + } else { + str << "\n"; + } + str << sec.to_string(); + } + return str.str(); +} + + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_spot.h b/src/lib_python_md/python_md/py_spot.h new file mode 100644 index 000000000..860e0ebba --- /dev/null +++ b/src/lib_python_md/python_md/py_spot.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace bts::md::py_interop { + +/** + * @brief A placeholder object that represents a Spot contract. + * + * @details Unlike the similarly named types `PyFuture` and `PyOption`, this class has no members. + * A Spot contract's properties are universal to all security types in our system, and are thus all + * defined in `PySecurity`. It exists so we can follow the "one of" paradigm per security type. + */ +class PySpot { + public: + std::string to_string() const; +}; + +std::ostream &operator<<(std::ostream &out, const PySpot &spot) noexcept; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::string PySpot::to_string() const { return ""; } + +inline std::ostream &operator<<( + std::ostream &out, const PySpot & /*spot*/) noexcept { + return out; +} + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_trade.h b/src/lib_python_md/python_md/py_trade.h new file mode 100644 index 000000000..258509507 --- /dev/null +++ b/src/lib_python_md/python_md/py_trade.h @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include +#include + +namespace bts::md::py_interop { + +class PyTrade { + public: + const std::string &venue() const; + void set_venue(std::string value); + + const std::string &exchange_security_id() const; + void set_exchange_security_id(std::string value); + + constexpr double price() const; + void set_price(double value); + + constexpr double quantity() const; + void set_quantity(double value); + + const std::string &aggressor_side() const; + void set_aggressor_side(std::string value); + + /** + * @brief The time at which the trade was executed. Expressed as the number of nanoseconds since + * the unix epoch: Jan 1, 1970 00:00:00 UTC. + * + * @return constexpr std::int64_t The time at which the trade was executed. + */ + constexpr std::int64_t timestamp() const; + void set_timestamp(std::int64_t value); + + std::string to_string() const; + + private: + std::string venue_; + std::string exchange_security_id_; + double price_; + double quantity_; + std::string aggressor_side_; + std::int64_t timestamp_; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline const std::string &PyTrade::venue() const { return venue_; } +inline void PyTrade::set_venue(std::string value) { venue_ = std::move(value); } + +inline const std::string &PyTrade::exchange_security_id() const { + return exchange_security_id_; +} +inline void PyTrade::set_exchange_security_id(std::string value) { + exchange_security_id_ = std::move(value); +} + +constexpr double PyTrade::price() const { return price_; } +inline void PyTrade::set_price(double value) { price_ = value; } + +constexpr double PyTrade::quantity() const { return quantity_; } +inline void PyTrade::set_quantity(double value) { quantity_ = value; } + +inline const std::string &PyTrade::aggressor_side() const { + return aggressor_side_; +} +inline void PyTrade::set_aggressor_side(std::string value) { + aggressor_side_ = std::move(value); +} + +constexpr std::int64_t PyTrade::timestamp() const { return timestamp_; } +inline void PyTrade::set_timestamp(std::int64_t value) { timestamp_ = value; } + +inline std::string PyTrade::to_string() const { + std::stringstream str; + if (!aggressor_side_.empty()) { + str << aggressor_side_ << " "; + } + str << quantity_ << " @ " << price_ << " (ts " << timestamp_ << ")"; + return str.str(); +} + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md/py_trade_list.h b/src/lib_python_md/python_md/py_trade_list.h new file mode 100644 index 000000000..2be04a5b7 --- /dev/null +++ b/src/lib_python_md/python_md/py_trade_list.h @@ -0,0 +1,52 @@ +#pragma once + +#include "python_md/py_trade.h" + +#include +#include + +namespace bts::md::py_interop { + +class PyTradeList { + public: + void append(const py_interop::PyTrade &trade); + void clear(); + const std::vector &items() const; + constexpr std::size_t size() const; + std::string to_string() const; + + private: + std::vector items_; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline void PyTradeList::append(const py_interop::PyTrade &trade) { + items_.push_back(trade); +} + +inline void PyTradeList::clear() { items_.clear(); } + +constexpr std::size_t PyTradeList::size() const { return items_.size(); } + +inline const std::vector &PyTradeList::items() const { + return items_; +} + +inline std::string PyTradeList::to_string() const { + std::stringstream str; + bool is_first = true; + for (const auto &sec : items_) { + if (is_first) { + is_first = false; + } else { + str << "\n"; + } + str << sec.to_string(); + } + return str.str(); +} + +} // namespace bts::md::py_interop diff --git a/src/lib_python_md/python_md_providers/dummy_md_provider.py b/src/lib_python_md/python_md_providers/dummy_md_provider.py new file mode 100644 index 000000000..691943529 --- /dev/null +++ b/src/lib_python_md/python_md_providers/dummy_md_provider.py @@ -0,0 +1,122 @@ +import python_md_module as pymd +from typing import Optional + + +def _get_security( + venue: str, + name: str, + asset: str, + group: str, + currency: str, + expiration: Optional[str] = None, + option_kind: Optional[str] = None, + strike: Optional[float] = None, +): + sec = pymd.PySecurity() + sec.set_venue(venue) + sec.set_exchange_id(name) + sec.set_name(name) + sec.set_currency(currency) + sec.set_asset(asset) + sec.set_group(group) + is_spot = expiration is None and option_kind is None and strike is None + is_future = expiration is not None and option_kind is None and strike is None + is_option = ( + expiration is not None and option_kind is not None and strike is not None + ) + if is_spot: + sec.set_spot(pymd.PySpot()) + elif is_future: + fut = pymd.PyFuture() + fut.set_expiration(expiration) + sec.set_future(fut) + elif is_option: + opt = pymd.PyOption() + opt.set_expiration(expiration) + opt.set_is_call(option_kind.lower().startswith("c")) + opt.set_strike(strike) + sec.set_option(opt) + else: + raise ValueError("Invalid parameters (unable to deduce security type)") + return sec + + +def _get_book_side(levels: list[(float, float)], is_bid_side: bool): + book_side = pymd.PyBookSide() + book_side.set_is_ascending(not is_bid_side) + for price, qty in levels: + book_level = pymd.PyBookLevel() + book_level.set_price(price) + book_level.set_quantity(qty) + book_side.set_level(book_level) + return book_side + + +def _get_trade( + venue: str, + exchange_security_id: str, + price: float, + quantity: float, + aggressor_side: str, +): + trade = pymd.PyTrade() + trade.set_venue(venue) + trade.set_exchange_security_id(exchange_security_id) + trade.set_price(price) + trade.set_quantity(quantity) + trade.set_aggressor_side(aggressor_side) + return trade + + +class DummyMdProvider(pymd.MdProvider): + def __init__(self): + super(DummyMdProvider, self).__init__() + pass + + def start(self, sink: pymd.MdSink): + # start subscriptions, bind callbacks etc. + def add_fake_securities(): + sec_list = pymd.PySecurityList() + sec_list.append( + _get_security( + venue="Glbx", + name="ESH5", + asset="ES", + group="ES", + currency="USD", + expiration="2025-03-28", + ) + ) + sink.on_securities(sec_list) + + def add_fake_books(): + bids = _get_book_side([(99.75, 8), (99.50, 11)], is_bid_side=True) + asks = _get_book_side([(100.25, 3)], is_bid_side=False) + book = pymd.PyBook() + book.set_venue("Glbx") + book.set_exchange_security_id("ESH5") + book.set_bids(bids) + book.set_asks(asks) + # pass book to sink + book_list = pymd.PyBookList() + book_list.append(book) + sink.on_books(book_list) + + def add_fake_trades(): + trade = _get_trade( + venue="Glbx", + exchange_security_id="ESH5", + price=100, + quantity=2, + aggressor_side="Buy", + ) + trade_list = pymd.PyTradeList() + trade_list.append(trade) + sink.on_trades(trade_list) + + add_fake_securities() + add_fake_books() + add_fake_trades() + + def stop(self): + print("Stopping python MdProviderImpl") diff --git a/src/lib_python_md/python_md_providers/py_md_utils.py b/src/lib_python_md/python_md_providers/py_md_utils.py new file mode 100644 index 000000000..a7a128489 --- /dev/null +++ b/src/lib_python_md/python_md_providers/py_md_utils.py @@ -0,0 +1,31 @@ +from typing import Optional +import python_md_module as pymd + + +def get_book_side(levels: list[(float, float)], is_bid_side: bool): + book_side = pymd.PyBookSide() + book_side.set_is_ascending(not is_bid_side) + for price, qty in levels: + book_level = pymd.PyBookLevel() + book_level.set_price(price) + book_level.set_quantity(qty) + book_side.set_level(book_level) + return book_side + + +def get_trade( + price: Optional[float] = None, + quantity: Optional[float] = None, + aggressor: Optional[str] = None, + security_id: Optional[int] = None, +): + trade = pymd.PyTrade() + if price is not None: + trade.set_price(100) + if quantity is not None: + trade.set_quantity(2) + if aggressor is not None: + trade.set_aggressor(aggressor) + if security_id is not None: + trade.set_security_id(security_id) + return trade diff --git a/src/lib_spark/README.md b/src/lib_spark/README.md new file mode 100644 index 000000000..05ef7da4e --- /dev/null +++ b/src/lib_spark/README.md @@ -0,0 +1,19 @@ + +[boost]: https://www.boost.org +[protobuf]: https://protobuf.dev/ +[grpc]: https://grpc.io/ + +# lib_spark +Cross platform (Windows, Linux) c++ library that defines the native Spark API. + +This library was built with a focus on performance, which naturally makes it more +complex and difficult to use than the websocket api. + +We generally recommend the websocket api except in cases where +performance is paramount. + +## **Dependencies** + +* **[boost][boost]** +* **[Google Protocol Buffers (aka protobuf)][protobuf]** +* **[gRPC][grpc]** diff --git a/src/lib_spark/spark/api/domain/SecurityType.h b/src/lib_spark/spark/api/domain/SecurityType.h index 61e528529..73c786972 100644 --- a/src/lib_spark/spark/api/domain/SecurityType.h +++ b/src/lib_spark/spark/api/domain/SecurityType.h @@ -5,13 +5,40 @@ #include "spark/core/FMT_IMPL.h" #include "spark/io/JsonReader.h" #include "spark/logging/logging.h" +#include "system/string_utils.h" #include namespace bts::api::domain { +/** + * Parses SecurityType from a string. Unlike `bts::api::domain::SecurityType_Parse`, this + * implementation is case insensitive, and makes the 'SECURITY_TYPE_' prefix optional. + */ +bool security_type_parse_smart(const std::string &str, SecurityType *result); + std::ostream &operator<<(std::ostream &out, const SecurityType &venue) noexcept; +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline bool security_type_parse_smart(const std::string &str, SecurityType *result) { + if (bts::api::domain::SecurityType_Parse(str, result)) { + return true; + } + std::string s = system::string_utils::copy_to_upper(str); + if (SecurityType_Parse(s, result)) { + return true; + } + const char *prefix = "SECURITY_TYPE_"; + if (!s.starts_with(prefix)) { + s.insert(0, prefix); + } + return SecurityType_Parse(s, result); +} + + PB_ENUM_FROM_JSON(SecurityType) PB_ENUM_TO_JSON(SecurityType) diff --git a/src/lib_spark/spark/api/domain/Venue.h b/src/lib_spark/spark/api/domain/Venue.h index 6e1748c04..6392201eb 100644 --- a/src/lib_spark/spark/api/domain/Venue.h +++ b/src/lib_spark/spark/api/domain/Venue.h @@ -5,13 +5,40 @@ #include "spark/core/FMT_IMPL.h" #include "spark/io/JsonReader.h" #include "spark/logging/logging.h" +#include "system/string_utils.h" +#include #include namespace bts::api::domain { +/** + * Parses Venue from a string. Unlike `bts::api::domain::Venue_Parse`, this implementation is case + * insensitive, and makes the 'VENUE_' prefix optional. + */ +bool venue_parse_smart(const std::string &str, Venue *result); + std::ostream &operator<<(std::ostream &out, const Venue &venue) noexcept; +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline bool venue_parse_smart(const std::string &str, Venue *result) { + if (bts::api::domain::Venue_Parse(str, result)) { + return true; + } + std::string s = system::string_utils::copy_to_upper(str); + if (Venue_Parse(s, result)) { + return true; + } + const char *prefix = "VENUE_"; + if (!s.starts_with(prefix)) { + s.insert(0, prefix); + } + return Venue_Parse(s, result); +} + PB_ENUM_FROM_JSON(Venue) PB_ENUM_TO_JSON(Venue) diff --git a/src/lib_spark/spark/api/md/MarketDataConfig.proto b/src/lib_spark/spark/api/md/MarketDataConfig.proto index c8bb4966c..187513a30 100644 --- a/src/lib_spark/spark/api/md/MarketDataConfig.proto +++ b/src/lib_spark/spark/api/md/MarketDataConfig.proto @@ -34,4 +34,5 @@ message MarketDataConfig { repeated EmumbaDeribitMarketDataConfig emumba_deribit = 100; repeated MarketDataRecordingConfig recording = 500; repeated MarketDataReplayConfig replay = 510; + repeated PyMarketDataProviderConfig python_md_providers = 600; } diff --git a/src/lib_spark/spark/domain/Book.h b/src/lib_spark/spark/domain/Book.h index bf8ca74a5..16505b835 100644 --- a/src/lib_spark/spark/domain/Book.h +++ b/src/lib_spark/spark/domain/Book.h @@ -5,6 +5,7 @@ #include "SecurityID.h" #include "Side.h" #include "spark/io/JsonReader.h" + #include #include @@ -17,7 +18,22 @@ class BookT { [[nodiscard]] static constexpr BookT empty( SecurityID security_id) noexcept; + /** + * @brief Gets the number of price levels with non-zero quantities (the sum of such levels on + * both sides of the market, bids and asks). + * + * @return constexpr std::size_t The number of price levels with non-zero quantities + */ + [[nodiscard]] std::size_t count_populated() const noexcept; [[nodiscard]] static constexpr size_t max_depth(); + + /** + * @brief Gets the number of elements in this internal array. It should ALWAYS be 2. + * The first element (index 0) is the bids, the second (index 1) is the asks. This + * method is here to support generic iterator operations. + * + * @return constexpr std::size_t + */ [[nodiscard]] constexpr std::size_t size() const noexcept; [[nodiscard]] constexpr BS &bids() noexcept; @@ -34,6 +50,7 @@ class BookT { template void merge_with(OtherBook const &b); + void clear(); void clear_to_end(std::size_t level = 0); void clear_after_empty(); [[nodiscard]] bool is_sorted() const; @@ -95,10 +112,18 @@ constexpr std::size_t BookT::max_depth() { return BS::max_depth(); } +template +std::size_t BookT::count_populated() const noexcept { + return _books[0].count_populated() + _books[1].count_populated(); +} + +// The size of our array should absolutely, ALWAYS be 2 (bids at index 0, asks at index 1). template constexpr std::size_t BookT::size() const noexcept { + static_assert(std::tuple_size::value == 2); return _books.size(); } +static_assert(BookT{}.size() == 2); template constexpr BS &BookT::bids() noexcept { @@ -186,6 +211,11 @@ void BookT::remove_crosses(Side dominant_side) { dominant_side.other(), (*this)[dominant_side][0].price); } +template +void BookT::clear() { + Side::foreach ([&](Side s) { operator[](s).clear(); }); +} + template void BookT::clear_to_end(std::size_t level) { Side::foreach ([&](Side s) { diff --git a/src/lib_spark/spark/domain/BookSide.h b/src/lib_spark/spark/domain/BookSide.h index f40316a1b..6833e3d7a 100644 --- a/src/lib_spark/spark/domain/BookSide.h +++ b/src/lib_spark/spark/domain/BookSide.h @@ -92,7 +92,7 @@ class BookSideT { PopulatedIterator end_populated() noexcept; [[nodiscard]] ConstPopulatedIterator end_populated() const noexcept; - [[nodiscard]] std::size_t count_populated() const; + [[nodiscard]] std::size_t count_populated() const noexcept; bool trim_quantity_front(Quantity quantity) noexcept; @@ -119,7 +119,8 @@ class BookSideT { [[nodiscard]] Quantity intersecting_front_quantity( const OtherBookSide &other, std::size_t num_levels = max_depth()) const; - void clear_to_end(std::uint8_t level = 0); + void clear(); + void clear_to_end(std::uint8_t level); void clear_after_empty(); /** @@ -234,12 +235,14 @@ BookSideT::end() const noexcept { } template -constexpr typename BookSideT::ReverseIterator BookSideT::rbegin() noexcept { +constexpr typename BookSideT::ReverseIterator +BookSideT::rbegin() noexcept { return _book.rbegin(); } template -constexpr typename BookSideT::ReverseIterator BookSideT::rend() noexcept { +constexpr typename BookSideT::ReverseIterator +BookSideT::rend() noexcept { return _book.rend(); } @@ -287,7 +290,7 @@ BookSideT::end_populated() const noexcept { } template -std::size_t BookSideT::count_populated() const { +std::size_t BookSideT::count_populated() const noexcept { static_assert(SZ <= std::numeric_limits::max()); std::int32_t len = max_depth(); std::int32_t lo = 0; @@ -568,6 +571,11 @@ Quantity BookSideT::intersecting_front_quantity( return ifq; } +template +void BookSideT::clear() { + clear_to_end(0); +} + template void BookSideT::clear_to_end(std::uint8_t level) { static_assert(SZ <= std::numeric_limits::max()); diff --git a/src/lib_system/system/string_utils.h b/src/lib_system/system/string_utils.h index 033a7755c..f4028e121 100644 --- a/src/lib_system/system/string_utils.h +++ b/src/lib_system/system/string_utils.h @@ -22,6 +22,9 @@ void split( const std::string &delimiter, std::vector &parts); +void to_upper(std::string &str); +std::string copy_to_upper(const std::string &str); + ////////////////////////////////////////////////// // Definition ////////////////////////////////////////////////// @@ -105,4 +108,20 @@ void split( } } +inline void to_upper(std::string &str) { + for (char &c : str) { + c = toupper(c); + } +} + +inline std::string copy_to_upper(const std::string &str) { + std::string s; + s.reserve(str.size()); + for (char c : str) { + s.push_back(toupper(c)); + } + return s; +} + + } // namespace bts::system::string_utils diff --git a/src/lib_system/system/thread_id.h b/src/lib_system/system/thread_id.h new file mode 100644 index 000000000..b6d280af4 --- /dev/null +++ b/src/lib_system/system/thread_id.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +#define BTS_THREAD_ID(tid) std::hash{}(tid) +#define BTS_THIS_THREAD_ID \ + std::hash{}(std::this_thread::get_id())