From bdd5e08b60d66883f8e601c626829cd7ef757728 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 5 Jun 2025 13:51:19 +0000 Subject: [PATCH] refactor: Consolidate all py interop into single lib It was just too hard trying to make it work with separate libs. The main issue was getting the python md and execution modules to recognize PySecurityList and other types defined in the core module. --- CMakeLists.txt | 4 +- cmake/InitPythonModule.cmake | 2 +- src/app_sparkd/CMakeLists.txt | 28 +- .../configurations/bybit_dev/spark.conf | 3 +- .../py_execution_gateway_adapter.cpp | 184 ++++--- .../sparkd/md/PyMarketDataSubscription.cpp | 32 +- .../python_core/python_activator_impl.h | 140 ----- .../python_core/python_core_module.cpp | 97 ---- .../python_core_plugins/pyrightconfig.json | 16 - src/lib_python_execution/CMakeLists.txt | 19 - src/lib_python_execution/Config.cmake.in | 12 - .../python_execution_gateways/binance.env | 191 ------- .../binance_client.py | 503 ------------------ .../binance_env_helper.py | 100 ---- .../binance_execution_gateway.py | 242 --------- .../py_execution_utils.py | 158 ------ .../pyrightconfig.json | 20 - .../python_execution_gateways/simple_test.py | 241 --------- .../CMakeLists.txt | 4 +- .../Config.cmake.in | 0 .../README.md | 76 ++- .../pyrightconfig.json | 6 +- .../execution}/execution_gateway.cpp | 0 .../execution}/execution_gateway.h | 10 +- .../execution}/execution_gateway_bridge.cpp | 0 .../execution}/execution_gateway_bridge.h | 4 +- .../execution_gateway_bridge_impl.cpp | 22 +- .../execution_gateway_bridge_impl.h | 12 +- .../execution}/py_cancel_order_request.cpp | 0 .../execution}/py_cancel_order_request.h | 0 .../py_cancel_replace_order_request.cpp | 0 .../py_cancel_replace_order_request.h | 0 .../execution}/py_create_order_request.cpp | 0 .../execution}/py_create_order_request.h | 0 .../execution}/py_execution_gateway.cpp | 0 .../execution}/py_execution_gateway.h | 6 +- .../execution}/py_execution_report.cpp | 3 +- .../execution}/py_execution_report.h | 62 +-- .../python_interop/execution}/py_module.cpp | 26 +- .../execution}/py_order_request.h | 6 +- .../execution}/py_position_info.cpp | 0 .../execution}/py_position_info.h | 0 .../python_interop/md}/md_bridge.cpp | 0 .../python_interop/md}/md_bridge.h | 6 +- .../python_interop/md}/md_bridge_impl.cpp | 8 +- .../python_interop/md}/md_bridge_impl.h | 4 +- .../python_interop/md}/md_provider.cpp | 2 +- .../python_interop/md}/md_provider.h | 2 +- .../python_interop/md}/py_book.cpp | 2 +- .../python_interop/md}/py_book.h | 4 +- .../python_interop/md}/py_book_level.h | 0 .../python_interop/md}/py_book_side.cpp | 0 .../python_interop/md}/py_book_side.h | 2 +- .../python_interop/md}/py_md_provider.cpp | 0 .../python_interop/md}/py_md_provider.h | 4 +- .../python_interop/md}/py_module.cpp | 8 +- .../python_interop/md}/py_trade.h | 0 .../python_interop}/py_contract_price_info.h | 38 ++ .../py_contract_quantity_info.h | 46 ++ .../py_contract_specification.h | 33 +- .../python_interop}/py_future.h | 0 .../python_interop}/py_option.h | 29 + .../python_interop}/py_security.cpp | 54 +- .../python_interop}/py_security.h | 8 +- .../python_interop}/py_spot.h | 0 .../python_interop}/python_activator.h | 6 +- .../python_interop/python_activator_impl.h | 118 ++++ .../python_interop/python_interop_module.cpp | 276 ++++++++++ .../python_interop}/python_interpreter.cpp | 4 +- .../python_interop}/python_interpreter.h | 4 +- .../bybit_execution_gateway.py | 86 +-- .../bybit_md_provider.py | 23 +- .../python_interop_plugins}/config.py | 0 .../dummy_execution_gateway.py | 14 +- .../dummy_md_provider.py | 32 +- .../gateio_md_provider.py | 22 +- .../py_execution_utils.py | 12 + .../python_interop_plugins}/py_md_utils.py | 10 +- .../python_interop_plugins}/spark_core.py | 93 +++- src/lib_python_md/CMakeLists.txt | 19 - src/lib_python_md/Config.cmake.in | 12 - src/lib_python_md/README.md | 87 --- .../python_md_providers/binance_test.py | 49 -- .../binance_test_md_provider.py | 161 ------ .../python_md_providers/bybit_conf.py | 484 ----------------- .../python_md_providers/pyrightconfig.json | 20 - src/lib_spark/spark/orderman/account_name.h | 8 + 87 files changed, 1093 insertions(+), 2926 deletions(-) delete mode 100644 src/lib_python_core/python_core/python_activator_impl.h delete mode 100644 src/lib_python_core/python_core/python_core_module.cpp delete mode 100644 src/lib_python_core/python_core_plugins/pyrightconfig.json delete mode 100644 src/lib_python_execution/CMakeLists.txt delete mode 100644 src/lib_python_execution/Config.cmake.in delete mode 100644 src/lib_python_execution/python_execution_gateways/binance.env delete mode 100644 src/lib_python_execution/python_execution_gateways/binance_client.py delete mode 100644 src/lib_python_execution/python_execution_gateways/binance_env_helper.py delete mode 100644 src/lib_python_execution/python_execution_gateways/binance_execution_gateway.py delete mode 100644 src/lib_python_execution/python_execution_gateways/py_execution_utils.py delete mode 100644 src/lib_python_execution/python_execution_gateways/pyrightconfig.json delete mode 100644 src/lib_python_execution/python_execution_gateways/simple_test.py rename src/{lib_python_core => lib_python_interop}/CMakeLists.txt (90%) rename src/{lib_python_core => lib_python_interop}/Config.cmake.in (100%) rename src/{lib_python_execution => lib_python_interop}/README.md (61%) rename src/{lib_python_execution => lib_python_interop}/pyrightconfig.json (59%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/execution_gateway.cpp (100%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/execution_gateway.h (67%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/execution_gateway_bridge.cpp (100%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/execution_gateway_bridge.h (76%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/execution_gateway_bridge_impl.cpp (91%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/execution_gateway_bridge_impl.h (84%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_cancel_order_request.cpp (100%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_cancel_order_request.h (100%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_cancel_replace_order_request.cpp (100%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_cancel_replace_order_request.h (100%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_create_order_request.cpp (100%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_create_order_request.h (100%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_execution_gateway.cpp (100%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_execution_gateway.h (90%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_execution_report.cpp (98%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_execution_report.h (81%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_module.cpp (87%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_order_request.h (58%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_position_info.cpp (100%) rename src/{lib_python_execution/python_execution => lib_python_interop/python_interop/execution}/py_position_info.h (100%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/md_bridge.cpp (100%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/md_bridge.h (79%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/md_bridge_impl.cpp (96%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/md_bridge_impl.h (93%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/md_provider.cpp (71%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/md_provider.h (83%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/py_book.cpp (97%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/py_book.h (98%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/py_book_level.h (100%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/py_book_side.cpp (100%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/py_book_side.h (98%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/py_md_provider.cpp (100%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/py_md_provider.h (90%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/py_module.cpp (95%) rename src/{lib_python_md/python_md => lib_python_interop/python_interop/md}/py_trade.h (100%) rename src/{lib_python_core/python_core => lib_python_interop/python_interop}/py_contract_price_info.h (74%) rename src/{lib_python_core/python_core => lib_python_interop/python_interop}/py_contract_quantity_info.h (72%) rename src/{lib_python_core/python_core => lib_python_interop/python_interop}/py_contract_specification.h (79%) rename src/{lib_python_core/python_core => lib_python_interop/python_interop}/py_future.h (100%) rename src/{lib_python_core/python_core => lib_python_interop/python_interop}/py_option.h (67%) rename src/{lib_python_core/python_core => lib_python_interop/python_interop}/py_security.cpp (58%) rename src/{lib_python_core/python_core => lib_python_interop/python_interop}/py_security.h (96%) rename src/{lib_python_core/python_core => lib_python_interop/python_interop}/py_spot.h (100%) rename src/{lib_python_core/python_core => lib_python_interop/python_interop}/python_activator.h (95%) create mode 100644 src/lib_python_interop/python_interop/python_activator_impl.h create mode 100644 src/lib_python_interop/python_interop/python_interop_module.cpp rename src/{lib_python_core/python_core => lib_python_interop/python_interop}/python_interpreter.cpp (95%) rename src/{lib_python_core/python_core => lib_python_interop/python_interop}/python_interpreter.h (98%) rename src/{lib_python_execution/python_execution_gateways => lib_python_interop/python_interop_plugins}/bybit_execution_gateway.py (82%) rename src/{lib_python_md/python_md_providers => lib_python_interop/python_interop_plugins}/bybit_md_provider.py (96%) rename src/{lib_python_md/python_md_providers => lib_python_interop/python_interop_plugins}/config.py (100%) rename src/{lib_python_execution/python_execution_gateways => lib_python_interop/python_interop_plugins}/dummy_execution_gateway.py (96%) rename src/{lib_python_md/python_md_providers => lib_python_interop/python_interop_plugins}/dummy_md_provider.py (77%) rename src/{lib_python_md/python_md_providers => lib_python_interop/python_interop_plugins}/gateio_md_provider.py (95%) create mode 100644 src/lib_python_interop/python_interop_plugins/py_execution_utils.py rename src/{lib_python_md/python_md_providers => lib_python_interop/python_interop_plugins}/py_md_utils.py (80%) rename src/{lib_python_core/python_core_plugins => lib_python_interop/python_interop_plugins}/spark_core.py (79%) delete mode 100644 src/lib_python_md/CMakeLists.txt delete mode 100644 src/lib_python_md/Config.cmake.in delete mode 100644 src/lib_python_md/README.md delete mode 100644 src/lib_python_md/python_md_providers/binance_test.py delete mode 100644 src/lib_python_md/python_md_providers/binance_test_md_provider.py delete mode 100644 src/lib_python_md/python_md_providers/bybit_conf.py delete mode 100644 src/lib_python_md/python_md_providers/pyrightconfig.json diff --git a/CMakeLists.txt b/CMakeLists.txt index aefed1c70..e77d3e05a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,12 +115,10 @@ else() # subdirs add_subdirectory(src/lib_spark) if (BUILD_SPARKD) - add_subdirectory(src/lib_python_core) + add_subdirectory(src/lib_python_interop) add_subdirectory(src/lib_onixs_fix) add_subdirectory(src/lib_binance_execution) add_subdirectory(src/lib_okx_execution) - add_subdirectory(src/lib_python_execution) - add_subdirectory(src/lib_python_md) add_subdirectory(src/app_sparkd) add_subdirectory(src/data_files) endif() diff --git a/cmake/InitPythonModule.cmake b/cmake/InitPythonModule.cmake index 301288016..4b73fa862 100644 --- a/cmake/InitPythonModule.cmake +++ b/cmake/InitPythonModule.cmake @@ -23,7 +23,7 @@ function(init_python_module) PUBLIC pybind11::embed spark - python_core + ${_name} ) target_include_directories( ${module_proj} diff --git a/src/app_sparkd/CMakeLists.txt b/src/app_sparkd/CMakeLists.txt index fd722841d..6f7724c87 100644 --- a/src/app_sparkd/CMakeLists.txt +++ b/src/app_sparkd/CMakeLists.txt @@ -107,39 +107,25 @@ target_link_directories( # The dirs are relative to sparkd's path, represented by "$ORIGIN". set_target_properties( ${PROJECT_NAME} PROPERTIES - BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_core;$ORIGIN/../lib_python_execution;$ORIGIN/../lib_python_md" + BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_interop" INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}" ) include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake) include(${CMAKE_SOURCE_DIR}/cmake/SymLinkPythonModule.cmake) -# Py core +# Py interop sym_link_python_module( - LIB_NAME python_core - MODULE_NAME python_core_module - PLUGIN_NAME python_core_plugins -) -# Py md -sym_link_python_module( - LIB_NAME python_md - MODULE_NAME python_md_module - PLUGIN_NAME python_md_providers -) -# Py execution -sym_link_python_module( - LIB_NAME python_execution - MODULE_NAME python_execution_module - PLUGIN_NAME python_execution_gateways + LIB_NAME python_interop + MODULE_NAME python_interop_module + PLUGIN_NAME python_interop_plugins ) target_link_libraries( sparkd spark binance_execution okx_execution - python_md - python_md_module - python_execution - python_execution_module + python_interop + python_interop_module databento::databento ${emumba-deribit-connector-library_LIBRARIES} tbb diff --git a/src/app_sparkd/configurations/bybit_dev/spark.conf b/src/app_sparkd/configurations/bybit_dev/spark.conf index 25667f682..2bf0b275a 100644 --- a/src/app_sparkd/configurations/bybit_dev/spark.conf +++ b/src/app_sparkd/configurations/bybit_dev/spark.conf @@ -31,8 +31,7 @@ MarketDataConfig { value { python_md_providers { venue: VENUE_BYBIT - module_path: "python_md_providers" - module_path: "python_core_plugins" + module_path: "python_interop_plugins" provider_type: "bybit_md_provider.BybitMdProvider" } } diff --git a/src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.cpp b/src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.cpp index 36088c15b..84bbf89b2 100644 --- a/src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.cpp +++ b/src/app_sparkd/sparkd/execution/py_execution_gateway_adapter.cpp @@ -1,16 +1,17 @@ #include "py_execution_gateway_adapter.h" -#include "python_core/py_contract_price_info.h" -#include "python_core/py_contract_specification.h" -#include "python_core/py_security.h" -#include "python_core/python_activator.h" -#include "python_execution/execution_gateway.h" -#include "python_execution/execution_gateway_bridge.h" -#include "python_execution/execution_gateway_bridge_impl.h" -#include "python_execution/py_cancel_order_request.h" -#include "python_execution/py_cancel_replace_order_request.h" -#include "python_execution/py_create_order_request.h" -#include "python_execution/py_execution_gateway.h" -#include "python_execution/py_execution_report.h" +#include "python_interop/execution/execution_gateway.h" +#include "python_interop/execution/execution_gateway_bridge.h" +#include "python_interop/execution/execution_gateway_bridge_impl.h" +#include "python_interop/execution/py_cancel_order_request.h" +#include "python_interop/execution/py_cancel_replace_order_request.h" +#include "python_interop/execution/py_create_order_request.h" +#include "python_interop/execution/py_execution_gateway.h" +#include "python_interop/execution/py_execution_report.h" +#include "python_interop/py_contract_price_info.h" +#include "python_interop/py_contract_quantity_info.h" +#include "python_interop/py_contract_specification.h" +#include "python_interop/py_security.h" +#include "python_interop/python_activator.h" #include "spark/api/domain/Venue.h" #include "spark/api/service_helpers/AdminServiceUtils.h" #include "spark/api/services/AdminService.pb.h" @@ -125,12 +126,12 @@ using bts::orderman::OrderMsg; using bts::orderman::RFQMsg; using bts::system::string_utils::parse_uint; using nlohmann::json; -using python_core::PythonActivatorT; using python_execution::ExecutionGatewayBridgeImpl; using python_execution::PyCancelOrderRequest; using python_execution::PyCancelReplaceOrderRequest; using python_execution::PyCreateOrderRequest; using python_execution::PyExecutionGateway; +using python_interop::PythonActivatorT; using std::int64_t; using std::optional; using std::shared_ptr; @@ -287,6 +288,17 @@ struct PyExecutionGatewayAdapter::PImpl { std::make_shared(std::move(req))); } + void on_securities(const std::vector &secs) { + std::vector> py_secs; + for (const auto *sec : secs) { + bts::md::py_interop::PySecurity py_sec; + to_py(py_sec, *sec); + py_secs.push_back(std::make_shared( + std::move(py_sec))); + } + bridge.on_securities(std::move(py_secs)); + } + void get_position_info() { sm.invoke_if_active([&](auto) { // @@ -334,9 +346,83 @@ struct PyExecutionGatewayAdapter::PImpl { er.avg_price.value() = py_er.avg_filled_price(); er.stop_price.value() = std::numeric_limits::quiet_NaN(); const string &tx_id = py_er.transact_id(); - bts_assert(tx_id.size() < er.exchange_fill_id.value().size()); - bts::system::string_utils::copy_to_array<40>( - py_er.transact_id(), er.exchange_fill_id.value()); + bts_assert(tx_id.size() < er.exchange_fill_id.value().max_size()); + bts_assert(!py_er.transact_id().empty()); + er.exchange_fill_id->copy_from(py_er.transact_id()); + } + + void to_py( + bts::md::py_interop::PySecurity &py_sec, const bts::api::Security &sec) { + py_sec.set_id(sec.id()); + py_sec.set_asset(sec.symbol()); + py_sec.set_venue(bts::api::venue_name_pretty(sec.exchange())); + py_sec.set_currency(sec.currency()); + // Exchange ID and name + std::string name = bts::securities::descriptor(sec); + py_sec.set_exchange_id(name); + py_sec.set_name(std::move(name)); + // Price ticks + const bts::api::ContractSpecification &spec = sec.contract_spec(); + md::py_interop::PyContractSpecification py_spec; + md::py_interop::PyContractPriceInfo &py_price_info = + py_spec.mutable_price_infos().emplace_back(); + float tick_unit; + float tick_base; + if (!domain::get_tick_factors(spec, tick_unit, tick_base)) { + WARN_FN( + "on_securities", "Failed to get price tick from {} {}. spec: {}. ", + sec.exchange(), name, utils::pb::to_string(sec.contract_spec())); + tick_unit = 0.01f; + tick_base = 0.01f; + } + py_price_info.set_tick_unit(tick_unit); + py_price_info.set_tick_base(tick_base); + // Quantity ticks + md::py_interop::PyContractQuantityInfo &py_qty_info = + py_spec.mutable_quantity_infos().emplace_back(); + if (spec.quantity_infos().empty()) { + tick_unit = 1.0f; + tick_base = 1.0f; + WARN_FN( + "on_securities", "Failed to get qty tick from {} {}. spec: {}. ", + sec.exchange(), name, utils::pb::to_string(sec.contract_spec())); + } else { + tick_unit = spec.quantity_infos(0).tick_unit(); + tick_base = spec.quantity_infos(0).tick_base(); + } + py_qty_info.set_tick_unit(tick_unit); + py_qty_info.set_tick_base(tick_base); + py_sec.set_contract_spec(std::move(py_spec)); + // Spot/Future/Option + switch (sec.Contract_case()) { + case bts::api::Security::kSpot: { + py_sec.set_spot(bts::md::py_interop::PySpot{}); + break; + } + case bts::api::Security::kFuture: { + const bts::api::FutureContract &fut = sec.future(); + bts::md::py_interop::PyFuture py_fut; + py_fut.set_expiration(fmt::format( + "{}-{:02}-{:02}", fut.expiration_year(), fut.expiration_month(), + fut.expiration_day())); + py_sec.set_future(std::move(py_fut)); + break; + } + case bts::api::Security::kOption: { + const bts::api::OptionContract &opt = sec.option(); + bts::md::py_interop::PyOption py_opt; + py_opt.set_expiration(fmt::format( + "{}-{:02}-{:02}", opt.expiration_year(), opt.expiration_month(), + opt.expiration_day())); + py_opt.set_strike(opt.strike()); + py_opt.set_is_call(opt.call_put() == bts::api::PUTORCALL_CALL); + py_sec.set_option(std::move(py_opt)); + break; + } + default: + bts_assert_msg(false, "Only spots, futures, and options supported"); + break; + } } void to_py(PyCreateOrderRequest &py_req, const CreateOrderMsg &msg) { @@ -384,7 +470,7 @@ struct PyExecutionGatewayAdapter::PImpl { sw.start(); while (bridge.pop_exe_report(py_er_p)) { n_dequed++; - const python_execution::PyExecutionReport py_er = *py_er_p; + const python_execution::PyExecutionReport &py_er = *py_er_p; shared_ptr er_p = orders.update(OrderKey{py_er.key()}, [&](auto &er_p, bool) { from_py(py_er, *er_p); @@ -456,69 +542,7 @@ void PyExecutionGatewayAdapter::on_securities( me->sec_oracle.get_securities([&](const bts::api::Security &sec) { return sec.exchange() == me->jetfix_connection->venue(); }); - std::vector> py_secs; - for (const auto *sec : secs) { - bts::md::py_interop::PySecurity py_sec; - py_sec.set_id(sec->id()); - py_sec.set_asset(sec->symbol()); - py_sec.set_venue(bts::api::venue_name_pretty(sec->exchange())); - py_sec.set_currency(sec->currency()); - // Exchange ID and name - std::string name = bts::securities::descriptor(*sec); - py_sec.set_exchange_id(name); - py_sec.set_name(std::move(name)); - // Contract spec - md::py_interop::PyContractSpecification py_spec; - md::py_interop::PyContractPriceInfo &py_price_info = - py_spec.mutable_price_infos().emplace_back(); - float tick_unit; - float tick_base; - if (!domain::get_tick_factors( - sec->contract_spec(), tick_unit, tick_base)) { - WARN_FN( - "on_securities", "Security {} {} has invalid contract spec: {}. ", - sec->exchange(), bts::securities::descriptor(*sec), - utils::pb::to_string(sec->contract_spec())); - tick_unit = 0.01f; - tick_base = 0.01f; - } - py_price_info.set_tick_unit(tick_unit); - py_price_info.set_tick_base(tick_base); - py_sec.set_contract_spec(std::move(py_spec)); - // Spot/Future/Option - switch (sec->Contract_case()) { - case bts::api::Security::kSpot: { - py_sec.set_spot(bts::md::py_interop::PySpot{}); - break; - } - case bts::api::Security::kFuture: { - const bts::api::FutureContract &fut = sec->future(); - bts::md::py_interop::PyFuture py_fut; - py_fut.set_expiration(fmt::format( - "{}-{:02}-{:02}", fut.expiration_year(), fut.expiration_month(), - fut.expiration_day())); - py_sec.set_future(std::move(py_fut)); - break; - } - case bts::api::Security::kOption: { - const bts::api::OptionContract &opt = sec->option(); - bts::md::py_interop::PyOption py_opt; - py_opt.set_expiration(fmt::format( - "{}-{:02}-{:02}", opt.expiration_year(), opt.expiration_month(), - opt.expiration_day())); - py_opt.set_strike(opt.strike()); - py_opt.set_is_call(opt.call_put() == bts::api::PUTORCALL_CALL); - py_sec.set_option(std::move(py_opt)); - break; - } - default: - bts_assert_msg(false, "Only spots, futures, and options supported"); - break; - } - py_secs.push_back( - std::make_shared(std::move(py_sec))); - } - me->bridge.on_securities(std::move(py_secs)); + me->on_securities(secs); } void PyExecutionGatewayAdapter::on_admin_data( diff --git a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp index 92c53f811..86294ae5f 100644 --- a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp +++ b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp @@ -1,9 +1,9 @@ #include "PyMarketDataSubscription.h" -#include "python_core/py_contract_specification.h" -#include "python_core/py_security.h" -#include "python_md/md_bridge_impl.h" -#include "python_md/py_book.h" -#include "python_md/py_trade.h" +#include "python_interop/md/md_bridge_impl.h" +#include "python_interop/md/py_book.h" +#include "python_interop/md/py_trade.h" +#include "python_interop/py_contract_specification.h" +#include "python_interop/py_security.h" #include "spark/api/domain/CallPut.pb.h" #include "spark/api/domain/SecurityType.h" #include "spark/api/domain/Venue.h" @@ -45,23 +45,24 @@ namespace bts::md { #define WARN_TRACE(name) _LOG_TRACE(bts::Warn, name) #define ERROR_TRACE(name) _LOG_TRACE(bts::Error, name) -#define LOG_DISPATCH_STATS(level, typ, n, dur) \ - _LOG( \ - level, \ - R"("[py_md_dispatch.json]{{ "typ": "{}", "n_recs": {}, "dur": "{:.2f}ms", "ts": {} }})", \ - n, dur.total_millis(), TimeStamp::now_ns()) +#define LOG_DISPATCH_STATS(level, typ, n, dur) \ + _LOG( \ + level, \ + R"([py_md_dispatch.json]{{ "typ": ")" typ \ + R"(", "n_recs": {}, "dur": "{:.2f}ms", "ts": {} }})", \ + n, dur.total_millis(), bts::core::TimeStamp::now_ns()) #define LOG_SECURITY(level, py_sec) \ LOG(md, level, R"([py_md_sec.json]{{ "sec": {}, "ts": {}}})", \ - json_formatter(py_sec), TimeStamp::now_ns()) + json_formatter(py_sec), bts::core::TimeStamp::now_ns()) #define LOG_BOOK(level, py_book) \ LOG(md, level, R"([py_md_book.json]{{ "bk": {}, "ts": {}}})", \ - json_formatter(py_book), TimeStamp::now_ns()) + json_formatter(py_book), bts::core::TimeStamp::now_ns()) #define LOG_TRADE(level, py_trade) \ LOG(md, level, R"([py_md_trade.json]{{ "trd": {}, "ts": {}}})", \ - json_formatter(py_trade), TimeStamp::now_ns()) + json_formatter(py_trade), bts::core::TimeStamp::now_ns()) using bts::api::domain::CallPut; using bts::api::domain::SecurityType; @@ -302,9 +303,8 @@ struct PyMarketDataSubscription::PImpl { bts_assert(qty_info.tick_unit() > 0 && qty_info.tick_base() > 0); sec_comm.min_quantity_increment = qty_info.tick_unit() / qty_info.tick_base(); - sec_comm.min_quantity = std::max( - qty_info.min_value(), - static_cast(sec_comm.min_quantity_increment)); + sec_comm.min_quantity = + std::max(qty_info.min_value(), sec_comm.min_quantity_increment); } sec_comm.desc = py_sec.name(); if (py_sec.is_spot()) { diff --git a/src/lib_python_core/python_core/python_activator_impl.h b/src/lib_python_core/python_core/python_activator_impl.h deleted file mode 100644 index 8f4fc750c..000000000 --- a/src/lib_python_core/python_core/python_activator_impl.h +++ /dev/null @@ -1,140 +0,0 @@ -#pragma once - -#include "python_core/python_interpreter.h" -#include "spark/logging/logging.h" - -#include -#include -#include -#include - -namespace python_core { - -struct PythonActivator_Impl { - std::string target; - std::vector module_paths; - std::optional target_obj; - - PythonActivator_Impl( - std::string target, std::vector module_paths) - : target(std::move(target)), module_paths(std::move(module_paths)) {} - - ~PythonActivator_Impl() { reset(); } - - template - pybind11::object create_target_with_args( - std::function init, Args &&...args) { - LOG(OrderMan, bts::Info, "PythonActivator::create_target. {}", target); - return PythonInterpreter::exec_t( - [this, &init, - args_tuple = - std::make_tuple(std::forward(args)...)]() mutable { - std::string target_module_name; - std::string target_type_name; - PythonInterpreter::parse_target( - target, &target_module_name, &target_type_name); - // Prepare module path - pybind11::module sys = pybind11::module::import("sys"); - for (const auto &module_path : module_paths) { - sys.attr("path").attr("append")(module_path.c_str()); - } - // Import target module - pybind11::module target_module = - pybind11::module::import(target_module_name.c_str()); - // Instantiate target using tuple expansion - pybind11::object obj = std::apply( - [&](auto &&...unpacked_args) { - return target_module.attr(target_type_name.c_str())( - std::forward( - unpacked_args)...); - }, - std::move(args_tuple)); - init(obj); - return obj; - }); - } - - pybind11::object &get_target_obj() { - if (!target_obj) { - // Prevent python from overwriting our signal handler (so ctrl+c - // actually terminates the process). - auto orig_signal_handler = std::signal(SIGINT, SIG_DFL); - target_obj = create_target(); - std::signal(SIGINT, orig_signal_handler); - } - return *target_obj; - } - - pybind11::object &get_target_obj( - std::function init) { - if (!target_obj) { - // Prevent python from overwriting our signal handler (so ctrl+c - // actually terminates the process). - auto orig_signal_handler = std::signal(SIGINT, SIG_DFL); - target_obj = create_target(init); - std::signal(SIGINT, orig_signal_handler); - } - return *target_obj; - } - - template - pybind11::object &get_target_obj( - std::function init, Args &&...args) { - if (!target_obj) { - // Prevent python from overwriting our signal handler (so ctrl+c - // actually terminates the process). - auto orig_signal_handler = std::signal(SIGINT, SIG_DFL); - target_obj = create_target_with_args( - init, std::forward(args)...); - std::signal(SIGINT, orig_signal_handler); - } - return *target_obj; - } - - pybind11::object create_target() { - LOG(OrderMan, bts::Info, "PythonActivator::create_target. {}", target); - return PythonInterpreter::exec_t([&]() { - std::string target_module_name; - std::string target_type_name; - PythonInterpreter::parse_target( - target, &target_module_name, &target_type_name); - // Prepare module path - pybind11::module sys = pybind11::module::import("sys"); - for (const auto &module_path : module_paths) { - sys.attr("path").attr("append")(module_path.c_str()); - } - // Import target module - pybind11::module target_module = - pybind11::module::import(target_module_name.c_str()); - // Instantiate target - return target_module.attr(target_type_name.c_str())(); - }); - } - - pybind11::object create_target( - std::function init) { - LOG(OrderMan, bts::Info, "PythonActivator::create_target. {}", target); - return PythonInterpreter::exec_t([&]() { - std::string target_module_name; - std::string target_type_name; - PythonInterpreter::parse_target( - target, &target_module_name, &target_type_name); - // Prepare module path - pybind11::module sys = pybind11::module::import("sys"); - for (const auto &module_path : module_paths) { - sys.attr("path").attr("append")(module_path.c_str()); - } - // Import target module - pybind11::module target_module = - pybind11::module::import(target_module_name.c_str()); - // Instantiate target - pybind11::object obj = target_module.attr(target_type_name.c_str())(); - init(obj); - return obj; - }); - } - - void reset() { target_obj.reset(); } -}; - -} // namespace python_core diff --git a/src/lib_python_core/python_core/python_core_module.cpp b/src/lib_python_core/python_core/python_core_module.cpp deleted file mode 100644 index 494340b65..000000000 --- a/src/lib_python_core/python_core/python_core_module.cpp +++ /dev/null @@ -1,97 +0,0 @@ -#include "python_core/py_contract_price_info.h" -#include "python_core/py_contract_quantity_info.h" -#include "python_core/py_contract_specification.h" -#include "python_core/py_security.h" - -#include -#include -#include - -using bts::md::py_interop::PyContractPriceInfo; -using bts::md::py_interop::PyContractQuantityInfo; -using bts::md::py_interop::PyContractSpecification; -using bts::md::py_interop::PyFuture; -using bts::md::py_interop::PyOption; -using bts::md::py_interop::PySecurity; -using bts::md::py_interop::PySpot; -using std::shared_ptr; - -PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr) - -namespace py = pybind11; - -#define _STR(x) #x -// In general, we bind c++ methods to python methods with the same name verbatim. -// The DEF macro avoids typos in the python method name for this common case. -#define DEF(type, member) def(_STR(member), &type::member) -#define DEF_ARGS(type, member, ...) \ - def(_STR(member), &type::member, __VA_ARGS__) -#define EXPLICIT_DEF(type, member, member_type) \ - def(_STR(member), (member_type) & type::member) -// Defines get/set methods for a pseudo property -#define PROP(type, name) \ - def(_STR(name), &type::name) \ - .def(_STR(set_##name), &type::set_##name, py::arg(#name)) - -// Define python module with c++ classes. -PYBIND11_MODULE(python_core_module, m) { - // price info - py::class_(m, "PyContractPriceInfo") - .def(py::init<>()) - .PROP(PyContractPriceInfo, tick_unit) - .PROP(PyContractPriceInfo, tick_base) - .PROP(PyContractPriceInfo, multiplier) - .PROP(PyContractPriceInfo, cash_per_point) - .PROP(PyContractPriceInfo, max_price) - .def("__str__", &PyContractPriceInfo::to_string); - py::bind_vector>( - m, "PyContractPriceInfoList"); - // quantity info - py::class_(m, "PyContractQuantityInfo") - .def(py::init<>()) - .PROP(PyContractQuantityInfo, min_value) - .PROP(PyContractQuantityInfo, max_value) - .PROP(PyContractQuantityInfo, tick_unit) - .PROP(PyContractQuantityInfo, tick_base) - .PROP(PyContractQuantityInfo, multiplier) - .PROP(PyContractQuantityInfo, max_price) - .def("__str__", &PyContractQuantityInfo::to_string); - py::bind_vector>( - m, "PyContractQuantityInfoList"); - // contract spec - py::class_(m, "PyContractSpecification") - .def(py::init<>()) - .PROP(PyContractSpecification, price_infos) - .PROP(PyContractSpecification, quantity_infos) - .PROP(PyContractSpecification, notional_quantity_infos) - .def("__str__", &PyContractSpecification::to_string); - // security - py::class_(m, "PySpot") - .def(py::init<>()) - .def("__str__", &PySpot::to_string); - py::class_(m, "PyFuture") - .def(py::init<>()) - .PROP(PyFuture, expiration) - .def("__str__", &PyFuture::to_string); - py::class_(m, "PyOption") - .def(py::init<>()) - .PROP(PyOption, expiration) - .PROP(PyOption, is_call) - .PROP(PyOption, strike) - .def("__str__", &PyOption::to_string); - py::class_>(m, "PySecurity") - .def(py::init<>()) - .PROP(PySecurity, id) - .PROP(PySecurity, exchange_id) - .PROP(PySecurity, venue) - .PROP(PySecurity, name) - .PROP(PySecurity, currency) - .PROP(PySecurity, group) - .PROP(PySecurity, asset) - .PROP(PySecurity, contract_spec) - .PROP(PySecurity, spot) - .PROP(PySecurity, future) - .PROP(PySecurity, option) - .def("__str__", &PySecurity::to_string); - py::bind_vector>>(m, "PySecurityList"); -} diff --git a/src/lib_python_core/python_core_plugins/pyrightconfig.json b/src/lib_python_core/python_core_plugins/pyrightconfig.json deleted file mode 100644 index be2a21871..000000000 --- a/src/lib_python_core/python_core_plugins/pyrightconfig.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "include": [ - ".", - ], - "exclude": [ - "**/__pycache__" - ], - "reportMissingImports": true, - "reportMissingTypeStubs": false, - "stubPath": "../../../build/sparkd-debug-clang/src/lib_python_core/stubs", - "pythonVersion": "3.10", - "pythonPlatform": "Linux", - "typeCheckingMode": "basic", - "venvPath": "../../../", - "venv": ".venv" -} diff --git a/src/lib_python_execution/CMakeLists.txt b/src/lib_python_execution/CMakeLists.txt deleted file mode 100644 index 3d58fb471..000000000 --- a/src/lib_python_execution/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -cmake_minimum_required(VERSION 3.23) -project(python_execution) -include(${CMAKE_SOURCE_DIR}/cmake/InitLibrary.cmake) -include(${CMAKE_SOURCE_DIR}/cmake/InitPythonModule.cmake) -include(${CMAKE_SOURCE_DIR}/cmake/InitPythonPlugin.cmake) - -# Find Python from the virtual environment -find_package(Python3 REQUIRED COMPONENTS Interpreter Development) -find_package(pybind11 REQUIRED) -# Shared lib (for c++ to link) -init_library(SHARED) -target_include_directories(${PROJECT_NAME} PUBLIC ${Python3_INCLUDE_DIRS}) -target_link_directories(${PROJECT_NAME} PUBLIC ${Python3_LIBRARY_DIRS}) -target_link_libraries(${PROJECT_NAME} PUBLIC spark python_core pybind11::embed - ${Python3_LIBRARIES}) -# CPython module (for .py files to import) -init_python_module() -# Py plugin (for sparkd to load at runtime) -init_python_plugin(PLUGIN_NAME python_execution_gateways) diff --git a/src/lib_python_execution/Config.cmake.in b/src/lib_python_execution/Config.cmake.in deleted file mode 100644 index 8cd046cf0..000000000 --- a/src/lib_python_execution/Config.cmake.in +++ /dev/null @@ -1,12 +0,0 @@ -@PACKAGE_INIT@ - -include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") -include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@_moduleTargets.cmake") -include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@_gatewaysTargets.cmake") - -find_dependency(Python3 REQUIRED COMPONENTS Interpreter Development) -find_dependency(pybind11) - -check_required_components(@PROJECT_NAME@) -check_required_components(@PROJECT_NAME@_module) -check_required_components(@PROJECT_NAME@_gateways) diff --git a/src/lib_python_execution/python_execution_gateways/binance.env b/src/lib_python_execution/python_execution_gateways/binance.env deleted file mode 100644 index c5dd93878..000000000 --- a/src/lib_python_execution/python_execution_gateways/binance.env +++ /dev/null @@ -1,191 +0,0 @@ -BINANCE_USDS_FUTURES_PROD_URL="https://fapi.binance.com" -BINANCE_USDS_FUTURES_PROD_WS_URL="wss://fstream.binance.com:443/ws" - -BINANCE_USDS_FUTURES_DEV_URL="https://testnet.binancefuture.com" -BINANCE_USDS_FUTURES_DEV_WS_URL="wss://stream.binancefuture.com:443/ws" -BINANCE_USDS_FUTURES_DEV_API_KEY="b838c108cd2648e34a4f088a7fc635a9cbccf3c2faa6bcaabcb532ffacc087b0" -BINANCE_USDS_FUTURES_DEV_SECRET_KEY="a68f0ab27403c42fa0306b6341b2ddd51d99ac42916f849404155449457d0492" - -BINANCE_USDS_FUTURES_EXCHANGE_INFO_PATH="/fapi/v1/exchangeInfo" - -BINANCE_USDS_FUTURES_USER_DATA_PATH="/ws/${LISTEN_KEY}" - -BINANCE_USDS_FUTURES_DEPTH_PATH="/fapi/v1/depth" -BINANCE_USDS_FUTURES_DEPTH_PARAMS='symbol=${SYMBOL}&limit=${LIMIT:-5}' - -BINANCE_USDS_FUTURES_RECENT_TRADES_PATH="/fapi/v1/trades" -BINANCE_USDS_FUTURES_RECENT_TRADES_PARAMS='symbol=${SYMBOL}&limit=${LIMIT:-500}' - -BINANCE_USDS_FUTURES_HISTORICAL_TRADES_PATH="/fapi/v1/historicalTrades" -BINANCE_USDS_FUTURES_HISTORICAL_TRADES_PARAMS='symbol=${SYMBOL}&limit=${LIMIT:-500}' - -BINANCE_USDS_FUTURES_MARK_PRICE_PATH="/fapi/v1/premiumIndex" -BINANCE_USDS_FUTURES_MARK_PRICE_PARAMS='symbol=${SYMBOL}' - -BINANCE_USDS_FUTURES_PRICE_TICKER_PATH="/fapi/v1/ticker/price" -BINANCE_USDS_FUTURES_PRICE_TICKER_PARAMS='symbol=${SYMBOL}' - -BINANCE_USDS_FUTURES_BBO_PATH="/fapi/v1/ticker/bookTicker" -BINANCE_USDS_FUTURES_BBO_PARAMS='symbol=${SYMBOL}' - -BINANCE_USDS_FUTURES_OPEN_INTEREST_PATH="/dapi/v1/openInterest" -BINANCE_USDS_FUTURES_OPEN_INTEREST_PARAMS='symbol=${SYMBOL}' - -BINANCE_USDS_FUTURES_LONG_SHORT_RATIO_PATH="/futures/data/globalLongShortAccountRatio" -BINANCE_USDS_FUTURES_LONG_SHORT_RATIO_PARAMS='symbol=${SYMBOL}&period=${PERIOD:-30}&limit=${LIMIT:-10}' - -BINANCE_USDS_FUTURES_CREATE_ORDER_PATH="/fapi/v1/order" -BINANCE_USDS_FUTURES_CREATE_ORDER_PARAMS='symbol=${SYMBOL}&side=${SIDE:-BUY}&type=${TYPE:-LIMIT}&timeInForce=${TIME_IN_FORCE:-GTC}&quantity=${QUANTITY:-OMIT}&price=${PRICE:-OMIT}&newClientOrderId=${NEW_CLIENT_ORDER_ID:-OMIT}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_USDS_FUTURES_CREATE_ORDER_METHOD="POST" -BINANCE_USDS_FUTURES_CREATE_ORDER_IS_SIGNED=1 - -BINANCE_USDS_FUTURES_MODIFY_ORDER_PATH="/fapi/v1/order" -BINANCE_USDS_FUTURES_MODIFY_ORDER_PARAMS='symbol=${SYMBOL}&side=${SIDE:-BUY}&quantity=${QUANTITY:-OMIT}&price=${PRICE:-OMIT}&orderId=${ORDER_ID:-OMIT}&origClientOrderId=${ORIG_CLIENT_ORDER_ID:-OMIT}&priceMatch=${PRICE_MATCH:-OMIT}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_USDS_FUTURES_MODIFY_ORDER_METHOD="PUT" -BINANCE_USDS_FUTURES_MODIFY_ORDER_IS_SIGNED=1 - -BINANCE_USDS_FUTURES_CANCEL_ORDER_PATH="/fapi/v1/order" -BINANCE_USDS_FUTURES_CANCEL_ORDER_PARAMS='symbol=${SYMBOL}&orderId=${ORDER_ID:-OMIT}&origClientOrderId=${ORIG_CLIENT_ORDER_ID:-OMIT}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_USDS_FUTURES_CANCEL_ORDER_METHOD="DELETE" -BINANCE_USDS_FUTURES_CANCEL_ORDER_IS_SIGNED=1 - -BINANCE_USDS_FUTURES_CANCEL_OPEN_ORDERS_PATH="/fapi/v1/allOpenOrders" -BINANCE_USDS_FUTURES_CANCEL_OPEN_ORDERS_PARAMS='symbol=${SYMBOL}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_USDS_FUTURES_CANCEL_OPEN_ORDERS_METHOD="DELETE" -BINANCE_USDS_FUTURES_CANCEL_OPEN_ORDERS_IS_SIGNED=1 - -BINANCE_USDS_FUTURES_QUERY_ORDER_PATH="/fapi/v1/order" -BINANCE_USDS_FUTURES_QUERY_ORDER_PARAMS='symbol=${SYMBOL}&orderId=${ORDER_ID:-OMIT}&origClientOrderId=${ORIG_CLIENT_ORDER_ID:-OMIT}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_USDS_FUTURES_QUERY_ORDER_METHOD="GET" -BINANCE_USDS_FUTURES_QUERY_ORDER_IS_SIGNED=1 - -BINANCE_USDS_FUTURES_QUERY_ALL_ORDERS_PATH="/fapi/v1/allOrders" -BINANCE_USDS_FUTURES_QUERY_ALL_ORDERS_PARAMS='symbol=${SYMBOL}&orderId=${ORDER_ID:-OMIT}&startTime=${START_TIME:-OMIT}&endTime=${END_TIME:-OMIT}&limit=${LIMIT:-OMIT}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_USDS_FUTURES_QUERY_ALL_ORDERS_METHOD="GET" -BINANCE_USDS_FUTURES_QUERY_ALL_ORDERS_IS_SIGNED=1 - -BINANCE_USDS_FUTURES_OPEN_ORDERS_PATH="/fapi/v1/openOrders" -BINANCE_USDS_FUTURES_OPEN_ORDERS_PARAMS='symbol=${SYMBOL:-OMIT}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_USDS_FUTURES_OPEN_ORDERS_METHOD="GET" -BINANCE_USDS_FUTURES_OPEN_ORDERS_IS_SIGNED=1 - -BINANCE_USDS_FUTURES_ACCOUNT_TRADES_PATH="/fapi/v1/userTrades" -BINANCE_USDS_FUTURES_ACCOUNT_TRADES_PARAMS='symbol=${SYMBOL}&orderId=${ORDER_ID:-OMIT}&startTime=${START_TIME:-OMIT}&endTime=${END_TIME:-OMIT}&fromId=${FROM_ID:-OMIT}&limit=${LIMIT:-OMIT}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_USDS_FUTURES_ACCOUNT_TRADES_METHOD="GET" -BINANCE_USDS_FUTURES_ACCOUNT_TRADES_IS_SIGNED=1 - -BINANCE_USDS_FUTURES_POSITION_INFO_PATH="/fapi/v3/positionRisk" -BINANCE_USDS_FUTURES_POSITION_INFO_PARAMS='symbol=${SYMBOL:-OMIT}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_USDS_FUTURES_POSITION_INFO_METHOD="GET" -BINANCE_USDS_FUTURES_POSITION_INFO_IS_SIGNED=1 - -BINANCE_USDS_FUTURES_LISTEN_KEY_PATH="/fapi/v1/listenKey" -BINANCE_USDS_FUTURES_LISTEN_KEY_PARAMS='' -BINANCE_USDS_FUTURES_LISTEN_KEY_METHOD="POST" -BINANCE_USDS_FUTURES_LISTEN_KEY_IS_KEYED=1 - -BINANCE_USDS_FUTURES_REFRESH_LISTEN_KEY_PATH="/fapi/v1/listenKey" -BINANCE_USDS_FUTURES_REFRESH_LISTEN_KEY_PARAMS='listenKey=${LISTEN_KEY:-OMIT}' -BINANCE_USDS_FUTURES_REFRESH_LISTEN_KEY_METHOD="PUT" -BINANCE_USDS_FUTURES_REFRESH_LISTEN_KEY_IS_KEYED=1 - -BINANCE_COIN_FUTURES_PROD_URL="https://dapi.binance.com" -BINANCE_COIN_FUTURES_DEV_URL="https://testnet.binancefuture.com" -BINANCE_COIN_FUTURES_EXCHANGE_INFO_PATH="/dapi/v1/exchangeInfo" -BINANCE_COIN_FUTURES_DEPTH_PATH="/dapi/v1/depth" -BINANCE_COIN_FUTURES_RECENT_TRADES_PATH="/dapi/v1/trades" -BINANCE_COIN_FUTURES_HISTORICAL_TRADES_PATH="/dapi/v1/historicalTrades" -BINANCE_COIN_FUTURES_MARK_PRICE_PATH="/dapi/v1/premiumIndex" -BINANCE_COIN_FUTURES_PRICE_TICKER_PATH="/dapi/v1/ticker/price" -BINANCE_COIN_FUTURES_BBO_PATH="/dapi/v1/ticker/bookTicker" -BINANCE_COIN_FUTURES_OPEN_INTEREST_PATH="/dapi/v1/openInterest" -BINANCE_COIN_FUTURES_LONG_SHORT_RATIO_PATH="/futures/data/globalLongShortAccountRatio" - -BINANCE_PROD_URL="https://api.binance.com" -BINANCE_PROD_WS_URL="wss://stream.binance.com:9443/ws" -BINANCE_DEV_URL="https://testnet.binance.vision" -BINANCE_DEV_WS_URL="wss://stream.testnet.binance.vision:9443/ws" -BINANCE_DEV_API_KEY="QGwn9OVcpRTU76gOLV0gmQyVAJUmXW6w7H6y9tH8ctanzVnpZN1TP27liObE1Wsd" -BINANCE_DEV_SECRET_KEY="7ohKg6X3DcsZMGaJmngC84EglsCFse9flMl8aKisa5FUY18LjGCEEdm0KEZljk9T" - -BINANCE_EXCHANGE_INFO_PATH="/api/v3/exchangeInfo" - -BINANCE_DEPTH_PATH="/api/v3/depth" -BINANCE_DEPTH_PARAMS='symbol=${SYMBOL}&limit=${LIMIT:-5}' - -BINANCE_OPEN_ORDERS_PATH="/api/v3/openOrders" -BINANCE_OPEN_ORDERS_PARAMS='symbol=${SYMBOL:-OMIT}' -BINANCE_OPEN_ORDERS_IS_SIGNED=1 - -BINANCE_ACCOUNT_PATH="/api/v3/account" -BINANCE_ACCOUNT_PARAMS='recvWindow=${RECV_WINDOW:-6000}' -BINANCE_ACCOUNT_IS_SIGNED=1 - -BINANCE_CANCEL_OPEN_ORDERS_PATH="/api/v3/openOrders" -BINANCE_CANCEL_OPEN_ORDERS_PARAMS='symbol=${SYMBOL}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_CANCEL_OPEN_ORDERS_METHOD="DELETE" -BINANCE_CANCEL_OPEN_ORDERS_IS_SIGNED=1 - -BINANCE_CREATE_ORDER_PATH="/api/v3/order" -BINANCE_CREATE_ORDER_PARAMS='symbol=${SYMBOL}&side=${SIDE:-BUY}&type=${TYPE:-LIMIT}&timeInForce=${TIME_IN_FORCE:-GTC}&quantity=${QUANTITY:-OMIT}&price=${PRICE:-OMIT}&newClientOrderId=${NEW_CLIENT_ORDER_ID:-OMIT}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_CREATE_ORDER_METHOD="POST" -BINANCE_CREATE_ORDER_IS_SIGNED=1 - -BINANCE_CANCEL_REPLACE_PATH="/api/v3/order/cancelReplace" -BINANCE_CANCEL_REPLACE_PARAMS='symbol=${SYMBOL}&side=${SIDE:-BUY}&type=${TYPE:-LIMIT}&timeInForce=${TIME_IN_FORCE:-GTC}&quantity=${QUANTITY:-OMIT}&price=${PRICE:-OMIT}&cancelReplaceMode=${CANCEL_REPLACE_MODE:-STOP_ON_FAILURE}&cancelNewClientOrderId=${CANCEL_NEW_CLIENT_ORDER_ID:-OMIT}&cancelOrigClientOrderId=${CANCEL_ORIG_CLIENT_ORDER_ID:-OMIT}&cancelOrderId=${CANCEL_ORDER_ID:-OMIT}&newClientOrderId=${NEW_CLIENT_ORDER_ID:-OMIT}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_CANCEL_REPLACE_METHOD="POST" -BINANCE_CANCEL_REPLACE_IS_SIGNED=1 - -BINANCE_LISTEN_KEY_PATH="/api/v3/userDataStream" -BINANCE_LISTEN_KEY_PARAMS='' -BINANCE_LISTEN_KEY_METHOD="POST" -BINANCE_LISTEN_KEY_IS_KEYED=1 - -BINANCE_REFRESH_LISTEN_KEY_PATH="/api/v3/userDataStream" -BINANCE_REFRESH_LISTEN_KEY_PARAMS='listenKey=${LISTEN_KEY}' -BINANCE_REFRESH_LISTEN_KEY_METHOD="PUT" -BINANCE_REFRESH_LISTEN_KEY_IS_KEYED=1 - -BINANCE_CROSS_MARGIN_COLLATERAL_RATIO_PATH="/sapi/v1/margin/crossMarginCollateralRatio" -BINANCE_CROSS_MARGIN_COLLATERAL_RATIO_METHOD="GET" -BINANCE_CROSS_MARGIN_COLLATERAL_RATIO_IS_SIGNED=1 - -BINANCE_CROSS_MARGIN_PAIRS_PATH="/sapi/v1/margin/allPairs" -BINANCE_CROSS_MARGIN_PAIRS_PARAMS='symbol=${SYMBOL:-OMIT}' -BINANCE_CROSS_MARGIN_PAIRS_METHOD="GET" -BINANCE_CROSS_MARGIN_PAIRS_IS_SIGNED=1 - -BINANCE_CROSS_MARGIN_ORDER_PATH="/sapi/v1/margin/order" -BINANCE_CROSS_MARGIN_ORDER_PARAMS='symbol=${SYMBOL}&side=${SIDE}&type=${TYPE}&quantity=${QUANTITY}&price=${PRICE}&timeInForce=GTC&autoRepayAtCancel=true&sideEffectType=AUTO_BORROW_REPAY&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_CROSS_MARGIN_ORDER_METHOD="POST" -BINANCE_CROSS_MARGIN_ORDER_IS_SIGNED=1 - -BINANCE_CROSS_MARGIN_OPEN_ORDERS_PATH="/sapi/v1/margin/openOrders" -BINANCE_CROSS_MARGIN_OPEN_ORDERS_PARAMS='symbol=${SYMBOL}&isIsolated=${IS_ISOLATED:-OMIT}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_CROSS_MARGIN_OPEN_ORDERS_METHOD="GET" -BINANCE_CROSS_MARGIN_OPEN_ORDERS_IS_SIGNED=1 - -BINANCE_CROSS_MARGIN_CANCEL_PATH="/sapi/v1/margin/order" -BINANCE_CROSS_MARGIN_CANCEL_PARAMS='symbol=${SYMBOL}&isIsolated=${IS_ISOLATED:-OMIT}&orderId=${ORDER_ID:-OMIT}&origClientOrderId=${ORIG_CLIENT_ORDER_ID:-OMIT}&newClientOrderId=${NEW_CLIENT_ORDER_ID:-OMIT}&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_CROSS_MARGIN_CANCEL_METHOD="DELETE" -BINANCE_CROSS_MARGIN_CANCEL_IS_SIGNED=1 - -BINANCE_CROSS_MARGIN_LISTEN_KEY_PATH="/sapi/v1/userDataStream" -BINANCE_CROSS_MARGIN_LISTEN_KEY_PARAMS='' -BINANCE_CROSS_MARGIN_LISTEN_KEY_METHOD="POST" -BINANCE_CROSS_MARGIN_LISTEN_KEY_IS_KEYED=1 - -BINANCE_CROSS_MARGIN_REFRESH_LISTEN_KEY_PATH="/sapi/v1/userDataStream" -BINANCE_CROSS_MARGIN_REFRESH_LISTEN_KEY_PARAMS='listenKey=${LISTEN_KEY}' -BINANCE_CROSS_MARGIN_REFRESH_LISTEN_KEY_METHOD="PUT" -BINANCE_CROSS_MARGIN_REFRESH_LISTEN_KEY_IS_KEYED=1 - -BINANCE_ISOLATED_MARGIN_SYMBOL_PATH="/sapi/v1/margin/order" -BINANCE_ISOLATED_MARGIN_SYMBOL_PARAMS='symbol=${SYMBOL}&side=${SIDE}&type=${TYPE}&quantity=${QUANTITY}&price=${PRICE}&autoRepayAtCancel=true&recvWindow=${RECV_WINDOW:-6000}' -BINANCE_ISOLATED_MARGIN_SYMBOL_METHOD="GET" -BINANCE_ISOLATED_MARGIN_SYMBOL_IS_SIGNED=1 - -BINANCE_ALL_MARGIN_ASSETS_PATH="/sapi/v1/margin/allAssets" -BINANCE_ALL_MARGIN_ASSETS_PARAMS='asset=${ASSET:-OMIT}' -BINANCE_ALL_MARGIN_ASSETS_METHOD="GET" -BINANCE_ALL_MARGIN_ASSETS_IS_SIGNED=1 diff --git a/src/lib_python_execution/python_execution_gateways/binance_client.py b/src/lib_python_execution/python_execution_gateways/binance_client.py deleted file mode 100644 index fd5d35a90..000000000 --- a/src/lib_python_execution/python_execution_gateways/binance_client.py +++ /dev/null @@ -1,503 +0,0 @@ -import argparse -import uuid -import hmac -import hashlib -import os -import logging -import sys -import time -import websocket -import json -import requests -import python_execution_module as pyexe -from string import Template -from threading import Thread -from typing import Any, Callable, Mapping, MutableMapping, Optional, TextIO - -from binance_env_helper import ( - required_env_var, - expand_vars, - expand_defaults, - load_env, -) - - -def hmac_hashing(api_secret: str, payload: str): - m = hmac.new(api_secret.encode("utf-8"), payload.encode("utf-8"), hashlib.sha256) - return m.hexdigest() - - -def rest_request(base_url: str, api_key: str, api_secret: str, parameters: dict = {}): - parameters["timestamp"] = int(time.time() * 1000) - sorted_keys = list(parameters) - sorted_keys.sort() - query_params = "&".join([f"{k}={parameters[k]}" for k in sorted_keys]) - signature = hmac_hashing(api_secret=api_secret, payload=query_params) - parameters["signature"] = signature - query_params += "&signature=" + signature - url = base_url + "/api/v3/userDataStream" + "?" + query_params - res = requests.post(url, headers={"X-MBX-APIKEY": api_key}) - return res.text - - -class CircularBufferWriter: - def __init__(self, buffer_line_count: int, sink: TextIO = sys.stdout): - self.buffer_lines = [] - self.buffer_line_count = buffer_line_count - self.sink = sink - - def print(self, msg): - if self.buffer_line_count <= 0: - self.sink.write(msg + "\n") - return - for _ in range(0, len(self.buffer_lines)): - self.sink.write("\x1b[1A\x1b[2K") # move up cursor and delete whole line - if len(self.buffer_lines) == self.buffer_line_count: - self.buffer_lines.pop() - self.buffer_lines.insert(0, msg) - lines = "" - num_lines = len(self.buffer_lines) - for i in range(0, num_lines): - lines += self.buffer_lines[num_lines - i - 1] + "\n" - self.sink.write(lines) - - -def _get_arg_parser(): - # Cli Args - arg_parser = argparse.ArgumentParser(description="Request market data at Binance") - arg_parser.add_argument( - "--streams", - help="Streams to subscribe (e.g, btcusdt@depth5@500ms)", - ) - arg_parser.add_argument( - "--prod", action="store_const", const="PROD", dest="mode", help="Run at prod" - ) - arg_parser.add_argument( - "--dev", action="store_const", const="DEV", dest="mode", help="Run at testnet" - ) - arg_parser.add_argument( - "--spot", - action="store_const", - const="BINANCE", - dest="venue", - help="Run at spots", - ) - arg_parser.add_argument("--op", help="The op to run") - arg_parser.add_argument( - "--param", help="Parameter(s) formatted as key=val)", nargs="+" - ) - arg_parser.add_argument("--create-order", action="store_true", help="Create order") - arg_parser.add_argument( - "--usds-future", - action="store_const", - const="BINANCE_USDS_FUTURES", - dest="venue", - help="Run at usds futures", - ) - arg_parser.add_argument("--user-data", action="store_true", help="User data") - arg_parser.add_argument( - "--exch-info", - action="store_const", - const="EXCHANGE_INFO", - dest="op", - help="Get exchangeInfo (REST)", - ) - arg_parser.add_argument( - "--list-symbols", action="store_true", help="Get symbols (REST)" - ) - arg_parser.add_argument( - "--listen-key", - action="store_const", - const="LISTEN_KEY", - dest="op", - help="Get listen key (needed for streams)", - ) - arg_parser.add_argument( - "--list-streams", action="store_true", help="List all streams" - ) - arg_parser.add_argument( - "--display-lines", - help="Cap the number of lines to display from incoming messages (applies circular buffer to console output).", - ) - arg_parser.add_argument("--url", help="The url to send requests to") - arg_parser.set_defaults(mode="DEV", venue="BINANCE", user_data=False, op=None) - return arg_parser - - -def _parse_bool(s: Optional[str]): - match s: - case "1": - return True - case "true": - return True - case "on": - return True - case "y": - return True - case "yes": - return True - case _: - return False - - -class BinanceClient: - def __init__(self, venue="BINANCE", mode="DEV", logger=None): - self.venue = venue - self.mode = mode - self.rest_url = required_env_var(f"{venue}_{mode}_URL") - self.ws_url = required_env_var(f"{venue}_{mode}_WS_URL") - self._user_data_ping_thread: Optional[Thread] = None - self.threads: list[Thread] = [] - self.logger = logger or logging.getLogger(__name__) - self.api_key = os.environ.get(f"{self.venue}_{self.mode}_API_KEY") - self.secret_key = os.environ.get(f"{self.venue}_{self.mode}_SECRET_KEY") - self.orders = {} - - def _on_open_with_ping( - self, on_open: Optional[Callable[[websocket.WebSocket], None]] - ): - def wrapper(ws): - def ping_loop(): - time.sleep(10) - # if isinstance(ws, websocket.WebSocketApp): - # wsa: websocket.WebSocketApp = ws - # ws = wsa.sock - while True: - ws.sock.ping() - time.sleep(25) - - self._user_data_ping_thread = Thread(target=ping_loop) - self._user_data_ping_thread.start() - if on_open: - on_open(ws) - - return wrapper - - def _get_base_url(self, op, is_ws: bool = False, vars: Optional[Mapping] = None): - """Gets the base url for the request (includes host and path parts, but not query params).""" - url_var_suffix = "WS_URL" if is_ws else "URL" - url = os.environ.get(f"{self.venue}_{self.mode}_{url_var_suffix}") or "" - req_path = os.environ.get(f"{self.venue}_{op}_PATH") or "" - if vars: - req_path = expand_vars(req_path, vars) - if req_path: - n = 1 if req_path.startswith("/") else 0 - n += 1 if self.rest_url.endswith("/") else 0 - match n: - case 0: - url += "/" + req_path - case 1: - url += req_path - case 2: - url += req_path[1:] - return url - - def _index_transaction(self, bts_key: int): - exch_key = uuid.uuid1().hex - self.orders[exch_key] = bts_key - return exch_key - - def _get_params_str(self, op, **kwargs): - params = os.environ.get(f"{self.venue}_{op}_PARAMS") or "" - params = expand_vars(params, kwargs) - params = expand_defaults(params) - return params - - def _get_params_dict(self, op, **kwargs): - params = self._get_params_str(op=op, **kwargs) - if not params: - return {} - return {p[0]: p[1] for p in map(lambda x: x.split("="), params.split("&"))} - - def _invoke(self, target: Callable[[], Any], run_in_background: bool): - if run_in_background: - t = Thread(target=target) - self.threads.append(t) - t.start() - else: - target() - - def request(self, op, **kwargs): - print( - "Preparing request. op=%s, params=[%s]" - % (op, ", ".join([f"{k}={v}" for k, v in kwargs.items()])), - flush=True, - ) - self.logger.info( - "Preparing request. op=%s, params=[%s]" - % (op, ", ".join([f"{k}={v}" for k, v in kwargs.items()])) - ) - is_signed = _parse_bool(os.environ.get(f"{self.venue}_{op}_IS_SIGNED")) - is_keyed = _parse_bool(os.environ.get(f"{self.venue}_{op}_IS_KEYED")) - method = os.environ.get(f"{self.venue}_{op}_METHOD") - params = self._get_params_dict(op, **kwargs) - base_url = self._get_base_url(op, is_ws=False) - res = None - if is_keyed or is_signed: - if not self.api_key: - self.logger.error("Unable to make signed request. Api key not defined") - return - headers = {"Content-Type": "application/json", "X-MBX-APIKEY": self.api_key} - if is_signed: - if not self.secret_key: - self.logger.error( - "Unable to make signed request. Secret key not defined" - ) - return - params["timestamp"] = str(int(time.time() * 1000)) - base_url = self._get_base_url(op) - sorted_keys = list(params) - sorted_keys.sort() - params_str = "&".join([f"{k}={params[k]}" for k in sorted_keys]) - signature = hmac_hashing(api_secret=self.secret_key, payload=params_str) - params_str += "&signature=" + signature - url = base_url + "?" + params_str - if not method: - method = "POST" - headers = {"X-MBX-APIKEY": self.api_key} - # print("ABOUT TO SEND.", flush=True) - self.logger.info("Sending %s %s" % (method, url)) - try: - print( - "Requesting %s %s (%s)" - % ( - method, - url, - ", ".join([f"{k}={v}" for k, v in headers.items()]), - ), - flush=True, - ) - res = requests.request(method, url, headers=headers) - # if res: - print("RESULT. %s", str(res), flush=True) - self.logger.log( - logging.INFO if res.ok else logging.ERROR, - "Received %i %s" % (res.status_code, str(res.text)), - ) - except Exception as ex: - print("OH NO %s" % str(ex), flush=True) - self.logger.error("Request threw %s" % str(ex)) - else: - params_str = "&".join([f"{k}={v}" for k, v in params.items()]) - url = base_url - if params_str: - url += "?" + params_str - if not method: - method = "GET" - headers = {"Content-Type": "application/json", "X-MBX-APIKEY": self.api_key} - self.logger.info( - "Sending request %s %s (headers=[%s])" - % (method, url, ", ".join([f"{k}={v}" for k, v in headers.items()])) - ) - try: - res = requests.request(method, url, headers=headers) - self.logger.log( - logging.INFO if res.ok else logging.ERROR, - "Received response. %s" % str(res), - ) - except Exception as e: - self.logger.error("Request threw: %s" % str(e)) - self.logger.log( - logging.INFO if res and res.ok else logging.ERROR, - "Request failed. op=%s, res=\n%s", - op, - str(res), - ) - return res - - def create_order(self, req: pyexe.PyCreateOrderRequest): - self.logger.info("Create order. %s" % str(req)) - self.request( - op="CREATE_ORDER", - symbol=req.exchange_security_id(), - side="BUY" if req.is_buy() else "SELL", - type=req.order_type(), - price=req.price(), - quantity=f"{req.quantity():.6f}", - new_client_order_id=self._index_transaction(req.key()), - ) - - def cancel_order(self, req: pyexe.PyCancelOrderRequest): - self.request( - op="CANCEL_ORDER", - symbol=req.exchange_security_id(), - side="BUY" if req.is_buy() else "SELL", - orig_client_order_id=str(req.exchange_order_id()), - new_client_order_id=self._index_transaction(req.key()), - ) - - def subscribe_user_data( - self, - on_message: Optional[Callable[[websocket.WebSocket, Any], None]] = None, - on_error: Optional[Callable[[websocket.WebSocket, Any], None]] = None, - on_close: Optional[Callable[[websocket.WebSocket, Any, Any], None]] = None, - on_open: Optional[Callable[[websocket.WebSocket], None]] = None, - run_in_background: bool = True, - ): - self.logger.info("Requesting listen key") - res = self.request("LISTEN_KEY") - if res is None: - print("'listen_key_res' is none (this is unexpected)") - return - self.logger.log( - logging.INFO if res and res.ok else logging.ERROR, - "Received listen key. %s" % str(res), - ) - if not res.ok: - self.logger.error("Failed to get listen key") - lkey = res.json().get("listenKey") - self.logger.info("Received listen key %s" % lkey) - url = self._get_base_url("USER_DATA", is_ws=True, vars={"LISTEN_KEY": lkey}) - ws_app = websocket.WebSocketApp( - url=url, - on_message=on_message, - on_error=on_error, - on_close=on_close, - on_open=self._on_open_with_ping(on_open), - ) - self.logger.info("Starting user data websocket %s" % url) - self._invoke( - lambda: ws_app.run_forever(ping_interval=600, ping_timeout=180), - run_in_background, - ) - - def subscribe_streams( - self, - streams, - on_message: Optional[Callable[[websocket.WebSocket, Any], None]] = None, - on_error: Optional[Callable[[websocket.WebSocket, Any], None]] = None, - on_close: Optional[Callable[[websocket.WebSocket, Any, Any], None]] = None, - run_in_background: bool = True, - **kwargs, - ): - def on_open(ws: websocket.WebSocket): - self.logger.info("Stream websocket connected") - try: - req = { - "method": "SUBSCRIBE", - "params": streams, - "id": 1, - } - req_str = json.dumps(req) - self.logger.info("Requesting stream. %s", req_str) - ws.send_text(req_str) - except Exception as ex: - self.logger.error("Stream request failed. %s", ex) - self.logger.info("Disconnecting stream websocket") - ws.close() - - url = self._get_base_url("STREAMS", is_ws=True, **kwargs) - - ws_app = websocket.WebSocketApp( - url=url, - on_open=self._on_open_with_ping(on_open), - on_message=on_message, - on_error=on_error, - on_close=on_close, - ) - self.logger.info("StreamSubscription @ %s", url) - self._invoke( - lambda: ws_app.run_forever(ping_interval=600, ping_timeout=180), - run_in_background, - ) - - -def _main(): - load_env() - arg_parser = _get_arg_parser() - args = arg_parser.parse_args() - if args.list_streams: - stream_descs = [ - "{symbol}@aggTrade", - "{symbol}@markPrice", - "{symbol}@markPrice@{interval} (intervals: 1s)", - "!markPrice@arr", - "!markPrice@arr@1s", - "{symbol}@kline_{interval} (intervals: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M)", - "{pair}_{contractType}@continuousKline_{interval} (intervals: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M)" - "{symbol}@miniTicker", - "!ticker@arr", - "{symbol}@ticker", - "!miniTicker@arr", - "{symbol}@bookTicker", - "!bookTicker", - "{symbol}@forceOrder", - "!forceOrder@arr", - "{symbol}@depth{levels} (levels: 5, 10, 20)", - "{symbol}@depth{levels}@{interval} (levels: 5, 10, 20)", - " (intervals: 100ms, 500ms)", - "{symbol}@depth", - "{symbol}@depth@{interval} (intervals: 100ms, 500ms)", - "{symbol}@compositeIndex", - "!contractInfo", - "!assetIndex@arr", - "{assetSymbol}@assetIndex", - ] - print("\n".join(stream_descs)) - return - logger = logging.getLogger() - cli = BinanceClient(args.venue, args.mode, logger=logger) - params = {kv[0]: kv[1] for kv in map(lambda x: x.split("="), args.param or [])} - if args.op: - res = cli.request(args.op, **params) - if res and res.ok: - print(res.text) - elif args.list_symbols: - # Get symbols - res = cli.request("EXCHANGE_INFO") - if not res or not res.ok: - return 1 - syms = {} - res_j = json.loads(res.text) - for sym in res_j["symbols"]: - syms[sym["symbol"]] = sym - sym_list = list(syms) - sym_list.sort() - for sym in sym_list: - print(sym) - elif args.user_data: - # Subscribe user data - writer = CircularBufferWriter(buffer_line_count=int(args.display_lines or -1)) - - def on_message(ws, msg): - logger.debug("on_message: %s", msg) - writer.print(msg) - - def on_error(ws, err): - logger.error("on_error: %s", err) - - def on_close(ws, code, msg): - logger.info("on_close (%d): %s", code, msg) - - cli.subscribe_user_data(on_message, on_error, on_close) - elif args.streams: - # Subscribe streams - writer = CircularBufferWriter(buffer_line_count=int(args.display_lines or -1)) - - def on_message(ws, msg): - logger.debug("Recv: %s", msg) - writer.print(msg) - - def on_error(ws, err): - logger.error("Error: %s", err) - - def on_close(ws, code, msg): - logger.info("Closed (%d): %s", code, msg) - - cli.subscribe_streams( - streams=args.streams if isinstance(args.streams, list) else [args.streams], - on_message=on_message, - on_error=on_error, - on_close=on_close, - ) - else: - sys.stderr.write( - "No data requested (run with --streams ... or --exch-info or --list-symbols)" - ) - return 1 - return 0 - - -if __name__ == "__main__": - code = _main() - exit(code) diff --git a/src/lib_python_execution/python_execution_gateways/binance_env_helper.py b/src/lib_python_execution/python_execution_gateways/binance_env_helper.py deleted file mode 100644 index 04aac3cc8..000000000 --- a/src/lib_python_execution/python_execution_gateways/binance_env_helper.py +++ /dev/null @@ -1,100 +0,0 @@ -from typing import Any, Mapping, Optional -import dotenv -import os - - -def required_env_var(key): - val = os.environ.get(key) - if val is None: - raise ValueError(f"{key} env var not defined") - return val - - -def expand_env(s: str, vars=os.environ): - if not s: - return "" - beg = 0 - while True: - beg = s.find("${", beg) - if beg == -1 or beg > len(s) - 4: - break - end = s.find("}", beg + 2) - if end == -1: - break - k = s[beg:end] - v = vars.get(k) or "" - s = s[:beg] + v + s[end + 1 :] - beg += len(v) + 1 - return s - - -def expand_var(req_params: str, k: str, v: Any): - ku = str(k).upper() - substitution = f"${{{ku}}}" - pos = req_params.find(substitution) - end_pos = -1 - if pos == -1: - substitution = f"${{{ku}:" - pos = req_params.find(substitution) - if pos == -1: - return req_params - end_pos = req_params.find("}", pos + len(substitution)) - if end_pos == -1: - return req_params - end_pos += 1 - else: - end_pos = pos + len(substitution) - if end_pos == -1: - raise ValueError(f"Invalid request params template '{req_params}'") - req_params = req_params[:pos] + str(v) + req_params[end_pos:] - return req_params - - -def expand_vars(req_params: str, key_vals: Mapping): - for k, v in key_vals.items(): - req_params = expand_var(req_params, k, v) - return req_params - - -def expand_defaults(req_params: str): - i = 0 - prefix = ":-" - while i < len(req_params): - start = req_params.find("${", i) - if start == -1: - break - stop = req_params.find("}", start) - if stop == -1: - raise ValueError(f"Invalid param str {req_params}") - prefix_pos = req_params.find(prefix, start, stop) - if prefix_pos < start: - req_params = req_params[:start] + req_params[stop + 1 :] - i = start - continue - default_val = req_params[prefix_pos + len(prefix) : stop] - if default_val == "OMIT": - i = req_params.rfind("&", 0, prefix_pos - 1) - if i == -1: - i = 0 - req_params = req_params[:i] + req_params[stop + 1 :] - else: - req_params = req_params[:start] + default_val + req_params[stop + 1 :] - i = start + len(default_val) + 1 - if req_params.startswith("&"): - req_params = req_params[1:] - return req_params - - -def load_env(env_path: Optional[str] = None): - if not env_path: - env_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "binance.env" - ) - dotenv.load_dotenv(env_path, interpolate=False) - - -if __name__ == "__main__": - req_params = required_env_var("BINANCE_CREATE_ORDER_PARAMS") - res = expand_var(req_params, "SYMBOL", "BTCUSDT") - res = expand_defaults(res) - print(res) diff --git a/src/lib_python_execution/python_execution_gateways/binance_execution_gateway.py b/src/lib_python_execution/python_execution_gateways/binance_execution_gateway.py deleted file mode 100644 index d06f7ac55..000000000 --- a/src/lib_python_execution/python_execution_gateways/binance_execution_gateway.py +++ /dev/null @@ -1,242 +0,0 @@ -import dotenv -import json -import logging -import math -import sys -import traceback -from typing import Any, Optional -import uuid -import websocket - -from binance_client import BinanceClient -import python_execution_module as pyexe - - -BINANCE_API_KEY = "2RJ9H02ztFGWxeY2oRpnFYZNU8mki4V1yMvKrDU08HCiEhO4VLe2ePSUFqEYTNxc" -BINANCE_USDS_FUTURES_API_KEY = ( - "b838c108cd2648e34a4f088a7fc635a9cbccf3c2faa6bcaabcb532ffacc087b0" -) -BINANCE_USDS_FUTURES_SECRET_KEY = ( - "a68f0ab27403c42fa0306b6341b2ddd51d99ac42916f849404155449457d0492" -) - - -def to_bts_exec_type(exec_type: str, ord_status: str): - match exec_type: - case "TRADE_PREVENTION": - return "REJECTED" - case "NONE": - return "UNDEFINED" - case "TRADE": - return "FILL" if ord_status == "FILLED" else "PARTIAL_FILL" - case _: - return exec_type - - -def to_bts_order_status(ord_status: str): - match ord_status: - case "EXPIRED_IN_MATCH": - return "EXPIRED" - case _: - return ord_status - - -class DummyExecutionGateway(pyexe.ExecutionGateway): - def __init__(self): - super(DummyExecutionGateway, self).__init__() - self.bridge = None - self.client = None - self.start_th = None - self.ws = None - self.orders = {} - self.logger = logging.getLogger(__name__) - stdout_handler = logging.StreamHandler(sys.stdout) - stdout_handler.formatter = logging.Formatter( - "%(asctime)s | %(levelname)7s | %(filename)s:%(lineno)s | %(message)s" - ) - self.logger.addHandler(stdout_handler) - self.logger.addHandler( - logging.FileHandler( - "/home/bts/sparkdata/data/logging.python_execution/dummy_log" - ) - ) - self.logger.setLevel(logging.INFO) - env_path = "/workspaces/spark/tools/binance/binance.env" - self.logger.info("DummyExecutionGateway ctr. Loading env") - dotenv.load_dotenv(env_path, interpolate=False) - - def _index_transaction(self, bts_key: int): - exch_key = uuid.uuid1().hex - self.orders[exch_key] = bts_key - return exch_key - - def _on_user_data_msg_core(self, ws: websocket.WebSocket, msg: Any): - self.logger.info("Recv usr data: %s", str(msg)) - if not msg: - self.logger.error("Recv empty usr data msg") - return - if not self.bridge: - self.logger.error("Cannot process usr data. Bridge not created") - return - upd = json.loads(str(msg)) - if "e" in upd: - if upd["e"] == "executionReport": - # Binance spots update - bts_ord_id = self.orders.get(upd["c"]) - is_old_ord = False - if not bts_ord_id: - bts_ord_id = self.orders.get(upd["C"]) - if not bts_ord_id: - self.logger.warning( - "Received update with unknown order id: %s", msg - ) - return - is_old_ord = True - er = pyexe.PyExecutionReport() - er.set_key(bts_ord_id) - er.set_venue("BINANCE") - er.set_exchange_security_id(upd["s"]) - er.set_security_type("SPOT") - er.set_exec_type(to_bts_exec_type(upd["x"], upd["X"])) - er.set_is_buy(upd["S"] == "BUY") - er.set_price(float(upd["p"])) - er.set_quantity(float(upd["q"])) - er.set_last_filled_price(upd["L"]) - er.set_last_filled_quantity(upd["l"]) - total_filled_qty = float(upd["z"]) - er.set_filled_quantity(total_filled_qty) - total_filled_notional = float(upd["Z"]) - avg_filled_price = ( - total_filled_notional / total_filled_qty - if total_filled_qty > 0 - else math.nan - ) - er.set_avg_filled_price(avg_filled_price) - er.set_order_status(to_bts_order_status(upd["X"])) - er.set_order_type(upd["o"]) - match upd["X"]: - case "CANCELED": - if not is_old_ord: - del self.orders[bts_ord_id] - case "FILLED": - del self.orders[bts_ord_id] - case "REJECTED": - del self.orders[bts_ord_id] - self.bridge.on_execution_report(er) - elif upd["e"] == "ORDER_TRADE_UPDATE": - data = upd["o"] - # Binance Usds Futures order update - bts_ord_id = self.orders.get(data["c"]) - is_old_ord = False - if not bts_ord_id: - bts_ord_id = self.orders.get(data["C"]) - if not bts_ord_id: - self.logger.warning( - "Received dataate with unknown order id: %s", msg - ) - return - is_old_ord = True - - def parse_float(s: Optional[str], default_val=math.nan): - if s is not None: - try: - return float(s) - except Exception: - pass - return default_val - - er = pyexe.PyExecutionReport() - er.set_key(bts_ord_id) - er.set_venue("BINANCE_USDS_FUTURES") - er.set_security_type("FUTURE") - er.set_time_in_force("GTC") - er.set_exchange_order_id(str(data.get("i") or 0)) - # er.set_exchange_order_id(data["i"]) - er.set_exchange_security_id(data["s"]) - er.set_exec_type(to_bts_exec_type(data["x"], data["X"])) - er.set_is_buy(data["S"] == "BUY") - er.set_price(parse_float(data["p"])) - er.set_quantity(parse_float(data["q"])) - er.set_last_filled_price(parse_float(data.get("L"))) - er.set_last_filled_quantity(parse_float(data["l"])) - er.set_filled_quantity(parse_float(data["z"])) - er.set_avg_filled_price(parse_float(data["ap"])) - er.set_order_status(to_bts_order_status(data["X"])) - er.set_order_type(data["o"]) - match data["X"]: - case "CANCELED": - if not is_old_ord: - del self.orders[bts_ord_id] - case "FILLED": - del self.orders[bts_ord_id] - case "REJECTED": - del self.orders[bts_ord_id] - print("Firing this exe report: %s" % str(er), flush=True) - self.bridge.on_execution_report(er) - - def _on_user_data_msg(self, ws: websocket.WebSocket, msg: Any): - try: - self._on_user_data_msg_core(ws, msg) - except Exception: - self.logger.error("on_user_data_msg. %s" % traceback.format_exc()) - - def _on_user_data_open(self, ws: websocket.WebSocket): - self.ws = ws - if self.bridge: - self.bridge.on_is_ready(True) - - def _on_user_data_err(self, ws: websocket.WebSocket, err: str): - self.logger.error("on_user_data_err: %s", err) - - def _on_user_data_close(self, ws: websocket.WebSocket, code: int, msg: str): - self.logger.warning("on_user_data_close: %s %s", str(code), msg) - if self.bridge: - self.bridge.on_is_ready(False) - - def start(self, bridge: pyexe.ExecutionGatewayBridge): - print("DummyExecutionGateway.start") - self.bridge = bridge - self.client = BinanceClient(venue="BINANCE_USDS_FUTURES") - print("DummyExecutionGateway.start. Subscribing to binance user data") - self.client.subscribe_user_data( - on_message=self._on_user_data_msg, - on_open=self._on_user_data_open, - on_close=self._on_user_data_close, - ) - print("DummyExecutionGateway.start. Subscribed to binance user data") - - def cancel_order(self, req: pyexe.PyCancelOrderRequest): - if not self.client: - self.logger.error("Cannot cancel order. BinanceClient not created.") - return - self.logger.info("[cancel_order.json]%s", str(req)) - try: - self.client.request( - "CANCEL_ORDER", - symbol=req.exchange_security_id(), - orig_client_order_id=req.exchange_order_id(), - new_client_order_id=self._index_transaction(req.key()), - ) - except Exception as ex: - self.logger.error("Create order request threw. %s" % str(ex)) - - def create_order(self, req: pyexe.PyCreateOrderRequest): - if not self.client: - self.logger.error("Cannot create order. BinanceClient not created.") - return - print("hey bud") - self.logger.info("[create_order.json]%s", str(req)) - side = "BUY" if req.is_buy() else "SELL" - try: - self.client.request( - op="CREATE_ORDER", - symbol=req.exchange_security_id(), - price=req.price(), - quantity=f"{req.quantity():.6f}", - side=side, - time_in_force="GTC", - type="LIMIT", - new_client_order_id=self._index_transaction(req.key()), - ) - except Exception as ex: - self.logger.error("Create order request threw. %s" % str(ex)) diff --git a/src/lib_python_execution/python_execution_gateways/py_execution_utils.py b/src/lib_python_execution/python_execution_gateways/py_execution_utils.py deleted file mode 100644 index e6f39af6c..000000000 --- a/src/lib_python_execution/python_execution_gateways/py_execution_utils.py +++ /dev/null @@ -1,158 +0,0 @@ -import json -import math -import string -import logging -import os -import sys -import threading -import time -from typing import Callable, Optional -import websocket - -import python_execution_module as pyexe - - -def as_bool(*args): - (True if arg else False for arg in args) - - -def to_execution_report( - req: pyexe.PyCreateOrderRequest - | pyexe.PyCancelReplaceOrderRequest - | pyexe.PyCancelOrderRequest, - sec_type: Optional[str] = None, - dst: Optional[pyexe.PyExecutionReport] = None, -): - if not dst: - dst = pyexe.PyExecutionReport() - dst.set_key(req.key()) - dst.set_exchange_security_id(req.exchange_security_id()) - dst.set_venue(req.venue()) - dst.set_is_buy(req.is_buy()) - if sec_type: - dst.set_security_type(sec_type) - if isinstance(req, pyexe.PyCreateOrderRequest): - dst.set_order_type(req.order_type()) - elif isinstance(req, pyexe.PyCancelReplaceOrderRequest): - pass - elif isinstance(req, pyexe.PyCancelOrderRequest): - pass - return dst - - -class StructuredLog: - def __init__(self, **kwargs): - self.kwargs = kwargs - - def __str__(self): - return json.dumps(self.kwargs) - - -class LogFormatter(logging.Formatter): - def __init__(self): - pass - - def formatTime(self, record, datefmt=None): - ct = self.converter(record.created) - if datefmt: - return time.strftime(datefmt, ct) - else: - t = time.strftime("%Y-%m-%d %H:%M:%S", ct) - return f"{t}.{int(record.msecs * 1000):06d}" # microseconds - - def format(self, record): - # Simulate a unique numeric ID - timestamp_ns = int(record.created * 1e9) - system = "OrderMan" - env = "dev" - thread_id = threading.get_ident() - filename_line = f"{record.filename}:{record.lineno}" - timestamp = self.formatTime(record) - - return ( - f"{timestamp}|{timestamp_ns}|{system:<7}|{record.levelname:<5}|" - f"{os.getpid():<3}|{record.threadName:<20}|{thread_id}|" - f"{filename_line:<30}|{env}|{record.getMessage()}" - ) - - -def configure_logger(logger: logging.Logger): - stdout_handler = logging.StreamHandler(sys.stdout) - file_handler = logging.FileHandler( - f"/home/bts/sparkdata/data/logging.sparkd/{logger.name}_log" - ) - formatter = LogFormatter() - stdout_handler.formatter = formatter - file_handler.formatter = formatter - logger.handlers.clear() - logger.addHandler(stdout_handler) - logger.addHandler(file_handler) - logger.setLevel(logging.INFO) - return logger - - -def parse_float(s: Optional[str], default_val=math.nan): - if s is not None: - try: - return float(s) - except Exception: - pass - return default_val - - -def load_gateway_config(name: str, expand_vars: bool = True): - s = "" - with open(f"/home/bts/sparkdata/conf/py_{name}.json") as f: - s = f.read() - if expand_vars: - s = string.Template(s).substitute(os.environ) - return json.loads(s) - - -def run_websocket( - url: str, - stop_event: threading.Event, - logger: logging.Logger, - on_message: Callable[[str], None], - send_ping: Optional[Callable[[websocket.WebSocket], None]] = None, - ping_interval_secs: Optional[int] = 25, -): - conn_num = 0 - if send_ping is None: - - def send_empty_ping(ws): - return ws.ping() - - send_ping = send_empty_ping - - while not stop_event.is_set(): - if conn_num > 0: - # Gradually increase sleep time between attempts, capping at 5min. - sleep_secs = min(conn_num * 5, 300) - logger.warning("Reconnecting to %s in %ds." % (url, sleep_secs)) - time.sleep(sleep_secs) - logger.info("Connecting to %s (attempt no %d)" % (url, conn_num)) - ws = websocket.create_connection(url) - ws.settimeout(5) - logger.info("Connected to %s. Listening for data" % url) - if ping_interval_secs: - - def ping_loop(orig_conn_num): - time.sleep(ping_interval_secs) - while not stop_event.is_set() and conn_num == orig_conn_num: - send_ping(ws) - time.sleep(ping_interval_secs) - - threading.Thread(target=ping_loop(conn_num), daemon=True).start() - try: - msg = ws.recv() - if isinstance(msg, bytes): - on_message(msg.decode("utf-8")) - elif isinstance(msg, str): - on_message(msg) - else: - logger.error("Websocket recv unexpected msg type %s" % str(type(msg))) - - except Exception: - logging.error("Websocket recv threw.", exc_info=True) - continue diff --git a/src/lib_python_execution/python_execution_gateways/pyrightconfig.json b/src/lib_python_execution/python_execution_gateways/pyrightconfig.json deleted file mode 100644 index eacf29f21..000000000 --- a/src/lib_python_execution/python_execution_gateways/pyrightconfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "include": [ - ".", - ], - "exclude": [ - "**/__pycache__" - ], - "reportMissingImports": true, - "reportMissingTypeStubs": false, - "stubPath": "../../../build/sparkd-debug-clang/src/lib_python_execution/stubs", - "extraPaths": [ - "../../lib_python_core/python_core_plugins", - "../../../build/sparkd-debug-clang/src/lib_python_core/stubs", - ], - "pythonVersion": "3.10", - "pythonPlatform": "Linux", - "typeCheckingMode": "basic", - "venvPath": "../../../", - "venv": ".venv" -} diff --git a/src/lib_python_execution/python_execution_gateways/simple_test.py b/src/lib_python_execution/python_execution_gateways/simple_test.py deleted file mode 100644 index 8ea52c7ff..000000000 --- a/src/lib_python_execution/python_execution_gateways/simple_test.py +++ /dev/null @@ -1,241 +0,0 @@ -import json -import logging -import math -import os -import sys - -import threading -from threading import Thread, Lock -from time import sleep -from typing import Any, Callable, Mapping -import uuid - -import requests - -import python_execution_module as pyexe -import py_execution_utils -import dotenv -from binance_client import BinanceClient -import websocket - - -BINANCE_API_KEY = "2RJ9H02ztFGWxeY2oRpnFYZNU8mki4V1yMvKrDU08HCiEhO4VLe2ePSUFqEYTNxc" -BINANCE_USDS_FUTURES_API_KEY = ( - "b838c108cd2648e34a4f088a7fc635a9cbccf3c2faa6bcaabcb532ffacc087b0" -) -BINANCE_USDS_FUTURES_SECRET_KEY = ( - "a68f0ab27403c42fa0306b6341b2ddd51d99ac42916f849404155449457d0492" -) - - -def to_bts_exec_type(exec_type: str, ord_status: str): - match exec_type: - case "TRADE_PREVENTION": - return "REJECTED" - case "NONE": - return "UNDEFINED" - case "TRADE": - return "FILL" if ord_status == "FILLED" else "PARTIAL_FILL" - case _: - return exec_type - - -def to_bts_order_status(ord_status: str): - match ord_status: - case "EXPIRED_IN_MATCH": - return "EXPIRED" - case _: - return ord_status - - -def invoke(target: Callable[[], Any]): - t = threading.Thread(target=target) - t.start() - return t - - -class SimpleTest(pyexe.ExecutionGateway): - def __init__(self): - super(SimpleTest, self).__init__() - self.bridge = None - self.client = None - self.create_order_reqs = [] - self.threads = [] - self.ws = None - self.lock = Lock() - self.orders = {} - self.logger = logging.getLogger(__name__) - stdout_handler = logging.StreamHandler(sys.stdout) - stdout_handler.formatter = logging.Formatter( - "%(asctime)s | %(levelname)7s | %(filename)s:%(lineno)s | %(message)s" - ) - self.logger.addHandler(stdout_handler) - self.logger.addHandler( - logging.FileHandler( - "/home/bts/sparkdata/data/logging.python_execution/simple_test" - ) - ) - self.logger.setLevel(logging.INFO) - env_path = "/workspaces/spark/tools/binance/binance.env" - self.logger.info("SimpleTest ctr. Loading env") - dotenv.load_dotenv(env_path, interpolate=False) - - def _invoke(self, target: Callable[[], Any], run_in_background: bool): - if run_in_background: - t = threading.Thread(target=target) - self.threads.append(t) - t.start() - else: - target() - - def _index_transaction(self, bts_key: int): - exch_key = uuid.uuid1().hex - self.orders[exch_key] = bts_key - return exch_key - - def _on_user_data_open(self, ws: websocket.WebSocket): - self.ws = ws - if self.bridge: - self.bridge.on_is_ready(True) - - def _on_user_data_close(self, code: int, msg: str): - self.logger.warning("on_user_data_close: %s %s", str(code), msg) - if self.bridge: - self.bridge.on_is_ready(False) - - def start(self, bridge: pyexe.ExecutionGatewayBridge): - print("DummyExecutionGateway.start") - self.bridge = bridge - self.client = BinanceClient(venue="BINANCE_USDS_FUTURES") - print("DummyExecutionGateway.start. Subscribing to binance user data") - print("DummyExecutionGateway.start. Subscribed to binance user data") - - def delayed_ready(): - self.logger.info("SimpleTest.delayed_ready") - sleep(0.5) - if self.bridge: - self.logger.info( - "SimpleTest.delayed_ready. Notifying bridge that we're ready" - ) - self.bridge.on_is_ready(True) - else: - self.logger.info( - "SimpleTest.delayed_ready. Unable to notify bridge that we're ready (bridge is None!)" - ) - - self._invoke(delayed_ready, True) - - def cancel_order(self, req: pyexe.PyCancelOrderRequest): - if not self.client: - self.logger.error("Cannot cancel order. BinanceClient not created.") - return - self.logger.info("[cancel_order.json]%s", str(req)) - try: - self.client.request( - "CANCEL_ORDER", - symbol=req.exchange_security_id(), - orig_client_order_id=req.exchange_order_id(), - new_client_order_id=self._index_transaction(req.key()), - ) - except Exception as ex: - self.logger.error("Create order request threw. %s" % str(ex)) - - def create_order(self, req: pyexe.PyCreateOrderRequest): - if not self.client: - self.logger.error("Cannot create order. BinanceClient not created.") - return - self.logger.info("create_order (async). %s" % str(req)) - - # local_req = pyexe.PyCreateOrderRequest() - # local_req.set_exchange_security_id(req.exchange_security_id()) - # local_req.set_key(req.key()) - def impl(): - sleep(0.7) - with self.lock: - self.logger.info("create_order. %s" % str(req)) - er = pyexe.PyExecutionReport() - er.set_key(7) - er.set_exchange_security_id("BNBUSDT") - er.set_exec_type("NEW") - er.set_is_buy(True) - er.set_price(650) - er.set_quantity(0.02) - er.set_last_filled_price(649.50) - er.set_last_filled_quantity(0.012) - er.set_filled_quantity(0.012) - er.set_avg_filled_price(649.50) - er.set_order_status("WORKING") - er.set_order_type("LIMIT") - - er.set_key(req.key()) - er.set_exchange_security_id(req.exchange_security_id()) - er.set_venue(req.venue()) - er.set_exec_type("NEW") - er.set_is_buy(req.is_buy()) - er.set_price(req.price()) - er.set_quantity(req.quantity()) - er.set_last_filled_price( - req.price() - 0.5 if req.is_buy() else req.price() + 0.5 - ) - er.set_last_filled_quantity(0.012) - er.set_filled_quantity(0.012) - er.set_avg_filled_price(er.last_filled_price()) - er.set_order_status("WORKING") - er.set_order_type(req.order_type()) - if self.bridge: - self.bridge.on_execution_report(er) - - self._invoke(impl, True) - self.logger.info("[create_order.json]%s", str(req)) - side = "BUY" if req.is_buy() else "SELL" - try: - res = self.client.request( - op="CREATE_ORDER", - symbol=req.exchange_security_id(), - price=req.price(), - quantity=f"{req.quantity():.6f}", - side=side, - time_in_force="GTC", - type="LIMIT", - new_client_order_id=self._index_transaction(req.key()), - ) - print(str(res), flush=True) - except Exception as ex: - self.logger.error("Create order request threw. %s" % str(ex)) - - -# def create_order(client: BinanceClient, price, quantity, side, logger): -# logger.info("[create_order.json]%s %d @ %d", side, quantity, price) -# try: -# res = client.request( -# op="CREATE_ORDER", -# symbol="BNBUSDT", -# price=price, -# quantity=f"{quantity:.5f}", -# side=side, -# time_in_force="GTC", -# type="LIMIT", -# # new_client_order_id=self._index_transaction(req.key()), -# ) -# if res: -# print("%d %s" % (res.status_code, res.text), flush=True) -# except Exception as ex: -# logger.error("Create order request threw. %s" % str(ex)) - - -# if __name__ == "__main__": -# print("DummyExecutionGateway.start") -# logger = logging.getLogger(__name__) -# logger.setLevel(logging.INFO) -# env_path = "/workspaces/spark/tools/binance/binance.env" -# dotenv.load_dotenv(env_path, interpolate=False) -# client = BinanceClient(venue="BINANCE_USDS_FUTURES") -# # req = pyexe.PyCreateOrderRequest() -# # req.set_exchange_security_id("BNBUSDT") -# # req.set_price(650) -# # req.set_quantity(0.015) -# # req.set_is_buy(True) -# # req.set_venue("BINANCE_USDS_FUTURES") -# # req.set_order_type("LIMIT") -# # req.set_key(5) -# create_order(client, 650, 0.02, "BUY", logger) diff --git a/src/lib_python_core/CMakeLists.txt b/src/lib_python_interop/CMakeLists.txt similarity index 90% rename from src/lib_python_core/CMakeLists.txt rename to src/lib_python_interop/CMakeLists.txt index ffc181509..b8d271266 100644 --- a/src/lib_python_core/CMakeLists.txt +++ b/src/lib_python_interop/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.23) -project(python_core) +project(python_interop) include(${CMAKE_SOURCE_DIR}/cmake/InitLibrary.cmake) include(${CMAKE_SOURCE_DIR}/cmake/InitPythonModule.cmake) include(${CMAKE_SOURCE_DIR}/cmake/InitPythonPlugin.cmake) @@ -16,4 +16,4 @@ target_link_libraries(${PROJECT_NAME} PUBLIC spark pybind11::embed # CPython module (for .py files to import) init_python_module() # Py plugin (for sparkd to load at runtime) -init_python_plugin(PLUGIN_NAME python_core_plugins) +init_python_plugin(PLUGIN_NAME python_interop_plugins) diff --git a/src/lib_python_core/Config.cmake.in b/src/lib_python_interop/Config.cmake.in similarity index 100% rename from src/lib_python_core/Config.cmake.in rename to src/lib_python_interop/Config.cmake.in diff --git a/src/lib_python_execution/README.md b/src/lib_python_interop/README.md similarity index 61% rename from src/lib_python_execution/README.md rename to src/lib_python_interop/README.md index 10b72436e..1ae576b44 100644 --- a/src/lib_python_execution/README.md +++ b/src/lib_python_interop/README.md @@ -1,44 +1,92 @@ [pybind11]: https://github.com/pybind/pybind11 +[pybind11-stubgen]: https://github.com/sizmailov/pybind11-stubgen [lib_spark]: ../lib_spark/README.md -# lib_python_execution +# lib_python_interop -This library binds c++ execution gateways to python modules which can be loaded at runtime. +This library binds c++ types to python market data and execution plugins which can be loaded at runtime. ## **Dependencies** * **[Pybind11][pybind11]** +* **[Pybind11][pybind11-stubgen]** * **[lib_spark][lib_spark]** ## Notes -* Because this is *python*, order/security enums are represented as strings +* Because this is *python*, market data enums (SecurityType, Venue, Side, etc.) are represented as strings * Parsing is **NOT** case sensitive (though it is conventional to use `UPPER_CASE` strings) * Type name prefixes (SECURITY_TYPE_, VENUE_, SIDE_ etc.) are **NOT** required * ***Having said that,*** we recommend UPPER_CASE strings without the type prefix. Here are a few examples: | Enum | Examples | |------|---------| - | **OrderType** | "LIMIT", "MARKET" | - | **OrderStatus** | "NEW", "PARTIALLY_FILLED", "FILLED", "CANCELED" | | **SecurityType** | "SPOT", "FUTURE", "OPTION" | | **Venue** | "BYBIT", "GATEIO", "DERIBIT", "COINBASE" | | **Side** | "BUY", "SELL" | + | **OrderType** | "LIMIT", "MARKET" | + | **OrderStatus** | "NEW", "PARTIALLY_FILLED", "FILLED", "CANCELED" | + | **SecurityType** | "SPOT", "FUTURE", "OPTION" | -* All .py files in the `python_execution_gateways` dir are copied to a special plugins directory +* All .py files in the `python_interop_plugins` dir are copied to a special plugins directory at build time, so they may be loaded at runtime. -* For intellisense, symlink the *.so* and *.pyi* files in the `python_execution_gateways` dir - > *These files are excluded from source control via .gitignore, so don't worry about accidentally checking them in* -```bash -# Run in src/lib_python_execution/python_execution_gateways -ln -s ../../../build/sparkd-debug-clang/src/lib_python_execution/stubs/python_execution_module.pyi . -ln -s ../../../build/sparkd-debug-clang/src/lib_python_execution/python_execution_module.cpython-310-x86_64-linux-gnu.so . +* Type hints for the c++ defined module, `python_interop_module`, should work as long sparkd-debug-clang is built. + > *We use [pybind11-stubgen] to generate stubs as part of the build* + +## Examples +### Md Provider Plugin +```python +import time +import python_md_module as pymd + + +class MdProviderExample(pymd.MdProvider): + def __init__(self): + super(MdProviderExample, self).__init__() + pass + + def start(self, sink: pymd.MdSink): + print("Starting MdProviderExample") + # An actual implementation would subscribe to market data here. + # This is just an example to show you how to pass market data + # back to the c++ Spark api. + + def add_dummy_security(): + # This example populates a dummy security for demonstrative purposes. + # Of course, an actual implementation would populate the security list + # with data received from the exchange (or some other provider). + sec = pymd.PySecurity() + sec.set_venue("GLBX") + sec.set_name("ESH8") + sec.set_asset("ES") + fut = pymd.PyFuture() + fut.set_expiration("2025-03-28") + sec.set_future(fut) + # Note: the example only adds 1 sec, but batch adds are supported. + sec_list = pymd.PySecurityList() + sec_list.append(sec) + sink.on_securities(sec_list) + + def add_dummy_trade(): + trade = pymd.PyTrade() + trade.set_venue("GLBX") + trade.set_security_type("FUTURE") + trade.set_exchange_security_id("ESH8") + trade.set_price(5684.25) + trade.set_quantity(1) + trade.set_aggressor_side("BUY") + trade.set_timestamp(time.time_ns()) + + add_dummy_security() + add_dummy_trade() + + def stop(self): + print("Stopping MdProviderExample") ``` - -## Example +### Execution Gateway Plugin ```python import json import logging diff --git a/src/lib_python_execution/pyrightconfig.json b/src/lib_python_interop/pyrightconfig.json similarity index 59% rename from src/lib_python_execution/pyrightconfig.json rename to src/lib_python_interop/pyrightconfig.json index 1cc374d36..199fc80fc 100644 --- a/src/lib_python_execution/pyrightconfig.json +++ b/src/lib_python_interop/pyrightconfig.json @@ -7,10 +7,10 @@ ], "reportMissingImports": true, "reportMissingTypeStubs": false, - "stubPath": "../../build/sparkd-debug-clang/src/lib_python_execution/stubs", + "stubPath": "../../build/sparkd-debug-clang/src/lib_python_interop/stubs", "extraPaths": [ - "../lib_python_core/python_core_plugins", - "../../build/sparkd-debug-clang/src/lib_python_core/stubs", + "../lib_python_interop/python_interop_plugins", + "../../build/sparkd-debug-clang/src/lib_python_interop/stubs", ], "pythonVersion": "3.10", "pythonPlatform": "Linux", diff --git a/src/lib_python_execution/python_execution/execution_gateway.cpp b/src/lib_python_interop/python_interop/execution/execution_gateway.cpp similarity index 100% rename from src/lib_python_execution/python_execution/execution_gateway.cpp rename to src/lib_python_interop/python_interop/execution/execution_gateway.cpp diff --git a/src/lib_python_execution/python_execution/execution_gateway.h b/src/lib_python_interop/python_interop/execution/execution_gateway.h similarity index 67% rename from src/lib_python_execution/python_execution/execution_gateway.h rename to src/lib_python_interop/python_interop/execution/execution_gateway.h index e8b5a0013..fa54bb992 100644 --- a/src/lib_python_execution/python_execution/execution_gateway.h +++ b/src/lib_python_interop/python_interop/execution/execution_gateway.h @@ -1,10 +1,10 @@ #pragma once -#include "python_core/py_security.h" -#include "python_execution/execution_gateway_bridge.h" -#include "python_execution/py_cancel_order_request.h" -#include "python_execution/py_cancel_replace_order_request.h" -#include "python_execution/py_create_order_request.h" +#include "python_interop/execution/execution_gateway_bridge.h" +#include "python_interop/execution/py_cancel_order_request.h" +#include "python_interop/execution/py_cancel_replace_order_request.h" +#include "python_interop/execution/py_create_order_request.h" +#include "python_interop/py_security.h" namespace python_execution { diff --git a/src/lib_python_execution/python_execution/execution_gateway_bridge.cpp b/src/lib_python_interop/python_interop/execution/execution_gateway_bridge.cpp similarity index 100% rename from src/lib_python_execution/python_execution/execution_gateway_bridge.cpp rename to src/lib_python_interop/python_interop/execution/execution_gateway_bridge.cpp diff --git a/src/lib_python_execution/python_execution/execution_gateway_bridge.h b/src/lib_python_interop/python_interop/execution/execution_gateway_bridge.h similarity index 76% rename from src/lib_python_execution/python_execution/execution_gateway_bridge.h rename to src/lib_python_interop/python_interop/execution/execution_gateway_bridge.h index 8b334c21b..68a7107f6 100644 --- a/src/lib_python_execution/python_execution/execution_gateway_bridge.h +++ b/src/lib_python_interop/python_interop/execution/execution_gateway_bridge.h @@ -1,7 +1,7 @@ #pragma once -#include "python_execution/py_execution_report.h" -#include "python_execution/py_position_info.h" +#include "python_interop/execution/py_execution_report.h" +#include "python_interop/execution/py_position_info.h" namespace python_execution { diff --git a/src/lib_python_execution/python_execution/execution_gateway_bridge_impl.cpp b/src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.cpp similarity index 91% rename from src/lib_python_execution/python_execution/execution_gateway_bridge_impl.cpp rename to src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.cpp index ca6d676c8..42fea7354 100644 --- a/src/lib_python_execution/python_execution/execution_gateway_bridge_impl.cpp +++ b/src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.cpp @@ -1,10 +1,10 @@ -#include "python_execution/execution_gateway_bridge_impl.h" -#include "python_core/python_activator.h" -#include "python_core/python_interpreter.h" -#include "python_execution/execution_gateway.h" -#include "python_execution/py_cancel_order_request.h" -#include "python_execution/py_create_order_request.h" -#include "python_execution/py_position_info.h" +#include "python_interop/execution/execution_gateway_bridge_impl.h" +#include "python_interop/execution/execution_gateway.h" +#include "python_interop/execution/py_cancel_order_request.h" +#include "python_interop/execution/py_create_order_request.h" +#include "python_interop/execution/py_position_info.h" +#include "python_interop/python_activator.h" +#include "python_interop/python_interpreter.h" #include "spark/core/TimeSpan.h" #include "spark/io/json_formatter.h" #include "spark/logging/logging.h" @@ -27,7 +27,7 @@ using bts::core::Delegate; using bts::core::TimeSpan; using bts::io::json_formatter; using bts::orderman::RFQMsg; -using python_core::PythonActivatorT; +using python_interop::PythonActivatorT; using std::string; using std::vector; @@ -140,7 +140,7 @@ int ExecutionGatewayBridgeImpl::run_py(volatile bool *keep_going) { "run_py", json_formatter(fmt::format( "Sending {} securities to py gw", securities.size()))); - python_core::PythonInterpreter::exec( + python_interop::PythonInterpreter::exec( [&]() { gw->on_securities(securities); }); is_securities_dirty = false; } @@ -155,7 +155,7 @@ int ExecutionGatewayBridgeImpl::run_py(volatile bool *keep_going) { static const std::string req_type_names[]{"CREATE", "CXL_REP", "CXL"}; ReqType req_type; sw.restart(); - python_core::PythonInterpreter::exec([&]() { + python_interop::PythonInterpreter::exec([&]() { std::visit( bts::overloaded{ [&](std::shared_ptr req) { @@ -183,7 +183,7 @@ int ExecutionGatewayBridgeImpl::run_py(volatile bool *keep_going) { R"({{ "typ": "{}", "hndl_dur": "{:.3f}ms" }})", req_type_names[req_type], sw.total_elapsed().total_millis())); } - python_core::PythonInterpreter::exec([&]() { gw->stop(); }); + python_interop::PythonInterpreter::exec([&]() { gw->stop(); }); return 0; } diff --git a/src/lib_python_execution/python_execution/execution_gateway_bridge_impl.h b/src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.h similarity index 84% rename from src/lib_python_execution/python_execution/execution_gateway_bridge_impl.h rename to src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.h index 795c4e6b9..6b334c966 100644 --- a/src/lib_python_execution/python_execution/execution_gateway_bridge_impl.h +++ b/src/lib_python_interop/python_interop/execution/execution_gateway_bridge_impl.h @@ -1,11 +1,11 @@ #pragma once -#include "python_core/py_security.h" -#include "python_execution/execution_gateway.h" -#include "python_execution/execution_gateway_bridge.h" -#include "python_execution/py_execution_report.h" -#include "python_execution/py_order_request.h" -#include "python_execution/py_position_info.h" +#include "python_interop/execution/execution_gateway.h" +#include "python_interop/execution/execution_gateway_bridge.h" +#include "python_interop/execution/py_execution_report.h" +#include "python_interop/execution/py_order_request.h" +#include "python_interop/execution/py_position_info.h" +#include "python_interop/py_security.h" #include "spark/collections/concurrent_queue.h" #include "spark/utils/Delegate.h" #include "spark/utils/concurrent_value_store.h" diff --git a/src/lib_python_execution/python_execution/py_cancel_order_request.cpp b/src/lib_python_interop/python_interop/execution/py_cancel_order_request.cpp similarity index 100% rename from src/lib_python_execution/python_execution/py_cancel_order_request.cpp rename to src/lib_python_interop/python_interop/execution/py_cancel_order_request.cpp diff --git a/src/lib_python_execution/python_execution/py_cancel_order_request.h b/src/lib_python_interop/python_interop/execution/py_cancel_order_request.h similarity index 100% rename from src/lib_python_execution/python_execution/py_cancel_order_request.h rename to src/lib_python_interop/python_interop/execution/py_cancel_order_request.h diff --git a/src/lib_python_execution/python_execution/py_cancel_replace_order_request.cpp b/src/lib_python_interop/python_interop/execution/py_cancel_replace_order_request.cpp similarity index 100% rename from src/lib_python_execution/python_execution/py_cancel_replace_order_request.cpp rename to src/lib_python_interop/python_interop/execution/py_cancel_replace_order_request.cpp diff --git a/src/lib_python_execution/python_execution/py_cancel_replace_order_request.h b/src/lib_python_interop/python_interop/execution/py_cancel_replace_order_request.h similarity index 100% rename from src/lib_python_execution/python_execution/py_cancel_replace_order_request.h rename to src/lib_python_interop/python_interop/execution/py_cancel_replace_order_request.h diff --git a/src/lib_python_execution/python_execution/py_create_order_request.cpp b/src/lib_python_interop/python_interop/execution/py_create_order_request.cpp similarity index 100% rename from src/lib_python_execution/python_execution/py_create_order_request.cpp rename to src/lib_python_interop/python_interop/execution/py_create_order_request.cpp diff --git a/src/lib_python_execution/python_execution/py_create_order_request.h b/src/lib_python_interop/python_interop/execution/py_create_order_request.h similarity index 100% rename from src/lib_python_execution/python_execution/py_create_order_request.h rename to src/lib_python_interop/python_interop/execution/py_create_order_request.h diff --git a/src/lib_python_execution/python_execution/py_execution_gateway.cpp b/src/lib_python_interop/python_interop/execution/py_execution_gateway.cpp similarity index 100% rename from src/lib_python_execution/python_execution/py_execution_gateway.cpp rename to src/lib_python_interop/python_interop/execution/py_execution_gateway.cpp diff --git a/src/lib_python_execution/python_execution/py_execution_gateway.h b/src/lib_python_interop/python_interop/execution/py_execution_gateway.h similarity index 90% rename from src/lib_python_execution/python_execution/py_execution_gateway.h rename to src/lib_python_interop/python_interop/execution/py_execution_gateway.h index 24a4e67bb..13d354f2f 100644 --- a/src/lib_python_execution/python_execution/py_execution_gateway.h +++ b/src/lib_python_interop/python_interop/execution/py_execution_gateway.h @@ -1,8 +1,8 @@ #pragma once -#include "python_execution/execution_gateway.h" -#include "python_execution/execution_gateway_bridge.h" -#include "python_execution/py_create_order_request.h" +#include "python_interop/execution/execution_gateway.h" +#include "python_interop/execution/execution_gateway_bridge.h" +#include "python_interop/execution/py_create_order_request.h" #include diff --git a/src/lib_python_execution/python_execution/py_execution_report.cpp b/src/lib_python_interop/python_interop/execution/py_execution_report.cpp similarity index 98% rename from src/lib_python_execution/python_execution/py_execution_report.cpp rename to src/lib_python_interop/python_interop/execution/py_execution_report.cpp index f53c98a66..45006220c 100644 --- a/src/lib_python_execution/python_execution/py_execution_report.cpp +++ b/src/lib_python_interop/python_interop/execution/py_execution_report.cpp @@ -43,7 +43,7 @@ void PyExecutionReport::set_data( double last_filled_quantity, double last_filled_price, double avg_filled_price, - std::int64_t transact_id, + std::string transact_id, std::int64_t transact_time) { key_ = key; account_ = std::move(account); @@ -118,6 +118,7 @@ void to_json(nlohmann::json &j, const PyExecutionReport &req) { j["afpx"] = req.avg_filled_price(); j["typ"] = req.order_type(); j["tt"] = req.transact_time(); + j["tid"] = req.transact_id(); } void from_json(const nlohmann::json &j, PyExecutionReport &req) { diff --git a/src/lib_python_execution/python_execution/py_execution_report.h b/src/lib_python_interop/python_interop/execution/py_execution_report.h similarity index 81% rename from src/lib_python_execution/python_execution/py_execution_report.h rename to src/lib_python_interop/python_interop/execution/py_execution_report.h index 50cac122c..f658d8d57 100644 --- a/src/lib_python_execution/python_execution/py_execution_report.h +++ b/src/lib_python_interop/python_interop/execution/py_execution_report.h @@ -50,23 +50,23 @@ class PyExecutionReport { bool is_buy() const; void set_is_buy(bool value); - float price() const; - void set_price(float value); + double price() const; + void set_price(double value); - float quantity() const; - void set_quantity(float value); + double quantity() const; + void set_quantity(double value); - float filled_quantity() const; - void set_filled_quantity(float value); + double filled_quantity() const; + void set_filled_quantity(double value); - float last_filled_quantity() const; - void set_last_filled_quantity(float value); + double last_filled_quantity() const; + void set_last_filled_quantity(double value); - float last_filled_price() const; - void set_last_filled_price(float value); + double last_filled_price() const; + void set_last_filled_price(double value); - float avg_filled_price() const; - void set_avg_filled_price(float value); + double avg_filled_price() const; + void set_avg_filled_price(double value); const std::string &transact_id() const; void set_transact_id(std::string value); @@ -96,7 +96,7 @@ class PyExecutionReport { double last_filled_quantity, double last_filled_price, double avg_filled_price, - std::int64_t transact_id, + std::string transact_id, std::int64_t transact_time); void to_execution_report(bts::orderman::ExecutionReport &er); @@ -119,12 +119,12 @@ class PyExecutionReport { std::string time_in_force_; std::string exec_type_; bool is_buy_; - float price_; - float quantity_; - float filled_quantity_; - float last_filled_quantity_; - float last_filled_price_; - float avg_filled_price_; + double price_; + double quantity_; + double filled_quantity_; + double last_filled_quantity_; + double last_filled_price_; + double avg_filled_price_; std::string transact_id_; std::int64_t transact_time_; @@ -215,37 +215,37 @@ inline void PyExecutionReport::set_exec_type(std::string value) { inline bool PyExecutionReport::is_buy() const { return is_buy_; } inline void PyExecutionReport::set_is_buy(bool value) { is_buy_ = value; } -inline float PyExecutionReport::price() const { return price_; } -inline void PyExecutionReport::set_price(float value) { price_ = value; } +inline double PyExecutionReport::price() const { return price_; } +inline void PyExecutionReport::set_price(double value) { price_ = value; } -inline float PyExecutionReport::quantity() const { return quantity_; } -inline void PyExecutionReport::set_quantity(float value) { quantity_ = value; } +inline double PyExecutionReport::quantity() const { return quantity_; } +inline void PyExecutionReport::set_quantity(double value) { quantity_ = value; } -inline float PyExecutionReport::filled_quantity() const { +inline double PyExecutionReport::filled_quantity() const { return filled_quantity_; } -inline void PyExecutionReport::set_filled_quantity(float value) { +inline void PyExecutionReport::set_filled_quantity(double value) { filled_quantity_ = value; } -inline float PyExecutionReport::last_filled_quantity() const { +inline double PyExecutionReport::last_filled_quantity() const { return last_filled_quantity_; } -inline void PyExecutionReport::set_last_filled_quantity(float value) { +inline void PyExecutionReport::set_last_filled_quantity(double value) { last_filled_quantity_ = value; } -inline float PyExecutionReport::last_filled_price() const { +inline double PyExecutionReport::last_filled_price() const { return last_filled_price_; } -inline void PyExecutionReport::set_last_filled_price(float value) { +inline void PyExecutionReport::set_last_filled_price(double value) { last_filled_price_ = value; } -inline float PyExecutionReport::avg_filled_price() const { +inline double PyExecutionReport::avg_filled_price() const { return avg_filled_price_; } -inline void PyExecutionReport::set_avg_filled_price(float value) { +inline void PyExecutionReport::set_avg_filled_price(double value) { avg_filled_price_ = value; } diff --git a/src/lib_python_execution/python_execution/py_module.cpp b/src/lib_python_interop/python_interop/execution/py_module.cpp similarity index 87% rename from src/lib_python_execution/python_execution/py_module.cpp rename to src/lib_python_interop/python_interop/execution/py_module.cpp index 173f11bf8..8cef1cbf9 100644 --- a/src/lib_python_execution/python_execution/py_module.cpp +++ b/src/lib_python_interop/python_interop/execution/py_module.cpp @@ -1,17 +1,31 @@ #include "py_cancel_order_request.h" #include "py_cancel_replace_order_request.h" #include "py_create_order_request.h" -#include "python_execution/execution_gateway.h" -#include "python_execution/execution_gateway_bridge.h" -#include "python_execution/py_execution_gateway.h" -#include "python_execution/py_execution_report.h" -#include "python_execution/py_position_info.h" +#include "python_interop/execution/execution_gateway.h" +#include "python_interop/execution/execution_gateway_bridge.h" +#include "python_interop/execution/py_execution_gateway.h" +#include "python_interop/execution/py_execution_report.h" +#include "python_interop/execution/py_position_info.h" +#include "python_interop/md/md_bridge.h" +#include "python_interop/md/md_provider.h" +#include "python_interop/md/py_book.h" +#include "python_interop/md/py_book_level.h" +#include "python_interop/md/py_book_side.h" +#include "python_interop/md/py_md_provider.h" +#include "python_interop/md/py_trade.h" #include #include #include #include +using bts::md::py_interop::MdBridge; +using bts::md::py_interop::MdProvider; +using bts::md::py_interop::PyBook; +using bts::md::py_interop::PyBookLevel; +using bts::md::py_interop::PyBookSide; +using bts::md::py_interop::PyMdProvider; +using bts::md::py_interop::PyTrade; using python_execution::ExecutionGateway; using python_execution::ExecutionGatewayBridge; using python_execution::PyCancelOrderRequest; @@ -22,6 +36,8 @@ using python_execution::PyExecutionReport; using python_execution::PyPositionInfo; using std::shared_ptr; +using std::shared_ptr; + PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr) namespace py = pybind11; diff --git a/src/lib_python_execution/python_execution/py_order_request.h b/src/lib_python_interop/python_interop/execution/py_order_request.h similarity index 58% rename from src/lib_python_execution/python_execution/py_order_request.h rename to src/lib_python_interop/python_interop/execution/py_order_request.h index 4fdc46dce..7a91179bc 100644 --- a/src/lib_python_execution/python_execution/py_order_request.h +++ b/src/lib_python_interop/python_interop/execution/py_order_request.h @@ -1,8 +1,8 @@ #pragma once -#include "python_execution/py_cancel_order_request.h" -#include "python_execution/py_cancel_replace_order_request.h" -#include "python_execution/py_create_order_request.h" +#include "python_interop/execution/py_cancel_order_request.h" +#include "python_interop/execution/py_cancel_replace_order_request.h" +#include "python_interop/execution/py_create_order_request.h" #include diff --git a/src/lib_python_execution/python_execution/py_position_info.cpp b/src/lib_python_interop/python_interop/execution/py_position_info.cpp similarity index 100% rename from src/lib_python_execution/python_execution/py_position_info.cpp rename to src/lib_python_interop/python_interop/execution/py_position_info.cpp diff --git a/src/lib_python_execution/python_execution/py_position_info.h b/src/lib_python_interop/python_interop/execution/py_position_info.h similarity index 100% rename from src/lib_python_execution/python_execution/py_position_info.h rename to src/lib_python_interop/python_interop/execution/py_position_info.h diff --git a/src/lib_python_md/python_md/md_bridge.cpp b/src/lib_python_interop/python_interop/md/md_bridge.cpp similarity index 100% rename from src/lib_python_md/python_md/md_bridge.cpp rename to src/lib_python_interop/python_interop/md/md_bridge.cpp diff --git a/src/lib_python_md/python_md/md_bridge.h b/src/lib_python_interop/python_interop/md/md_bridge.h similarity index 79% rename from src/lib_python_md/python_md/md_bridge.h rename to src/lib_python_interop/python_interop/md/md_bridge.h index 48478b3ad..231a1bdc2 100644 --- a/src/lib_python_md/python_md/md_bridge.h +++ b/src/lib_python_interop/python_interop/md/md_bridge.h @@ -1,8 +1,8 @@ #pragma once -#include "python_core/py_security.h" -#include "python_md/py_book.h" -#include "python_md/py_trade.h" +#include "python_interop/md/py_book.h" +#include "python_interop/md/py_trade.h" +#include "python_interop/py_security.h" #include diff --git a/src/lib_python_md/python_md/md_bridge_impl.cpp b/src/lib_python_interop/python_interop/md/md_bridge_impl.cpp similarity index 96% rename from src/lib_python_md/python_md/md_bridge_impl.cpp rename to src/lib_python_interop/python_interop/md/md_bridge_impl.cpp index ab9bff940..c157dcec8 100644 --- a/src/lib_python_md/python_md/md_bridge_impl.cpp +++ b/src/lib_python_interop/python_interop/md/md_bridge_impl.cpp @@ -1,6 +1,6 @@ -#include "python_md/md_bridge_impl.h" -#include "python_core/python_activator.h" -#include "python_md/md_provider.h" +#include "python_interop/md/md_bridge_impl.h" +#include "python_interop/md/md_provider.h" +#include "python_interop/python_activator.h" #include "spark/io/json_formatter.h" #include "spark/logging/logging.h" #include "spark/logging/trace_log.h" @@ -19,7 +19,7 @@ namespace bts::md::py_interop { using bts::core::Delegate; using bts::io::json_formatter; using bts::utils::Stopwatch; -using python_core::PythonActivatorT; +using python_interop::PythonActivatorT; using std::string; using std::vector; diff --git a/src/lib_python_md/python_md/md_bridge_impl.h b/src/lib_python_interop/python_interop/md/md_bridge_impl.h similarity index 93% rename from src/lib_python_md/python_md/md_bridge_impl.h rename to src/lib_python_interop/python_interop/md/md_bridge_impl.h index b3472bea6..7d5a5027f 100644 --- a/src/lib_python_md/python_md/md_bridge_impl.h +++ b/src/lib_python_interop/python_interop/md/md_bridge_impl.h @@ -1,7 +1,7 @@ #pragma once -#include "python_md/md_bridge.h" -#include "python_md/md_provider.h" +#include "python_interop/md/md_bridge.h" +#include "python_interop/md/md_provider.h" #include "spark/collections/concurrent_queue.h" namespace bts::md::py_interop { diff --git a/src/lib_python_md/python_md/md_provider.cpp b/src/lib_python_interop/python_interop/md/md_provider.cpp similarity index 71% rename from src/lib_python_md/python_md/md_provider.cpp rename to src/lib_python_interop/python_interop/md/md_provider.cpp index 4e9de1a48..4bff6471e 100644 --- a/src/lib_python_md/python_md/md_provider.cpp +++ b/src/lib_python_interop/python_interop/md/md_provider.cpp @@ -1,4 +1,4 @@ -#include "python_md/md_provider.h" +#include "python_interop/md/md_provider.h" namespace bts::md::py_interop { diff --git a/src/lib_python_md/python_md/md_provider.h b/src/lib_python_interop/python_interop/md/md_provider.h similarity index 83% rename from src/lib_python_md/python_md/md_provider.h rename to src/lib_python_interop/python_interop/md/md_provider.h index f23848156..7bb3d20d1 100644 --- a/src/lib_python_md/python_md/md_provider.h +++ b/src/lib_python_interop/python_interop/md/md_provider.h @@ -1,6 +1,6 @@ #pragma once -#include "python_md/md_bridge.h" +#include "python_interop/md/md_bridge.h" namespace bts::md::py_interop { diff --git a/src/lib_python_md/python_md/py_book.cpp b/src/lib_python_interop/python_interop/md/py_book.cpp similarity index 97% rename from src/lib_python_md/python_md/py_book.cpp rename to src/lib_python_interop/python_interop/md/py_book.cpp index 180ce1ff5..9b3ef96c0 100644 --- a/src/lib_python_md/python_md/py_book.cpp +++ b/src/lib_python_interop/python_interop/md/py_book.cpp @@ -1,5 +1,5 @@ #include "py_book.h" -#include "python_md/py_book_level.h" +#include "python_interop/md/py_book_level.h" namespace bts::md::py_interop { diff --git a/src/lib_python_md/python_md/py_book.h b/src/lib_python_interop/python_interop/md/py_book.h similarity index 98% rename from src/lib_python_md/python_md/py_book.h rename to src/lib_python_interop/python_interop/md/py_book.h index a78c1fe4e..01ea33436 100644 --- a/src/lib_python_md/python_md/py_book.h +++ b/src/lib_python_interop/python_interop/md/py_book.h @@ -1,7 +1,7 @@ #pragma once -#include "python_md/py_book_level.h" -#include "python_md/py_book_side.h" +#include "python_interop/md/py_book_level.h" +#include "python_interop/md/py_book_side.h" #include "spark/core/FMT_IMPL.h" #include "spark/io/json_formatter.h" #include "spark/utils/bts_assert.h" diff --git a/src/lib_python_md/python_md/py_book_level.h b/src/lib_python_interop/python_interop/md/py_book_level.h similarity index 100% rename from src/lib_python_md/python_md/py_book_level.h rename to src/lib_python_interop/python_interop/md/py_book_level.h diff --git a/src/lib_python_md/python_md/py_book_side.cpp b/src/lib_python_interop/python_interop/md/py_book_side.cpp similarity index 100% rename from src/lib_python_md/python_md/py_book_side.cpp rename to src/lib_python_interop/python_interop/md/py_book_side.cpp diff --git a/src/lib_python_md/python_md/py_book_side.h b/src/lib_python_interop/python_interop/md/py_book_side.h similarity index 98% rename from src/lib_python_md/python_md/py_book_side.h rename to src/lib_python_interop/python_interop/md/py_book_side.h index 0bd970ba6..62b1c962a 100644 --- a/src/lib_python_md/python_md/py_book_side.h +++ b/src/lib_python_interop/python_interop/md/py_book_side.h @@ -1,6 +1,6 @@ #pragma once -#include "python_md/py_book_level.h" +#include "python_interop/md/py_book_level.h" #include "spark/core/FMT_IMPL.h" #include "spark/io/json_formatter.h" diff --git a/src/lib_python_md/python_md/py_md_provider.cpp b/src/lib_python_interop/python_interop/md/py_md_provider.cpp similarity index 100% rename from src/lib_python_md/python_md/py_md_provider.cpp rename to src/lib_python_interop/python_interop/md/py_md_provider.cpp diff --git a/src/lib_python_md/python_md/py_md_provider.h b/src/lib_python_interop/python_interop/md/py_md_provider.h similarity index 90% rename from src/lib_python_md/python_md/py_md_provider.h rename to src/lib_python_interop/python_interop/md/py_md_provider.h index a76147737..4029d5ab5 100644 --- a/src/lib_python_md/python_md/py_md_provider.h +++ b/src/lib_python_interop/python_interop/md/py_md_provider.h @@ -1,7 +1,7 @@ #pragma once -#include "python_md/md_bridge.h" -#include "python_md/md_provider.h" +#include "python_interop/md/md_bridge.h" +#include "python_interop/md/md_provider.h" #include diff --git a/src/lib_python_md/python_md/py_module.cpp b/src/lib_python_interop/python_interop/md/py_module.cpp similarity index 95% rename from src/lib_python_md/python_md/py_module.cpp rename to src/lib_python_interop/python_interop/md/py_module.cpp index 8c21ba12f..4d2614e67 100644 --- a/src/lib_python_md/python_md/py_module.cpp +++ b/src/lib_python_interop/python_interop/md/py_module.cpp @@ -1,8 +1,8 @@ #include "md_provider.h" -#include "python_md/py_book_level.h" -#include "python_md/py_book_side.h" -#include "python_md/py_md_provider.h" -#include "python_md/py_trade.h" +#include "python_interop/md/py_book_level.h" +#include "python_interop/md/py_book_side.h" +#include "python_interop/md/py_md_provider.h" +#include "python_interop/md/py_trade.h" #include #include diff --git a/src/lib_python_md/python_md/py_trade.h b/src/lib_python_interop/python_interop/md/py_trade.h similarity index 100% rename from src/lib_python_md/python_md/py_trade.h rename to src/lib_python_interop/python_interop/md/py_trade.h diff --git a/src/lib_python_core/python_core/py_contract_price_info.h b/src/lib_python_interop/python_interop/py_contract_price_info.h similarity index 74% rename from src/lib_python_core/python_core/py_contract_price_info.h rename to src/lib_python_interop/python_interop/py_contract_price_info.h index fb5b41e30..ee7e9b888 100644 --- a/src/lib_python_core/python_core/py_contract_price_info.h +++ b/src/lib_python_interop/python_interop/py_contract_price_info.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -42,8 +43,14 @@ class PyContractPriceInfo { float multiplier_ = 1; float cash_per_point_ = 1; float max_price_ = std::numeric_limits::max(); + + friend void to_json(nlohmann::json &j, const PyContractPriceInfo &cpi); + friend void from_json(const nlohmann::json &j, PyContractPriceInfo &cpi); }; +void to_json(nlohmann::json &j, const PyContractPriceInfo &cpi); +void from_json(const nlohmann::json &j, PyContractPriceInfo &cpi); + std::ostream &operator<<( std::ostream &out, const PyContractPriceInfo &qty_info) noexcept; @@ -84,6 +91,37 @@ inline std::string PyContractPriceInfo::to_string() const { return str.str(); } +inline void to_json(nlohmann::json &j, const PyContractPriceInfo &cpi) { + j["tick_unit"] = cpi.tick_unit_; + j["tick_base"] = cpi.tick_base_; + j["multiplier"] = cpi.multiplier_; + j["cash_per_point"] = cpi.cash_per_point_; + j["max_price"] = cpi.max_price_; +} + +inline void from_json(const nlohmann::json &j, PyContractPriceInfo &cpi) { + auto it = j.find("tick_unit"); + if (it != j.end()) { + it->get_to(cpi.tick_unit_); + } + it = j.find("tick_base"); + if (it != j.end()) { + it->get_to(cpi.tick_base_); + } + it = j.find("multiplier"); + if (it != j.end()) { + it->get_to(cpi.multiplier_); + } + it = j.find("cash_per_point"); + if (it != j.end()) { + it->get_to(cpi.cash_per_point_); + } + it = j.find("max_price"); + if (it != j.end()) { + it->get_to(cpi.max_price_); + } +} + } // namespace bts::md::py_interop template <> diff --git a/src/lib_python_core/python_core/py_contract_quantity_info.h b/src/lib_python_interop/python_interop/py_contract_quantity_info.h similarity index 72% rename from src/lib_python_core/python_core/py_contract_quantity_info.h rename to src/lib_python_interop/python_interop/py_contract_quantity_info.h index d64fd6ceb..a15a56287 100644 --- a/src/lib_python_core/python_core/py_contract_quantity_info.h +++ b/src/lib_python_interop/python_interop/py_contract_quantity_info.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -44,8 +45,16 @@ class PyContractQuantityInfo { float tick_base_ = std::numeric_limits::quiet_NaN(); float multiplier_ = 1; float max_price_ = std::numeric_limits::max(); + + friend void to_json( + nlohmann::json &j, const PyContractQuantityInfo &qty_info); + friend void from_json( + const nlohmann::json &j, PyContractQuantityInfo &qty_info); }; +void to_json(nlohmann::json &j, const PyContractQuantityInfo &qty_info); +void from_json(const nlohmann::json &j, PyContractQuantityInfo &qty_info); + std::ostream &operator<<( std::ostream &out, const PyContractQuantityInfo &qty_info) noexcept; @@ -89,6 +98,43 @@ inline std::string PyContractQuantityInfo::to_string() const { return str.str(); } +inline void to_json(nlohmann::json &j, const PyContractQuantityInfo &qty_info) { + j["min_value"] = qty_info.min_value_; + j["max_value"] = qty_info.max_value_; + j["tick_unit"] = qty_info.tick_unit_; + j["tick_base"] = qty_info.tick_base_; + j["multiplier"] = qty_info.multiplier_; + j["max_price"] = qty_info.max_price_; +} + +inline void from_json( + const nlohmann::json &j, PyContractQuantityInfo &qty_info) { + auto it = j.find("min_value"); + if (it != j.end()) { + it->get_to(qty_info.min_value_); + } + it = j.find("max_value"); + if (it != j.end()) { + it->get_to(qty_info.max_value_); + } + it = j.find("tick_unit"); + if (it != j.end()) { + it->get_to(qty_info.tick_unit_); + } + it = j.find("tick_base"); + if (it != j.end()) { + it->get_to(qty_info.tick_base_); + } + it = j.find("multiplier"); + if (it != j.end()) { + it->get_to(qty_info.multiplier_); + } + it = j.find("max_price"); + if (it != j.end()) { + it->get_to(qty_info.max_price_); + } +} + } // namespace bts::md::py_interop template <> diff --git a/src/lib_python_core/python_core/py_contract_specification.h b/src/lib_python_interop/python_interop/py_contract_specification.h similarity index 79% rename from src/lib_python_core/python_core/py_contract_specification.h rename to src/lib_python_interop/python_interop/py_contract_specification.h index 790898c02..49ac35eb5 100644 --- a/src/lib_python_core/python_core/py_contract_specification.h +++ b/src/lib_python_interop/python_interop/py_contract_specification.h @@ -1,11 +1,12 @@ #pragma once -#include "python_core/py_contract_price_info.h" -#include "python_core/py_contract_quantity_info.h" +#include "python_interop/py_contract_price_info.h" +#include "python_interop/py_contract_quantity_info.h" #include "spark/core/FMT_IMPL.h" #include #include +#include #include #include @@ -36,11 +37,18 @@ class PyContractSpecification { std::vector price_infos_; std::vector quantity_infos_; std::vector notional_quantity_infos_; + + friend void to_json(nlohmann::json &j, const PyContractSpecification &spec); + friend void from_json( + const nlohmann::json &j, PyContractSpecification &spec); }; std::ostream &operator<<( std::ostream &out, const PyContractSpecification &opt) noexcept; +void to_json(nlohmann::json &j, const PyContractSpecification &spec); +void from_json(const nlohmann::json &j, PyContractSpecification &spec); + ////////////////////////////////////////////////// // Definition ////////////////////////////////////////////////// @@ -120,6 +128,27 @@ inline std::string PyContractSpecification::to_string() const { return fmt::format("{}", *this); } +inline void to_json(nlohmann::json &j, const PyContractSpecification &spec) { + j["price_infos"] = spec.price_infos(); + j["quantity_infos"] = spec.quantity_infos(); + j["notional_quantity_infos"] = spec.notional_quantity_infos(); +} + +inline void from_json(const nlohmann::json &j, PyContractSpecification &spec) { + auto it = j.find("price_infos"); + if (it != j.end()) { + it->get_to(spec.price_infos_); + } + it = j.find("quantity_infos"); + if (it != j.end()) { + it->get_to(spec.quantity_infos_); + } + it = j.find("notional_quantity_infos"); + if (it != j.end()) { + it->get_to(spec.notional_quantity_infos_); + } +} + inline std::ostream &operator<<( std::ostream &out, const PyContractSpecification &spec) noexcept { out << fmt::format("{}", spec); diff --git a/src/lib_python_core/python_core/py_future.h b/src/lib_python_interop/python_interop/py_future.h similarity index 100% rename from src/lib_python_core/python_core/py_future.h rename to src/lib_python_interop/python_interop/py_future.h diff --git a/src/lib_python_core/python_core/py_option.h b/src/lib_python_interop/python_interop/py_option.h similarity index 67% rename from src/lib_python_core/python_core/py_option.h rename to src/lib_python_interop/python_interop/py_option.h index 17f2db898..a49027401 100644 --- a/src/lib_python_core/python_core/py_option.h +++ b/src/lib_python_interop/python_interop/py_option.h @@ -4,6 +4,7 @@ #include #include +#include #include namespace bts::md::py_interop { @@ -30,8 +31,14 @@ class PyOption { std::string expiration_; bool is_call_{false}; double strike_{std::numeric_limits::quiet_NaN()}; + + friend void to_json(nlohmann::json &j, const PyOption &opt); + friend void from_json(const nlohmann::json &j, PyOption &opt); }; +void to_json(nlohmann::json &j, const PyOption &opt); +void from_json(const nlohmann::json &j, PyOption &opt); + std::ostream &operator<<(std::ostream &out, const PyOption &opt) noexcept; } // namespace bts::md::py_interop @@ -65,6 +72,28 @@ inline std::string PyOption::to_string() const { return fmt::format("{}", *this); } +inline void to_json(nlohmann::json &j, const PyOption &opt) { + j["exp"] = opt.expiration(); + j["k"] = opt.strike(); + j["cp"] = opt.is_call() ? "C" : "P"; +} + +inline void from_json(const nlohmann::json &j, PyOption &opt) { + auto it = j.find("exp"); + if (it != j.end()) { + it->get_to(opt.expiration_); + } + it = j.find("k"); + if (it != j.end()) { + it->get_to(opt.strike_); + } + it = j.find("cp"); + if (it != j.end()) { + const std::string cp = it->get(); + opt.is_call_ = !cp.empty() && (cp[0] == 'C' || cp[0] == 'c'); + } +} + inline std::ostream &operator<<( std::ostream &out, const PyOption &opt) noexcept { out << fmt::format("{}", opt); diff --git a/src/lib_python_core/python_core/py_security.cpp b/src/lib_python_interop/python_interop/py_security.cpp similarity index 58% rename from src/lib_python_core/python_core/py_security.cpp rename to src/lib_python_interop/python_interop/py_security.cpp index 6b120215c..94da8fdfc 100644 --- a/src/lib_python_core/python_core/py_security.cpp +++ b/src/lib_python_interop/python_interop/py_security.cpp @@ -9,13 +9,14 @@ using std::string; using std::uint32_t; void to_json(nlohmann::json &j, const PySecurity &sec) { - j["id"] = sec.id(); - j["eid"] = sec.exchange_id(); - j["name"] = sec.name(); - j["venue"] = sec.venue(); - j["cxy"] = sec.currency(); - j["grp"] = sec.group(); - j["ass"] = sec.asset(); + j["id"] = sec.id_; + j["eid"] = sec.exchange_id_; + j["name"] = sec.name_; + j["venue"] = sec.venue_; + j["cxy"] = sec.currency_; + j["grp"] = sec.group_; + j["ass"] = sec.asset_; + j["spec"] = sec.contract_spec_; std::visit( overloaded{ [&](const PySpot &) { j["typ"] = "SPOT"; }, @@ -34,13 +35,38 @@ void to_json(nlohmann::json &j, const PySecurity &sec) { } void from_json(const nlohmann::json &j, PySecurity &sec) { - sec.set_id(j.at("id").get()); - sec.set_exchange_id(j.at("eid").get()); - sec.set_name(j.at("name").get()); - sec.set_venue(j.at("venue").get()); - sec.set_currency(j.at("cxy").get()); - sec.set_group(j.at("grp").get()); - sec.set_asset(j.at("ass").get()); + auto it = j.find("id"); + if (it != j.end()) { + it->get_to(sec.id_); + } + it = j.find("eid"); + if (it != j.end()) { + it->get_to(sec.exchange_id_); + } + it = j.find("name"); + if (it != j.end()) { + it->get_to(sec.name_); + } + it = j.find("venue"); + if (it != j.end()) { + it->get_to(sec.venue_); + } + it = j.find("cxy"); + if (it != j.end()) { + it->get_to(sec.currency_); + } + it = j.find("grp"); + if (it != j.end()) { + it->get_to(sec.group_); + } + it = j.find("ass"); + if (it != j.end()) { + it->get_to(sec.asset_); + } + it = j.find("spec"); + if (it != j.end()) { + it->get_to(sec.contract_spec_); + } const auto typ = j.at("typ").get(); if (typ.starts_with('S')) { sec.sec_.emplace(); diff --git a/src/lib_python_core/python_core/py_security.h b/src/lib_python_interop/python_interop/py_security.h similarity index 96% rename from src/lib_python_core/python_core/py_security.h rename to src/lib_python_interop/python_interop/py_security.h index 3eabcc36c..0964404b2 100644 --- a/src/lib_python_core/python_core/py_security.h +++ b/src/lib_python_interop/python_interop/py_security.h @@ -1,9 +1,9 @@ #pragma once -#include "python_core/py_contract_specification.h" -#include "python_core/py_future.h" -#include "python_core/py_option.h" -#include "python_core/py_spot.h" +#include "python_interop/py_contract_specification.h" +#include "python_interop/py_future.h" +#include "python_interop/py_option.h" +#include "python_interop/py_spot.h" #include "spark/io/json_formatter.h" #include diff --git a/src/lib_python_core/python_core/py_spot.h b/src/lib_python_interop/python_interop/py_spot.h similarity index 100% rename from src/lib_python_core/python_core/py_spot.h rename to src/lib_python_interop/python_interop/py_spot.h diff --git a/src/lib_python_core/python_core/python_activator.h b/src/lib_python_interop/python_interop/python_activator.h similarity index 95% rename from src/lib_python_core/python_core/python_activator.h rename to src/lib_python_interop/python_interop/python_activator.h index e693e162f..66688b5c0 100644 --- a/src/lib_python_core/python_core/python_activator.h +++ b/src/lib_python_interop/python_interop/python_activator.h @@ -1,13 +1,13 @@ #pragma once -#include "python_core/python_activator_impl.h" +#include "python_interop/python_activator_impl.h" #include #include #include #include -namespace python_core { +namespace python_interop { class PythonActivator { public: @@ -90,4 +90,4 @@ class PythonActivatorT { Target *target_ = nullptr; }; -} // namespace python_core +} // namespace python_interop diff --git a/src/lib_python_interop/python_interop/python_activator_impl.h b/src/lib_python_interop/python_interop/python_activator_impl.h new file mode 100644 index 000000000..87d28863d --- /dev/null +++ b/src/lib_python_interop/python_interop/python_activator_impl.h @@ -0,0 +1,118 @@ +#pragma once + +#include "python_interop/python_interpreter.h" +#include "spark/core/TimeStamp.h" +#include "spark/io/json_formatter.h" +#include "spark/logging/logging.h" + +#include +#include +#include +#include + +namespace python_interop { + +struct PythonActivator_Impl { + std::string target; + std::vector module_paths; + std::optional target_obj; + + PythonActivator_Impl( + std::string target, std::vector module_paths) + : target(std::move(target)), module_paths(std::move(module_paths)) {} + + ~PythonActivator_Impl() { reset(); } + + pybind11::object &get_target_obj() { + return get_target_obj([](pybind11::object &) {}); + } + + template + pybind11::object &get_target_obj( + std::function init, Args &&...args) { + if (!target_obj) { + // Prevent python from overwriting our signal handler (so ctrl+c + // actually terminates the process). + auto orig_signal_handler = std::signal(SIGINT, SIG_DFL); + target_obj = create_target_with_args( + init, std::forward(args)...); + std::signal(SIGINT, orig_signal_handler); + } + return *target_obj; + } + +// Undef after use (this is a header) +#define LOG_EVENT(ev) \ + LOG(OrderMan, bts::Info, \ + R"([py_activator.json]{{ "tgt": "{}", "path": {}, "ev": )" ev \ + R"(, "ts": {} }})", \ + target, bts::io::json_formatter(module_paths), \ + bts::core::TimeStamp::now_ns()) +#define LOG_ERROR(err) \ + LOG(OrderMan, bts::Info, \ + R"([py_activator.json]{{ "tgt": "{}", "path": {}, "ev": "ERROR", "err": "{}", "ts": {} }})", \ + target, bts::io::json_formatter(module_paths), err, \ + bts::core::TimeStamp::now_ns()) +#define LOG_CREATE LOG_EVENT("CREATE") +#define LOG_RESET LOG_EVENT("RESET") + + void reset() { + LOG_RESET; + target_obj.reset(); + } + + private: + template + pybind11::object create_target_with_args( + std::function init, Args &&...args) { + LOG_CREATE; + return PythonInterpreter::exec_t( + [this, &init, + args_tuple = + std::make_tuple(std::forward(args)...)]() mutable { + std::string target_module_name; + std::string target_type_name; + PythonInterpreter::parse_target( + target, &target_module_name, &target_type_name); + pybind11::object obj; + try { + // Prepare module path + pybind11::module sys = pybind11::module::import("sys"); + for (const auto &module_path : module_paths) { + sys.attr("path").attr("append")(module_path.c_str()); + } + // Import target module + pybind11::module target_module = + pybind11::module::import(target_module_name.c_str()); + // Instantiate target using tuple expansion + obj = std::apply( + [&](auto &&...unpacked_args) { + return target_module.attr(target_type_name.c_str())( + std::forward( + unpacked_args)...); + }, + std::move(args_tuple)); + init(obj); + } catch (const std::exception &ex) { + LOG_ERROR(std::string{ex.what()}); + } + return obj; + }); + } + + pybind11::object create_target() { + return create_target_with_args([](pybind11::object &) {}); + } + + pybind11::object create_target( + std::function init) { + return create_target_with_args(init); + } + +#undef LOG_EVENT +#undef LOG_ERROR +#undef LOG_CREATE +#undef LOG_RESET +}; + +} // namespace python_interop diff --git a/src/lib_python_interop/python_interop/python_interop_module.cpp b/src/lib_python_interop/python_interop/python_interop_module.cpp new file mode 100644 index 000000000..d380b84e7 --- /dev/null +++ b/src/lib_python_interop/python_interop/python_interop_module.cpp @@ -0,0 +1,276 @@ +#include "python_interop/execution/execution_gateway.h" +#include "python_interop/execution/execution_gateway_bridge.h" +#include "python_interop/execution/py_cancel_order_request.h" +#include "python_interop/execution/py_cancel_replace_order_request.h" +#include "python_interop/execution/py_create_order_request.h" +#include "python_interop/execution/py_execution_gateway.h" +#include "python_interop/execution/py_position_info.h" +#include "python_interop/md/md_bridge.h" +#include "python_interop/md/md_provider.h" +#include "python_interop/md/py_book.h" +#include "python_interop/md/py_book_level.h" +#include "python_interop/md/py_book_side.h" +#include "python_interop/md/py_md_provider.h" +#include "python_interop/md/py_trade.h" +#include "python_interop/py_contract_price_info.h" +#include "python_interop/py_contract_quantity_info.h" +#include "python_interop/py_contract_specification.h" +#include "python_interop/py_security.h" + +#include +#include +#include +#include + +using bts::md::py_interop::MdBridge; +using bts::md::py_interop::MdProvider; +using bts::md::py_interop::PyBook; +using bts::md::py_interop::PyBookLevel; +using bts::md::py_interop::PyBookSide; +using bts::md::py_interop::PyContractPriceInfo; +using bts::md::py_interop::PyContractQuantityInfo; +using bts::md::py_interop::PyContractSpecification; +using bts::md::py_interop::PyFuture; +using bts::md::py_interop::PyMdProvider; +using bts::md::py_interop::PyOption; +using bts::md::py_interop::PySecurity; +using bts::md::py_interop::PySpot; +using bts::md::py_interop::PyTrade; +using python_execution::ExecutionGateway; +using python_execution::ExecutionGatewayBridge; +using python_execution::PyCancelOrderRequest; +using python_execution::PyCancelReplaceOrderRequest; +using python_execution::PyCreateOrderRequest; +using python_execution::PyExecutionGateway; +using python_execution::PyExecutionReport; +using python_execution::PyPositionInfo; +using std::shared_ptr; + +PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr) + +namespace py = pybind11; + +#define _STR(x) #x +// In general, we bind c++ methods to python methods with the same name verbatim. +// The DEF macro avoids typos in the python method name for this common case. +#define DEF(type, member) def(_STR(member), &type::member) +#define DEF_ARGS(type, member, ...) \ + def(_STR(member), &type::member, __VA_ARGS__) +#define EXPLICIT_DEF(type, member, member_type) \ + def(_STR(member), (member_type) & type::member) +// Defines get/set methods for a pseudo property +#define PROP(type, name) \ + def(_STR(name), &type::name) \ + .def(_STR(set_##name), &type::set_##name, py::arg(#name)) + +// Define python module with c++ classes. +PYBIND11_MODULE(python_interop_module, m) { + // price info + py::class_(m, "PyContractPriceInfo") + .def(py::init<>()) + .PROP(PyContractPriceInfo, tick_unit) + .PROP(PyContractPriceInfo, tick_base) + .PROP(PyContractPriceInfo, multiplier) + .PROP(PyContractPriceInfo, cash_per_point) + .PROP(PyContractPriceInfo, max_price) + .def("__str__", &PyContractPriceInfo::to_string); + py::bind_vector>( + m, "PyContractPriceInfoList"); + // quantity info + py::class_(m, "PyContractQuantityInfo") + .def(py::init<>()) + .PROP(PyContractQuantityInfo, min_value) + .PROP(PyContractQuantityInfo, max_value) + .PROP(PyContractQuantityInfo, tick_unit) + .PROP(PyContractQuantityInfo, tick_base) + .PROP(PyContractQuantityInfo, multiplier) + .PROP(PyContractQuantityInfo, max_price) + .def("__str__", &PyContractQuantityInfo::to_string); + py::bind_vector>( + m, "PyContractQuantityInfoList"); + // contract spec + py::class_(m, "PyContractSpecification") + .def(py::init<>()) + .PROP(PyContractSpecification, price_infos) + .PROP(PyContractSpecification, quantity_infos) + .PROP(PyContractSpecification, notional_quantity_infos) + .def("__str__", &PyContractSpecification::to_string); + // security + py::class_(m, "PySpot") + .def(py::init<>()) + .def("__str__", &PySpot::to_string); + py::class_(m, "PyFuture") + .def(py::init<>()) + .PROP(PyFuture, expiration) + .def("__str__", &PyFuture::to_string); + py::class_(m, "PyOption") + .def(py::init<>()) + .PROP(PyOption, expiration) + .PROP(PyOption, is_call) + .PROP(PyOption, strike) + .def("__str__", &PyOption::to_string); + py::class_>(m, "PySecurity") + .def(py::init<>()) + .PROP(PySecurity, id) + .PROP(PySecurity, exchange_id) + .PROP(PySecurity, venue) + .PROP(PySecurity, name) + .PROP(PySecurity, currency) + .PROP(PySecurity, group) + .PROP(PySecurity, asset) + .PROP(PySecurity, contract_spec) + .PROP(PySecurity, spot) + .PROP(PySecurity, future) + .PROP(PySecurity, option) + .def("__str__", &PySecurity::to_string); + py::bind_vector>>(m, "PySecurityList"); + + ////////////////////////////////////////////////// + // Md + ////////////////////////////////////////////////// + using BookSideGetter = PyBookSide &(PyBook::*)(); + using BookLevelGetter = PyBookLevel (PyBookSide::*)(int) const; + // book + py::class_(m, "PyBookLevel") + .def(py::init<>()) + .PROP(PyBookLevel, price) + .PROP(PyBookLevel, quantity); + py::class_(m, "PyBookSide") + .def(py::init<>()) + .EXPLICIT_DEF(PyBookSide, get_level, BookLevelGetter) + .DEF(PyBookSide, set_level) + .PROP(PyBookSide, is_ascending) + .def("__len__", &PyBookSide::size) + .def("__str__", &PyBookSide::to_string); + py::class_>(m, "PyBook") + .def(py::init<>()) + .PROP(PyBook, venue) + .PROP(PyBook, security_type) + .PROP(PyBook, exchange_security_id) + .EXPLICIT_DEF(PyBook, bids, BookSideGetter) + .DEF(PyBook, set_bids) + .EXPLICIT_DEF(PyBook, asks, BookSideGetter) + .DEF(PyBook, set_asks) + .def( + "set_data", &PyBook::set_data, py::arg("exch_sec_id"), + py::arg("venue"), py::arg("sec_type"), py::arg("bids"), + py::arg("asks")) + .def("__len__", &PyBook::size) + .def("__str__", &PyBook::to_string); + py::bind_vector>>(m, "PyBookList"); + // trade + py::class_>(m, "PyTrade") + .def(py::init<>()) + .PROP(PyTrade, venue) + .PROP(PyTrade, security_type) + .PROP(PyTrade, exchange_security_id) + .PROP(PyTrade, price) + .PROP(PyTrade, quantity) + .PROP(PyTrade, aggressor_side) + .PROP(PyTrade, timestamp); + py::bind_vector>>(m, "PyTradeList"); + // md bridge + py::class_(m, "MdBridge") + .DEF_ARGS(MdBridge, on_securities, py::arg("securities")) + .DEF_ARGS(MdBridge, on_books, py::arg("books")) + .DEF_ARGS(MdBridge, on_trades, py::arg("trades")); + // md provider + py::class_(m, "MdProvider") + .def(py::init<>()) + .DEF_ARGS(MdProvider, start, py::arg("bridge")) + .DEF(MdProvider, stop); + + ////////////////////////////////////////////////// + // Execution + ////////////////////////////////////////////////// + py::class_>( + m, "PyCreateOrderRequest") + .def(py::init<>()) + .PROP(PyCreateOrderRequest, key) + .PROP(PyCreateOrderRequest, exchange_security_id) + .PROP(PyCreateOrderRequest, venue) + .PROP(PyCreateOrderRequest, is_buy) + .PROP(PyCreateOrderRequest, price) + .PROP(PyCreateOrderRequest, quantity) + .PROP(PyCreateOrderRequest, order_type) + .PROP(PyCreateOrderRequest, time_in_force) + .def("__str__", &PyCreateOrderRequest::to_string); + py::class_>( + m, "PyCancelOrderRequest") + .def(py::init<>()) + .PROP(PyCancelOrderRequest, key) + .PROP(PyCancelOrderRequest, exchange_order_id) + .PROP(PyCancelOrderRequest, exchange_security_id) + .PROP(PyCancelOrderRequest, venue) + .PROP(PyCancelOrderRequest, is_buy) + .def("__str__", &PyCancelOrderRequest::to_string); + py::class_< + PyCancelReplaceOrderRequest, shared_ptr>( + m, "PyCancelReplaceOrderRequest") + .def(py::init<>()) + .PROP(PyCancelReplaceOrderRequest, key) + .PROP(PyCancelReplaceOrderRequest, exchange_order_id) + .PROP(PyCancelReplaceOrderRequest, exchange_security_id) + .PROP(PyCancelReplaceOrderRequest, venue) + .PROP(PyCancelReplaceOrderRequest, is_buy) + .PROP(PyCancelReplaceOrderRequest, price) + .PROP(PyCancelReplaceOrderRequest, quantity) + .def("__str__", &PyCancelReplaceOrderRequest::to_string); + py::class_>( + m, "PyExecutionReport") + .def(py::init<>()) + .PROP(PyExecutionReport, key) + .PROP(PyExecutionReport, account) + .PROP(PyExecutionReport, exchange_order_id) + .PROP(PyExecutionReport, exchange_security_id) + .PROP(PyExecutionReport, order_status) + .PROP(PyExecutionReport, time_in_force) + .PROP(PyExecutionReport, exec_type) + .PROP(PyExecutionReport, venue) + .PROP(PyExecutionReport, security_type) + .PROP(PyExecutionReport, is_buy) + .PROP(PyExecutionReport, price) + .PROP(PyExecutionReport, quantity) + .PROP(PyExecutionReport, filled_quantity) + .PROP(PyExecutionReport, last_filled_quantity) + .PROP(PyExecutionReport, last_filled_price) + .PROP(PyExecutionReport, avg_filled_price) + .PROP(PyExecutionReport, order_type) + .PROP(PyExecutionReport, transact_time) + .def( + "set_data", &PyExecutionReport::set_data, py::arg("key"), + py::arg("account"), py::arg("exchange_order_id"), + py::arg("exchange_security_id"), py::arg("venue"), + py::arg("security_type"), py::arg("order_type"), + py::arg("order_status"), py::arg("time_in_force"), + py::arg("exec_type"), py::arg("is_buy"), py::arg("price"), + py::arg("quantity"), py::arg("filled_quantity"), + py::arg("last_filled_quantity"), py::arg("last_filled_price"), + py::arg("avg_filled_price"), py::arg("transact_id"), + py::arg("transact_time")) + .def("__str__", &PyExecutionReport::to_string); + py::class_>(m, "PyPositionInfo") + .def(py::init<>()) + .PROP(PyPositionInfo, account) + .PROP(PyPositionInfo, exchange_security_id) + .PROP(PyPositionInfo, venue) + .PROP(PyPositionInfo, security_type) + .PROP(PyPositionInfo, long_quantity) + .PROP(PyPositionInfo, long_avg_cost) + .PROP(PyPositionInfo, short_quantity) + .PROP(PyPositionInfo, short_avg_cost) + .PROP(PyPositionInfo, transact_time) + .def("__str__", &PyPositionInfo::to_string); + py::class_(m, "ExecutionGatewayBridge") + .def(py::init<>()) + .DEF(ExecutionGatewayBridge, on_execution_report) + .DEF(ExecutionGatewayBridge, on_position_info) + .DEF(ExecutionGatewayBridge, on_is_ready); + py::class_(m, "ExecutionGateway") + .def(py::init<>()) + .DEF_ARGS(ExecutionGateway, start, py::arg("bridge")) + .DEF_ARGS(ExecutionGateway, create_order, py::arg("req")) + .DEF_ARGS(ExecutionGateway, cancel_order, py::arg("req")) + .DEF_ARGS(ExecutionGateway, cancel_replace_order, py::arg("req")) + .DEF(ExecutionGateway, stop); +} diff --git a/src/lib_python_core/python_core/python_interpreter.cpp b/src/lib_python_interop/python_interop/python_interpreter.cpp similarity index 95% rename from src/lib_python_core/python_core/python_interpreter.cpp rename to src/lib_python_interop/python_interop/python_interpreter.cpp index 401609116..5c9b450eb 100644 --- a/src/lib_python_core/python_core/python_interpreter.cpp +++ b/src/lib_python_interop/python_interop/python_interpreter.cpp @@ -7,7 +7,7 @@ #include #include -namespace python_core { +namespace python_interop { using bts::system::string_utils::snake_to_pascal_case; using bts::system::string_utils::split; @@ -47,4 +47,4 @@ bool PythonInterpreter::parse_target( return !module_name->empty() && !type_name->empty(); } -} // namespace python_core +} // namespace python_interop diff --git a/src/lib_python_core/python_core/python_interpreter.h b/src/lib_python_interop/python_interop/python_interpreter.h similarity index 98% rename from src/lib_python_core/python_core/python_interpreter.h rename to src/lib_python_interop/python_interop/python_interpreter.h index 3cb240f2d..71a3c744a 100644 --- a/src/lib_python_core/python_core/python_interpreter.h +++ b/src/lib_python_interop/python_interop/python_interpreter.h @@ -9,7 +9,7 @@ #include #include -namespace python_core { +namespace python_interop { class PythonInterpreter { static std::mutex mx; @@ -122,4 +122,4 @@ class PythonInterpreter { } }; -} // namespace python_core +} // namespace python_interop diff --git a/src/lib_python_execution/python_execution_gateways/bybit_execution_gateway.py b/src/lib_python_interop/python_interop_plugins/bybit_execution_gateway.py similarity index 82% rename from src/lib_python_execution/python_execution_gateways/bybit_execution_gateway.py rename to src/lib_python_interop/python_interop_plugins/bybit_execution_gateway.py index a77952703..1398b18e2 100644 --- a/src/lib_python_execution/python_execution_gateways/bybit_execution_gateway.py +++ b/src/lib_python_interop/python_interop_plugins/bybit_execution_gateway.py @@ -1,5 +1,4 @@ import json -import math import time from typing import Optional import uuid @@ -8,23 +7,21 @@ from py_execution_utils import load_gateway_config from spark_core import ( as_bool, configure_logger, - get_price_decimal_places, - get_qty_decimal_places, + format_price, + format_quantity, parse_float, StructuredLog as SLog, ) import logging -from python_core_module import ( - PySecurity, - PySecurityList, -) -from python_execution_module import ( +from python_interop_module import ( ExecutionGateway, ExecutionGatewayBridge, PyCreateOrderRequest, PyCancelReplaceOrderRequest, PyCancelOrderRequest, PyExecutionReport, + PySecurity, + PySecurityList, ) import threading @@ -36,30 +33,35 @@ class OrderData: self.price = price self.quantity = quantity self.filled_quantity: float = 0 - self.last_filled_quantity: float = math.nan - self.avg_filled_price: float = math.nan - self.last_filled_price: float = math.nan + self.last_filled_quantity: float = 0 + self.avg_filled_price: float = 0 + self.last_filled_price: float = 0 self.exec_type = "PENDING_NEW" - self.securities: PySecurityList = [] def on_update( - self, new_filled_quantity: float, new_avg_filled_price: float, status: str + self, + price: float, + quantity: float, + filled_quantity: float, + avg_filled_price: float, + status: str, ): - lfq = new_filled_quantity - self.filled_quantity + self.price = price + self.filled_quantity = 0 + self.quantity = quantity + lfq = filled_quantity - self.filled_quantity if lfq > 0: if self.filled_quantity == 0: - self.last_filled_quantity = new_filled_quantity - self.last_filled_price = new_avg_filled_price + self.last_filled_quantity = filled_quantity + self.last_filled_price = avg_filled_price else: - price_diff = new_avg_filled_price - self.avg_filled_price - lfq_weight = new_filled_quantity / ( - self.filled_quantity + new_filled_quantity - ) + price_diff = avg_filled_price - self.avg_filled_price + lfq_weight = filled_quantity / (self.filled_quantity + filled_quantity) lfp = self.avg_filled_price + (price_diff / lfq_weight) self.last_filled_price = lfp self.last_filled_quantity = lfq - self.filled_quantity = new_filled_quantity - self.avg_filled_price = new_avg_filled_price + self.filled_quantity = filled_quantity + self.avg_filled_price = avg_filled_price match status: case "Filled": self.exec_type = "FILL" @@ -68,7 +70,7 @@ class OrderData: case "Rejected": self.exec_type = "REJECTED" case _: - self.exec_type = "NEW" if new_filled_quantity == 0 else "PARTIAL_FILL" + self.exec_type = "NEW" if filled_quantity == 0 else "PARTIAL_FILL" class BybitExecutionGateway(ExecutionGateway): @@ -136,27 +138,39 @@ class BybitExecutionGateway(ExecutionGateway): # brits can't spell if status == "Cancelled": status = "Canceled" - ord_data.on_update(total_filled_qty, avg_filled_price, status) + exec_type = ord_data.exec_type + match ord_data.exec_type: + case "PARTIAL_FILL": + exec_type = "PARTIALLY_FILLED" + case _: + pass + px = parse_float(d["price"]) + qty = parse_float(d["qty"]) + ord_data.on_update( + px, qty, total_filled_qty, avg_filled_price, status + ) + transact_time = int(d["updatedTime"]) * 1_000_000 er.set_data( key=ord_data.bts_id, exchange_order_id=d["orderLinkId"], - transact_id=d["seq"], venue="BYBIT", security_type="FUTURE", exchange_security_id=d["symbol"], account=self.account, order_type=d["orderType"], is_buy=d["side"] == "Buy", - price=parse_float(d["price"]), - quantity=parse_float(d["qty"]), + price=px, + quantity=qty, time_in_force=d["timeInForce"], - exec_type=ord_data.exec_type, + exec_type=exec_type, order_status=status, last_filled_price=ord_data.last_filled_price, last_filled_quantity=ord_data.last_filled_quantity, filled_quantity=total_filled_qty, avg_filled_price=avg_filled_price, - transact_time=int(d["updatedTime"]) * 1_000_000, + # There is no transact_id in Bybit, so we use updatedTime + transact_id=str(transact_time), + transact_time=transact_time, ) self.logger.info("[bybit_gw_exe_rep_out.json]%s" % er) bridge.on_execution_report(er) @@ -235,18 +249,16 @@ class BybitExecutionGateway(ExecutionGateway): sec = self.securities.get(sec_name) if not sec: return - price_decimal_places = get_price_decimal_places(sec) - qty_decimal_places = get_qty_decimal_places(sec) params = { "orderLinkId": exch_ord_id, "category": "linear", "symbol": sec_name, "side": "Buy" if req.is_buy() else "Sell", "orderType": req.order_type().title(), - "price": f"{price_decimal_places:.2f}", - "qty": f"{qty_decimal_places:.2f}", + "price": format_price(sec, px), + "qty": format_quantity(sec, qty), } - self.logger.info("[bybit_gw_create_order_msg.json]%s" % json.dumps(params)) + self.logger.info("[bybit_gw_crt_ord_send.json]%s" % json.dumps(params)) self.trade_cli.place_order(lambda msg: self._on_order_ack("crt", msg), **params) def cancel_replace_order(self, req: PyCancelReplaceOrderRequest): @@ -263,14 +275,12 @@ class BybitExecutionGateway(ExecutionGateway): sec = self.securities.get(sec_name) if not sec: return - price_decimal_places = get_price_decimal_places(sec) - qty_decimal_places = get_qty_decimal_places(sec) params = { "orderLinkId": ord_data.exch_id, "category": "linear", "symbol": req.exchange_security_id(), - "price": f"{price_decimal_places:.2f}", - "qty": f"{qty_decimal_places:.2f}", + "price": format_price(sec, req.price()), + "qty": format_quantity(sec, req.quantity()), } self.trade_cli.amend_order( lambda msg: self._on_order_ack("cxlrep", msg), **params diff --git a/src/lib_python_md/python_md_providers/bybit_md_provider.py b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py similarity index 96% rename from src/lib_python_md/python_md_providers/bybit_md_provider.py rename to src/lib_python_interop/python_interop_plugins/bybit_md_provider.py index 3fde0ac87..fcfc812e9 100644 --- a/src/lib_python_md/python_md_providers/bybit_md_provider.py +++ b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py @@ -7,8 +7,13 @@ import threading import signal from typing import Optional from pybit.unified_trading import WebSocket -import python_md_module as pymd -import python_core_module as pycore +from python_interop_module import ( + MdBridge, + MdProvider, + PyBook, + PyBookList, + PySecurityList, +) from bybit_conf import CONTRACTS from spark_core import ( configure_logger, @@ -69,12 +74,12 @@ def fetch_contract_specs() -> dict: logger.error(f"Failed to fetch contract specs: {e}", exc_info=True) return specs_dict -class BybitMdProvider(pymd.MdProvider): +class BybitMdProvider(MdProvider): def __init__(self): super().__init__() self.ws = None self.ws_thread: Optional[threading.Thread] = None - self.bridge: Optional[pymd.MdBridge] = None + self.bridge: Optional[MdBridge] = None self.stop_event = threading.Event() self._setup_signal_handlers() @@ -88,12 +93,12 @@ class BybitMdProvider(pymd.MdProvider): signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGTSTP, signal_handler) - def start_impl(self, bridge: pymd.MdBridge): + def start_impl(self, bridge: MdBridge): self.bridge = bridge logger.info("Starting BybitMdProvider") contract_specs = fetch_contract_specs() - sec_list = pycore.PySecurityList() + sec_list = PySecurityList() for contract in CONTRACTS: normalized_contract = normalize_contract_name(contract) @@ -177,7 +182,7 @@ class BybitMdProvider(pymd.MdProvider): logger.error(f"Bybit WebSocket fatal error: {e}", exc_info=True) self.stop() - def start(self, bridge: pymd.MdBridge): + def start(self, bridge: MdBridge): def start_wrapper(): try: self.start_impl(bridge) @@ -206,7 +211,7 @@ class BybitMdProvider(pymd.MdProvider): logger.debug(f"📘 BOOK DATA for {contract}: bids={sorted_bids} asks={sorted_asks}") - book = pymd.PyBook() + book = PyBook() book.set_data( contract, "Bybit", @@ -214,7 +219,7 @@ class BybitMdProvider(pymd.MdProvider): np.array(sorted_bids, dtype=np.float64), np.array(sorted_asks, dtype=np.float64), ) - book_list = pymd.PyBookList() + book_list = PyBookList() book_list.append(book) if self.bridge: self.bridge.on_books(book_list) diff --git a/src/lib_python_md/python_md_providers/config.py b/src/lib_python_interop/python_interop_plugins/config.py similarity index 100% rename from src/lib_python_md/python_md_providers/config.py rename to src/lib_python_interop/python_interop_plugins/config.py diff --git a/src/lib_python_execution/python_execution_gateways/dummy_execution_gateway.py b/src/lib_python_interop/python_interop_plugins/dummy_execution_gateway.py similarity index 96% rename from src/lib_python_execution/python_execution_gateways/dummy_execution_gateway.py rename to src/lib_python_interop/python_interop_plugins/dummy_execution_gateway.py index ea2ede70f..73b4714bc 100644 --- a/src/lib_python_execution/python_execution_gateways/dummy_execution_gateway.py +++ b/src/lib_python_interop/python_interop_plugins/dummy_execution_gateway.py @@ -9,7 +9,7 @@ import uuid import websocket from binance_client import BinanceClient -import python_execution_module as pyexe +import python_interop_module as pyinter BINANCE_API_KEY = "2RJ9H02ztFGWxeY2oRpnFYZNU8mki4V1yMvKrDU08HCiEhO4VLe2ePSUFqEYTNxc" @@ -41,7 +41,7 @@ def to_bts_order_status(ord_status: str): return ord_status -class DummyExecutionGateway(pyexe.ExecutionGateway): +class DummyExecutionGateway(pyinter.ExecutionGateway): def __init__(self): super(DummyExecutionGateway, self).__init__() self.bridge = None @@ -92,7 +92,7 @@ class DummyExecutionGateway(pyexe.ExecutionGateway): ) return is_old_ord = True - er = pyexe.PyExecutionReport() + er = pyinter.PyExecutionReport() er.set_key(bts_ord_id) er.set_exchange_security_id(upd["s"]) er.set_security_type("SPOT") @@ -144,7 +144,7 @@ class DummyExecutionGateway(pyexe.ExecutionGateway): pass return default_val - er = pyexe.PyExecutionReport() + er = pyinter.PyExecutionReport() er.set_key(bts_ord_id) er.set_venue("BINANCE_USDS_FUTURES") er.set_security_type("FUTURE") @@ -192,7 +192,7 @@ class DummyExecutionGateway(pyexe.ExecutionGateway): if self.bridge: self.bridge.on_is_ready(False) - def start(self, bridge: pyexe.ExecutionGatewayBridge): + def start(self, bridge: pyinter.ExecutionGatewayBridge): print("DummyExecutionGateway.start") self.bridge = bridge self.client = BinanceClient(venue="BINANCE_USDS_FUTURES") @@ -204,7 +204,7 @@ class DummyExecutionGateway(pyexe.ExecutionGateway): ) print("DummyExecutionGateway.start. Subscribed to binance user data") - def cancel_order(self, req: pyexe.PyCancelOrderRequest): + def cancel_order(self, req: pyinter.PyCancelOrderRequest): if not self.client: self.logger.error("Cannot cancel order. BinanceClient not created.") return @@ -219,7 +219,7 @@ class DummyExecutionGateway(pyexe.ExecutionGateway): except Exception as ex: self.logger.error("Create order request threw. %s" % str(ex)) - def create_order(self, req: pyexe.PyCreateOrderRequest): + def create_order(self, req: pyinter.PyCreateOrderRequest): if not self.client: self.logger.error("Cannot create order. BinanceClient not created.") return diff --git a/src/lib_python_md/python_md_providers/dummy_md_provider.py b/src/lib_python_interop/python_interop_plugins/dummy_md_provider.py similarity index 77% rename from src/lib_python_md/python_md_providers/dummy_md_provider.py rename to src/lib_python_interop/python_interop_plugins/dummy_md_provider.py index 22983a692..350e6ddb2 100644 --- a/src/lib_python_md/python_md_providers/dummy_md_provider.py +++ b/src/lib_python_interop/python_interop_plugins/dummy_md_provider.py @@ -1,16 +1,29 @@ -import python_md_module as pymd -from py_md_utils import get_security, get_book_side, get_trade, get_contract_spec, get_price_info, get_quantity_info +from python_interop_module import ( + MdBridge, + MdProvider, + PySecurityList, + PyBook, + PyBookList, + PyTradeList, +) +from spark_core import ( + get_security, + get_contract_spec, + get_price_info, + get_quantity_info, +) +from py_md_utils import get_book_side, get_trade -class DummyMdProvider(pymd.MdProvider): +class DummyMdProvider(MdProvider): def __init__(self): super(DummyMdProvider, self).__init__() pass - def start(self, bridge: pymd.MdBridge): + def start(self, bridge: MdBridge): # start subscriptions, bind callbacks etc. def add_fake_securities(): - sec_list = pymd.PySecurityList() + sec_list = PySecurityList() sec_list.append( get_security( venue="Glbx", @@ -27,20 +40,21 @@ class DummyMdProvider(pymd.MdProvider): ), qty_info=get_quantity_info(tick_unit=1, tick_base=1), ), - )) + ) + ) bridge.on_securities(sec_list) def add_fake_books(): bids = get_book_side([(99.75, 8), (99.50, 11)], is_bid_side=True) asks = get_book_side([(100.25, 3)], is_bid_side=False) - book = pymd.PyBook() + book = PyBook() book.set_venue("Glbx") book.set_security_type("FUTURE") book.set_exchange_security_id("ESH8") book.set_bids(bids) book.set_asks(asks) # pass book to sink - book_list = pymd.PyBookList() + book_list = PyBookList() book_list.append(book) bridge.on_books(book_list) @@ -53,7 +67,7 @@ class DummyMdProvider(pymd.MdProvider): quantity=2, aggressor_side="Buy", ) - trade_list = pymd.PyTradeList() + trade_list = PyTradeList() trade_list.append(trade) bridge.on_trades(trade_list) diff --git a/src/lib_python_md/python_md_providers/gateio_md_provider.py b/src/lib_python_interop/python_interop_plugins/gateio_md_provider.py similarity index 95% rename from src/lib_python_md/python_md_providers/gateio_md_provider.py rename to src/lib_python_interop/python_interop_plugins/gateio_md_provider.py index 8223d3d35..e2548ddb4 100644 --- a/src/lib_python_md/python_md_providers/gateio_md_provider.py +++ b/src/lib_python_interop/python_interop_plugins/gateio_md_provider.py @@ -7,7 +7,13 @@ import threading import requests from websocket import create_connection, WebSocketConnectionClosedException from typing import Optional, Any -import python_md_module as pymd +from python_interop_module import ( + MdBridge, + MdProvider, + PyBook, + PyBookList, + PySecurityList, +) from config import WS_URL, CONTRACTS from spark_core import ( configure_logger, @@ -54,12 +60,12 @@ def fetch_contract_specs() -> dict: logger.error(f"Failed to fetch batch specs: {e}", exc_info=True) return specs_dict -class GateIoMdProvider(pymd.MdProvider): +class GateIoMdProvider(MdProvider): def __init__(self): super().__init__() self.ws = None self.ws_thread: Optional[threading.Thread] = None - self.bridge: Optional[pymd.MdBridge] = None + self.bridge: Optional[MdBridge] = None self.stop_event = threading.Event() self.ping_thread = None @@ -80,7 +86,7 @@ class GateIoMdProvider(pymd.MdProvider): break time.sleep(15) - def start_impl(self, bridge: pymd.MdBridge): + def start_impl(self, bridge: MdBridge): self.bridge = bridge logger.info(f"Connecting to {WS_URL}") self.ws = create_connection(WS_URL) @@ -96,7 +102,7 @@ class GateIoMdProvider(pymd.MdProvider): contract_specs = fetch_contract_specs() # 1) Send security info - sec_list = pymd.PySecurityList() + sec_list = PySecurityList() for contract in CONTRACTS: asset, quote = contract.split("_") specs = contract_specs[contract] @@ -174,7 +180,7 @@ class GateIoMdProvider(pymd.MdProvider): asks = [(np.float64(l["p"]), np.float64(l["s"])) for l in payload.get("asks", [])] logger.debug(f"📘 BOOK DATA for {contract}: bids={bids} asks={asks}") - book = pymd.PyBook() + book = PyBook() book.set_data( contract, "GateIO", @@ -182,7 +188,7 @@ class GateIoMdProvider(pymd.MdProvider): np.array(bids, dtype=np.float64), np.array(asks, dtype=np.float64), ) - book_list = pymd.PyBookList() + book_list = PyBookList() book_list.append(book) self.bridge.on_books(book_list) logger.debug(f"✔ on_books called for {contract}") @@ -194,7 +200,7 @@ class GateIoMdProvider(pymd.MdProvider): except Exception as e: logger.error("Fatal error in listen loop:", exc_info=True) - def start(self, bridge: pymd.MdBridge): + def start(self, bridge: MdBridge): self.ws_thread = threading.Thread(target=lambda: self.start_impl(bridge)) self.ws_thread.start() diff --git a/src/lib_python_interop/python_interop_plugins/py_execution_utils.py b/src/lib_python_interop/python_interop_plugins/py_execution_utils.py new file mode 100644 index 000000000..6a3ff9ab5 --- /dev/null +++ b/src/lib_python_interop/python_interop_plugins/py_execution_utils.py @@ -0,0 +1,12 @@ +import json +import string +import os + + +def load_gateway_config(name: str, expand_vars: bool = True): + s = "" + with open(f"/home/bts/sparkdata/conf/py_{name}.json") as f: + s = f.read() + if expand_vars: + s = string.Template(s).substitute(os.environ) + return json.loads(s) diff --git a/src/lib_python_md/python_md_providers/py_md_utils.py b/src/lib_python_interop/python_interop_plugins/py_md_utils.py similarity index 80% rename from src/lib_python_md/python_md_providers/py_md_utils.py rename to src/lib_python_interop/python_interop_plugins/py_md_utils.py index b73bc6ff3..1598c578e 100644 --- a/src/lib_python_md/python_md_providers/py_md_utils.py +++ b/src/lib_python_interop/python_interop_plugins/py_md_utils.py @@ -5,17 +5,17 @@ import threading import time from typing import Optional -import python_md_module as pymd +from python_interop_module import PyBookSide, PyBookLevel, PyTrade -PERP_EXP_STRING="2100-01-01T00:00" +PERP_EXP_STRING = "2100-01-01T00:00" def get_book_side(levels: list[tuple[float, float]], is_bid_side: bool): - book_side = pymd.PyBookSide() + book_side = PyBookSide() book_side.set_is_ascending(not is_bid_side) for price, qty in levels: - book_level = pymd.PyBookLevel() + book_level = PyBookLevel() book_level.set_price(price) book_level.set_quantity(qty) book_side.set_level(book_level) @@ -30,7 +30,7 @@ def get_trade( quantity: float, aggressor_side: str, ): - trade = pymd.PyTrade() + trade = PyTrade() trade.set_venue(venue) trade.set_security_type(security_type) trade.set_exchange_security_id(exchange_security_id) diff --git a/src/lib_python_core/python_core_plugins/spark_core.py b/src/lib_python_interop/python_interop_plugins/spark_core.py similarity index 79% rename from src/lib_python_core/python_core_plugins/spark_core.py rename to src/lib_python_interop/python_interop_plugins/spark_core.py index b5726b3fa..dde351f59 100644 --- a/src/lib_python_core/python_core_plugins/spark_core.py +++ b/src/lib_python_interop/python_interop_plugins/spark_core.py @@ -8,7 +8,25 @@ import threading import time from typing import Callable, Optional import websocket -import python_core_module as pycore +from python_interop_module import ( + PyContractPriceInfo, + PyContractPriceInfoList, + PyContractQuantityInfo, + PyContractQuantityInfoList, + PyContractSpecification, + PyFuture, + PyOption, + PySecurity, + PySpot, +) + + +class JsonElement: + def __init__(self, value): + self.value = value + + def __str__(self): + return json.dumps(self.value) class StructuredLog: @@ -21,6 +39,22 @@ class StructuredLog: return json.dumps(self.kwargs) +def _log_level_name(level: int): + match level: + case logging.DEBUG: + return "Diag" + case logging.INFO: + return "Info" + case logging.WARNING: + return "Warn" + case logging.ERROR: + return "Error" + case logging.CRITICAL: + return "Fatal" + case _: + return str(level) + + class FileLogFormatter(logging.Formatter): def __init__(self, subsystem: str): self.subsystem = subsystem @@ -39,7 +73,7 @@ class FileLogFormatter(logging.Formatter): filename_line = f"{record.filename}:{record.lineno}" timestamp = self.formatTime(record) timestamp_ns = int(record.created * 1e9) - return f"{timestamp}|{timestamp_ns}|{self.subsystem:<8}|{record.levelname.title():<5}|{os.getpid():<3}|{record.threadName:<20}|{thread_id}|{filename_line:<35}|{env}| {record.getMessage()}" + return f"{timestamp}|{timestamp_ns}|{self.subsystem:<8}|{_log_level_name(record.levelno):<5}|{os.getpid():<3}|{record.threadName:<20}|{thread_id}|{filename_line:<35}|{env}| {record.getMessage()}" class ConsoleLogFormatter(logging.Formatter): @@ -57,7 +91,7 @@ class ConsoleLogFormatter(logging.Formatter): def format(self, record): filename_line = f"{record.filename}:{record.lineno}" timestamp = self.formatTime(record) - return f"{timestamp}|{self.subsystem:<8}|{record.levelname.title():<5}|{filename_line:<35}| {record.getMessage()}" + return f"{timestamp}|{self.subsystem:<8}|{_log_level_name(record.levelno):<5}|{filename_line:<35}| {record.getMessage()}" def as_bool(*args): @@ -189,20 +223,35 @@ def get_num_decimal_places(tick_sz: float): return math.pow(10, -math.floor(math.log10(tick_sz))) -def get_qty_decimal_places(sec: pycore.PySecurity) -> float: +def get_qty_decimal_places(sec: PySecurity) -> int: qty_infos = sec.contract_spec().quantity_infos() if not qty_infos: return 2 tick_sz = qty_infos[0].tick_unit() / qty_infos[0].tick_base() - return math.pow(10, -math.floor(math.log10(tick_sz))) + return round(-math.floor(math.log10(tick_sz))) -def get_price_decimal_places(sec: pycore.PySecurity) -> float: +def _get_decimal_places(tick_base: float) -> int: + # tick base of 100 -> 2 decimals, tick base of 1000 -> 3 decimals, etc. + tick_sz = 1 / tick_base + return round(-math.floor(math.log10(tick_sz))) + + +def get_price_decimal_places(sec: PySecurity) -> int: price_infos = sec.contract_spec().price_infos() if not price_infos: return 2 - tick_sz = price_infos[0].tick_unit() / price_infos[0].tick_base() - return math.pow(10, -math.floor(math.log10(tick_sz))) + return _get_decimal_places(price_infos[0].tick_base()) + + +def format_price(sec: PySecurity, value: float) -> str: + decimals = get_price_decimal_places(sec) + return f"{value:.{decimals}f}" + + +def format_quantity(sec: PySecurity, value: float) -> str: + decimals = get_qty_decimal_places(sec) + return f"{value:.{decimals}f}" def get_price_info( @@ -215,7 +264,7 @@ def get_price_info( # Use 'not' checks to account for nans (for which every op evaluates to false) if not (tick_unit > 0) or not (tick_base >= tick_unit): raise ValueError(f"Invalid tick factors {tick_unit}/{tick_base}") - price_info = pycore.PyContractPriceInfo() + price_info = PyContractPriceInfo() price_info.set_tick_unit(tick_unit) price_info.set_tick_base(tick_base) if cash_per_point: @@ -243,7 +292,7 @@ def get_quantity_info( elif min_value < (tick_unit / tick_base): # min_value should at least be 1 tick raise ValueError(f"min_value ({min_value}) < tick ({tick_unit}/{tick_base})") - qty_info = pycore.PyContractQuantityInfo() + qty_info = PyContractQuantityInfo() qty_info.set_tick_unit(tick_unit) qty_info.set_tick_base(tick_base) qty_info.set_min_value(min_value) @@ -261,24 +310,24 @@ def get_quantity_info( def get_contract_spec( - price_info: Optional[pycore.PyContractPriceInfo] = None, - qty_info: Optional[pycore.PyContractQuantityInfo] = None, - notional_qty_info: Optional[pycore.PyContractQuantityInfo] = None, + price_info: Optional[PyContractPriceInfo] = None, + qty_info: Optional[PyContractQuantityInfo] = None, + notional_qty_info: Optional[PyContractQuantityInfo] = None, ): - price_infos = pycore.PyContractPriceInfoList() + price_infos = PyContractPriceInfoList() if not price_info: # Default to 3 decimal places price_info = get_price_info(1, 1000) price_infos.append(price_info) - qty_infos = pycore.PyContractQuantityInfoList() + qty_infos = PyContractQuantityInfoList() if not qty_info: # Default to 3 decimal places qty_info = get_quantity_info(1, 1000) qty_infos.append(qty_info) - notional_qty_infos = pycore.PyContractQuantityInfoList() + notional_qty_infos = PyContractQuantityInfoList() if notional_qty_info: notional_qty_infos.append(notional_qty_info) - spec = pycore.PyContractSpecification() + spec = PyContractSpecification() spec.set_price_infos(price_infos) spec.set_quantity_infos(qty_infos) spec.set_notional_quantity_infos(notional_qty_infos) @@ -294,9 +343,9 @@ def get_security( expiration: Optional[str] = None, option_kind: Optional[str] = None, strike: Optional[float] = None, - spec: Optional[pycore.PyContractSpecification] = None, + spec: Optional[PyContractSpecification] = None, ): - sec = pycore.PySecurity() + sec = PySecurity() sec.set_venue(venue) sec.set_exchange_id(name) sec.set_name(name) @@ -307,17 +356,17 @@ def get_security( spec = get_contract_spec() sec.set_contract_spec(spec) if expiration and strike and option_kind: - opt = pycore.PyOption() + opt = PyOption() opt.set_expiration(expiration) opt.set_is_call(option_kind[0].lower() == "c") opt.set_strike(strike) sec.set_option(opt) elif expiration and not strike and not option_kind: - fut = pycore.PyFuture() + fut = PyFuture() fut.set_expiration(expiration) sec.set_future(fut) elif not expiration and not strike and not option_kind: - sec.set_spot(pycore.PySpot()) + sec.set_spot(PySpot()) else: raise ValueError("Invalid parameters (unable to deduce security type)") return sec diff --git a/src/lib_python_md/CMakeLists.txt b/src/lib_python_md/CMakeLists.txt deleted file mode 100644 index e1a881347..000000000 --- a/src/lib_python_md/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -cmake_minimum_required(VERSION 3.23) -project(python_md) -include(${CMAKE_SOURCE_DIR}/cmake/InitLibrary.cmake) -include(${CMAKE_SOURCE_DIR}/cmake/InitPythonModule.cmake) -include(${CMAKE_SOURCE_DIR}/cmake/InitPythonPlugin.cmake) - -# Find Python from the virtual environment -find_package(Python3 REQUIRED COMPONENTS Interpreter Development) -find_package(pybind11 REQUIRED) -# Shared lib (for c++ to link) -init_library(SHARED) -target_include_directories(${PROJECT_NAME} PUBLIC ${Python3_INCLUDE_DIRS}) -target_link_directories(${PROJECT_NAME} PUBLIC ${Python3_LIBRARY_DIRS}) -target_link_libraries(${PROJECT_NAME} PUBLIC spark python_core pybind11::embed - ${Python3_LIBRARIES}) -# CPython module (for .py files to import) -init_python_module() -# Py plugin (for sparkd to load at runtime) -init_python_plugin(PLUGIN_NAME python_md_providers) diff --git a/src/lib_python_md/Config.cmake.in b/src/lib_python_md/Config.cmake.in deleted file mode 100644 index ea8dbc30c..000000000 --- a/src/lib_python_md/Config.cmake.in +++ /dev/null @@ -1,12 +0,0 @@ -@PACKAGE_INIT@ - -include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") -include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@_moduleTargets.cmake") -include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@_providersTargets.cmake") - -find_dependency(Python3 REQUIRED COMPONENTS Interpreter Development) -find_dependency(pybind11) - -check_required_components(@PROJECT_NAME@) -check_required_components(@PROJECT_NAME@_module) -check_required_components(@PROJECT_NAME@_providers) diff --git a/src/lib_python_md/README.md b/src/lib_python_md/README.md deleted file mode 100644 index 40e672e04..000000000 --- a/src/lib_python_md/README.md +++ /dev/null @@ -1,87 +0,0 @@ - - -[pybind11]: https://github.com/pybind/pybind11 -[lib_spark]: ../lib_spark/README.md - -# lib_python_md - -This library binds c++ market data providers to python modules -which can be loaded at runtime. - -## **Dependencies** - -* **[Pybind11][pybind11]** -* **[lib_spark][lib_spark]** - -## Notes - -* Because this is *python*, market data enums (SecurityType, Venue, Side, etc.) are represented as strings - * Parsing is **NOT** case sensitive (though it is conventional to use `UPPER_CASE` strings) - * Type name prefixes (SECURITY_TYPE_, VENUE_, SIDE_ etc.) are **NOT** required - * ***Having said that,*** we recommend UPPER_CASE strings without the type prefix. Here are a few examples: - - | Enum | Examples | - |------|---------| - | **SecurityType** | "SPOT", "FUTURE", "OPTION" | - | **Venue** | "BYBIT", "GATEIO", "DERIBIT", "COINBASE" | - | **Side** | "BUY", "SELL" | - -* All .py files in the `python_md_providers` dir are copied to a special plugins directory - at build time, so they may be loaded at runtime. -* For intellisense, create symlinks to the *.so* and *.pyi* files in the `python_md_providers` dir - > *These files are excluded from source control via .gitignore, so don't worry about accidentally checking them in* -```bash -# Run in src/lib_python_md/python_md_providers -ln -s ../../../build/sparkd-debug-clang/src/lib_python_md/stubs/python_md_module.pyi . -ln -s ../../../build/sparkd-debug-clang/src/lib_python_md/python_md_module.cpython-310-x86_64-linux-gnu.so . -``` - -## Example -```python -import time -import python_md_module as pymd - - -class MdProviderExample(pymd.MdProvider): - def __init__(self): - super(MdProviderExample, self).__init__() - pass - - def start(self, sink: pymd.MdSink): - print("Starting MdProviderExample") - # An actual implementation would subscribe to market data here. - # This is just an example to show you how to pass market data - # back to the c++ Spark api. - - def add_dummy_security(): - # This example populates a dummy security for demonstrative purposes. - # Of course, an actual implementation would populate the security list - # with data received from the exchange (or some other provider). - sec = pymd.PySecurity() - sec.set_venue("GLBX") - sec.set_name("ESH8") - sec.set_asset("ES") - fut = pymd.PyFuture() - fut.set_expiration("2025-03-28") - sec.set_future(fut) - # Note: the example only adds 1 sec, but batch adds are supported. - sec_list = pymd.PySecurityList() - sec_list.append(sec) - sink.on_securities(sec_list) - - def add_dummy_trade(): - trade = pymd.PyTrade() - trade.set_venue("GLBX") - trade.set_security_type("FUTURE") - trade.set_exchange_security_id("ESH8") - trade.set_price(5684.25) - trade.set_quantity(1) - trade.set_aggressor_side("BUY") - trade.set_timestamp(time.time_ns()) - - add_dummy_security() - add_dummy_trade() - - def stop(self): - print("Stopping MdProviderExample") -``` diff --git a/src/lib_python_md/python_md_providers/binance_test.py b/src/lib_python_md/python_md_providers/binance_test.py deleted file mode 100644 index d19fc32b9..000000000 --- a/src/lib_python_md/python_md_providers/binance_test.py +++ /dev/null @@ -1,49 +0,0 @@ -import json -from typing import Optional -from pathlib import Path - - -test_data_dir = Path("/").joinpath( - "home", - "bts", - ".local", - "share", - "bts", - "sparkd", - "data", - "test_data", - "binance", -) - - -def load_securities() -> list: - with open(test_data_dir.joinpath("exch_info.json")) as f: - return json.load(f)["symbols"] - return [] - - -def load_books() -> list: - with open(test_data_dir.joinpath("books.json")) as f: - return json.load(f)["books"] - return [] - - -if __name__ == "__main__": - raw_secs = load_securities() - for s in raw_secs: - obj = { - "venue": "Binance", - "name": s["symbol"], - "asset": s["baseAsset"], - "group": s["baseAsset"], - "currency": s["quoteAsset"], - } - print(json.dumps(obj)) - raw_books = load_books() - for b in raw_books: - obj = { - "name": b["name"], - "bids": b["bids"], - "asks": b["asks"], - } - print(json.dumps(obj)) diff --git a/src/lib_python_md/python_md_providers/binance_test_md_provider.py b/src/lib_python_md/python_md_providers/binance_test_md_provider.py deleted file mode 100644 index 7ce058dc8..000000000 --- a/src/lib_python_md/python_md_providers/binance_test_md_provider.py +++ /dev/null @@ -1,161 +0,0 @@ -import python_md_module as pymd -import json -from typing import Optional -from pathlib import Path - - -test_data_dir = Path("/").joinpath( - "home", - "bts", - ".local", - "share", - "bts", - "sparkd", - "data", - "test_data", - "binance", -) - - -def load_securities() -> list: - with open(test_data_dir.joinpath("exch_info.json")) as f: - return json.load(f)["symbols"] - return [] - - -def load_books() -> list: - with open(test_data_dir.joinpath("books.json")) as f: - return json.load(f)["books"] - return [] - - -def _get_security( - venue: str, - name: str, - asset: str, - group: str, - currency: str, - expiration: Optional[str] = None, - option_kind: Optional[str] = None, - strike: Optional[float] = None, -): - sec = pymd.PySecurity() - sec.set_venue(venue) - sec.set_exchange_id(name) - sec.set_name(name) - sec.set_currency(currency) - sec.set_asset(asset) - sec.set_group(group) - is_spot = expiration is None and option_kind is None and strike is None - is_future = expiration is not None and option_kind is None and strike is None - is_option = ( - expiration is not None and option_kind is not None and strike is not None - ) - if is_spot: - sec.set_spot(pymd.PySpot()) - elif is_future: - fut = pymd.PyFuture() - fut.set_expiration(expiration) - sec.set_future(fut) - elif is_option: - opt = pymd.PyOption() - opt.set_expiration(expiration) - opt.set_is_call(option_kind.lower().startswith("c")) - opt.set_strike(strike) - sec.set_option(opt) - else: - raise ValueError("Invalid parameters (unable to deduce security type)") - return sec - - -def _get_book_side(levels: list[(float, float)], is_bid_side: bool): - book_side = pymd.PyBookSide() - book_side.set_is_ascending(not is_bid_side) - for price, qty in levels: - book_level = pymd.PyBookLevel() - book_level.set_price(price) - book_level.set_quantity(qty) - book_side.set_level(book_level) - return book_side - - -def _get_trade( - venue: str, - security_type: str, - exchange_security_id: str, - price: float, - quantity: float, - aggressor_side: str, -): - trade = pymd.PyTrade() - trade.set_venue(venue) - trade.set_security_type(security_type) - trade.set_exchange_security_id(exchange_security_id) - trade.set_price(price) - trade.set_quantity(quantity) - trade.set_aggressor_side(aggressor_side) - return trade - - -class BinanceTestMdProvider(pymd.MdProvider): - def __init__(self): - super(BinanceTestMdProvider, self).__init__() - pass - - def start(self, sink: pymd.MdSink): - # start subscriptions, bind callbacks etc. - def add_securities(): - sec_list = pymd.PySecurityList() - raw_secs = load_securities() - for s in raw_secs: - sec_list.append( - _get_security( - venue="Binance", - name=s["symbol"], - asset=s["baseAsset"], - group=s["baseAsset"], - currency=s["quoteAsset"], - ) - ) - sink.on_securities(sec_list) - - def add_books(): - books = load_books() - for b in books: - bids = _get_book_side( - [(lvl["price"], lvl["quantity"]) for lvl in b["bids"]], - is_bid_side=True, - ) - asks = _get_book_side( - [(lvl["price"], lvl["quantity"]) for lvl in b["asks"]], - is_bid_side=False, - ) - book = pymd.PyBook() - book.set_venue("Binance") - book.set_security_type("SPOT") - book.set_exchange_security_id(b["name"]) - book.set_bids(bids) - book.set_asks(asks) - # pass book to sink - book_list = pymd.PyBookList() - book_list.append(book) - sink.on_books(book_list) - - add_securities() - add_books() - - def stop(self): - print("Stopping python DummyMdProvider") - - -if __name__ == "__main__": - raw_secs = load_securities() - for s in raw_secs: - obj = { - "venue": "Binance", - "name": s["symbol"], - "asset": s["baseAsset"], - "group": s["baseAsset"], - "currency": s["quoteAsset"], - } - print(json.dump(obj)) diff --git a/src/lib_python_md/python_md_providers/bybit_conf.py b/src/lib_python_md/python_md_providers/bybit_conf.py deleted file mode 100644 index 6c195db21..000000000 --- a/src/lib_python_md/python_md_providers/bybit_conf.py +++ /dev/null @@ -1,484 +0,0 @@ -# Bybit API configuration -BYBIT_API_KEY = 'api_key' -BYBIT_API_SECRET = 'api_secret' -BYBIT_TESTNET = True # Set to False for mainnet -WS_URL = "wss://stream-testnet.bybit.com/v5/public/linear" - -CONTRACTS = ["BTCUSDT", "ETHUSDT", "BNBUSDT", "XRPUSDT", "ADAUSDT", "PRIMEUSDT"] - - -# CONTRACTS = [ -# '1000000BABYDOGEUSDT', -# '1000000CHEEMSUSDT', -# '1000000MOGUSDT', -# '1000000PEIPEIUSDT', -# '10000COQUSDT', -# '10000ELONUSDT', -# '10000LADYSUSDT', -# '10000QUBICUSDT', -# '10000SATSUSDT', -# '10000WENUSDT', -# '10000WHYUSDT', -# '1000APUUSDT', -# '1000BONKPERP', -# '1000BONKUSDT', -# '1000BTTUSDT', -# '1000CATSUSDT', -# '1000CATUSDT', -# '1000FLOKIUSDT', -# '1000LUNCUSDT', -# '1000MUMUUSDT', -# '1000NEIROCTOUSDT', -# '1000PEPEPERP', -# '1000PEPEUSDT', -# '1000RATSUSDT', -# '1000TOSHIUSDT', -# '1000TURBOUSDT', -# '1000XECUSDT', -# '1000XUSDT', -# '1INCHUSDT', -# 'A8USDT', -# 'AAVEPERP', -# 'AAVEUSDT', -# 'ACEUSDT', -# 'ACHUSDT', -# 'ACTUSDT', -# 'ACXUSDT', -# 'ADAUSDT', -# 'AERGOUSDT', -# 'AEROUSDT', -# 'AEVOPERP', -# 'AEVOUSDT', -# 'AGIUSDT', -# 'AGLDUSDT', -# 'AI16ZUSDT', -# 'AIOZUSDT', -# 'AIUSDT', -# 'AIXBTUSDT', -# 'AKTUSDT', -# 'ALCHUSDT', -# 'ALEOUSDT', -# 'ALICEUSDT', -# 'ALPHAUSDT', -# 'ALTUSDT', -# 'ALUUSDT', -# 'ANIMEUSDT', -# 'ANKRUSDT', -# 'APEUSDT', -# 'APEXUSDT', -# 'API3USDT', -# 'APTUSDT', -# 'ARBPERP', -# 'ARCUSDT', -# 'ARKMUSDT', -# 'ARKUSDT', -# 'ARPAUSDT', -# 'ARUSDT', -# 'ASTRUSDT', -# 'ATAUSDT', -# 'ATHUSDT', -# 'AUCTIONUSDT', -# 'AUDIOUSDT', -# 'AVAAIUSDT', -# 'AVAILUSDT', -# 'AVAUSDT', -# 'AVLUSDT', -# 'AXLUSDT', -# 'AXSUSDT', -# 'B3USDT', -# 'BABYUSDT', -# 'BADGERUSDT', -# 'BAKEUSDT', -# 'BALUSDT', -# 'BANANAS31USDT', -# 'BANANAUSDT', -# 'BANDUSDT', -# 'BANKUSDT', -# 'BANUSDT', -# 'BATUSDT', -# 'BBUSDT', -# 'BCHPERP', -# 'BCHUSDT', -# 'BEAMUSDT', -# 'BELUSDT', -# 'BERAUSDT', -# 'BIGTIMEUSDT', -# 'BIOUSDT', -# 'BLASTUSDT', -# 'BLURUSDT', -# 'BMTUSDT', -# 'BNBUSDT', -# 'BNTUSDT', -# 'BOBAUSDT', -# 'BOMEUSDT', -# 'BRETTUSDT', -# 'BROCCOLIUSDT', -# 'BRUSDT', -# 'BSVUSDT', -# 'BTCPERP', -# 'BTCUSDT', -# 'C98USDT', -# 'CAKEUSDT', -# 'CARVUSDT', -# 'CATIUSDT', -# 'CELOUSDT', -# 'CELRUSDT', -# 'CETUSUSDT', -# 'CFXUSDT', -# 'CGPTUSDT', -# 'CHESSUSDT', -# 'CHILLGUYUSDT', -# 'CHZUSDT', -# 'CLANKERUSDT', -# 'CLOUDUSDT', -# 'COMPUSDT', -# 'COOKIEUSDT', -# 'COOKUSDT', -# 'COREUSDT', -# 'COSUSDT', -# 'COTIUSDT', -# 'COWUSDT', -# 'CPOOLUSDT', -# 'CROUSDT', -# 'CRVPERP', -# 'CRVUSDT', -# 'CTKUSDT', -# 'CTSIUSDT', -# 'CVCUSDT', -# 'CVXUSDT', -# 'DARKUSDT', -# 'DASHUSDT', -# 'DATAUSDT', -# 'DBRUSDT', -# 'DEEPUSDT', -# 'DEGENUSDT', -# 'DENTUSDT', -# 'DEXEUSDT', -# 'DGBUSDT', -# 'DODOUSDT', -# 'DOGEPERP', -# 'DOGEUSDT', -# 'DOGSUSDT', -# 'DOGUSDT', -# 'DOODUSDT', -# 'DOTPERP', -# 'DOTUSDT', -# 'DRIFTUSDT', -# 'DUCKUSDT', -# 'DUSKUSDT', -# 'DYDXUSDT', -# 'DYMUSDT', -# 'EDUUSDT', -# 'EGLDUSDT', -# 'ELXUSDT', -# 'ENAPERP', -# 'ENAUSDT', -# 'ENJUSDT', -# 'ENSUSDT', -# 'EOSUSDT', -# 'EPICUSDT', -# 'EPTUSDT', -# 'ETCUSDT', -# 'ETHBTCUSDT', -# 'ETHFIPERP', -# 'ETHFIUSDT', -# 'ETHPERP', -# 'ETHUSDT', -# 'ETHWUSDT', -# 'FARTCOINUSDT', -# 'FBUSDT', -# 'FHEUSDT', -# 'FIDAUSDT', -# 'FILUSDT', -# 'FIOUSDT', -# 'FLMUSDT', -# 'FLOCKUSDT', -# 'FLOWUSDT', -# 'FLRUSDT', -# 'FLUXUSDT', -# 'FORMUSDT', -# 'FORTHUSDT', -# 'FOXYUSDT', -# 'FTNUSDT', -# 'FUELUSDT', -# 'FUSDT', -# 'FWOGUSDT', -# 'FXSUSDT', -# 'GALAUSDT', -# 'GASUSDT', -# 'GIGAUSDT', -# 'GLMRUSDT', -# 'GLMUSDT', -# 'GMTUSDT', -# 'GMXUSDT', -# 'GNOUSDT', -# 'GOATUSDT', -# 'GODSUSDT', -# 'GOMININGUSDT', -# 'GORKUSDT', -# 'GPSUSDT', -# 'GRIFFAINUSDT', -# 'GRTUSDT', -# 'GUNUSDT', -# 'GUSDT', -# 'HAEDALUSDT', -# 'HBARUSDT', -# 'HEIUSDT', -# 'HFTUSDT', -# 'HIFIUSDT', -# 'HIGHUSDT', -# 'HIPPOUSDT', -# 'HIVEUSDT', -# 'HMSTRUSDT', -# 'HNTUSDT', -# 'HOOKUSDT', -# 'HOTUSDT', -# 'HPOS10IUSDT', -# 'HYPEPERP', -# 'HYPERUSDT', -# 'HYPEUSDT', -# 'ICPUSDT', -# 'ICXUSDT', -# 'IDEXUSDT', -# 'IDUSDT', -# 'ILVUSDT', -# 'IMXUSDT', -# 'INITUSDT', -# 'INJUSDT', -# 'IOSTUSDT', -# 'IOTAUSDT', -# 'IOTXUSDT', -# 'IOUSDT', -# 'IPUSDT', -# 'JASMYUSDT', -# 'JELLYJELLYUSDT', -# 'JOEUSDT', -# 'JSTUSDT', -# 'JTOUSDT', -# 'JUPUSDT', -# 'JUSDT', -# 'KAIAUSDT', -# 'KAITOUSDT', -# 'KASUSDT', -# 'KAVAUSDT', -# 'KDAUSDT', -# 'KERNELUSDT', -# 'KMNOUSDT', -# 'KNCUSDT', -# 'KOMAUSDT', -# 'KSMUSDT', -# 'L3USDT', -# 'LAUNCHCOINUSDT', -# 'LDOUSDT', -# 'LEVERUSDT', -# 'LINKPERP', -# 'LINKUSDT', -# 'LISTAUSDT', -# 'LOOKSUSDT', -# 'LPTUSDT', -# 'LQTYUSDT', -# 'LRCUSDT', -# 'LSKUSDT', -# 'LTCPERP', -# 'LUMIAUSDT', -# 'LUNA2USDT', -# 'MAGICUSDT', -# 'MAJORUSDT', -# 'MANAUSDT', -# 'MANTAUSDT', -# 'MASAUSDT', -# 'MASKUSDT', -# 'MAVIAUSDT', -# 'MAVUSDT', -# 'MBLUSDT', -# 'MBOXUSDT', -# 'MDTUSDT', -# 'MELANIAUSDT', -# 'MEMEUSDT', -# 'MERLUSDT', -# 'METISUSDT', -# 'MEUSDT', -# 'MEWUSDT', -# 'MICHIUSDT', -# 'MILKUSDT', -# 'MINAUSDT', -# 'MKRUSDT', -# 'MLNUSDT', -# 'MNTPERP', -# 'MNTUSDT', -# 'MOBILEUSDT', -# 'MOCAUSDT', -# 'MOODENGUSDT', -# 'MORPHOUSDT', -# # 'MOTHER1USDT', -# # 'MOTHER2USDT', -# # 'MOTHER3USDT', -# 'MOVEUSDT', -# 'MOVRUSDT', -# 'MTLUSDT', -# 'MUBARAKUSDT', -# 'MVLUSDT', -# 'MYRIAUSDT', -# 'NCUSDT', -# 'NEARUSDT', -# 'NEIROETHUSDT', -# 'NEOUSDT', -# 'NFPUSDT', -# 'NILUSDT', -# 'NKNUSDT', -# 'NMRUSDT', -# 'NOTPERP', -# 'NOTUSDT', -# 'NSUSDT', -# 'NTRNUSDT', -# 'NXPCUSDT', -# 'OBOLUSDT', -# 'OBTUSDT', -# 'OGUSDT', -# # 'OKBUSDT', -# 'OLUSDT', -# 'OMGUSDT', -# 'OMUSDT', -# 'ONDOPERP', -# 'ONDOUSDT', -# 'ONEUSDT', -# 'ONGUSDT', -# 'ONTUSDT', -# 'OPUSDT', -# 'ORBSUSDT', -# 'ORCAUSDT', -# 'ORDERUSDT', -# 'ORDIPERP', -# 'ORDIUSDT', -# 'OSMOUSDT', -# 'PARTIUSDT', -# 'PAXGUSDT', -# 'PEAQUSDT', -# 'PENDLEUSDT', -# 'PEOPLEUSDT', -# # 'PERPUSDT', -# 'PHAUSDT', -# 'PHBUSDT', -# 'PIPPINUSDT', -# 'PIXELUSDT', -# 'PLUMEUSDT', -# 'PNUTUSDT', -# 'POLPERP', -# 'POLUSDT', -# 'POLYXUSDT', -# 'PONKEUSDT', -# 'POPCATPERP', -# 'POPCATUSDT', -# 'PORTALUSDT', -# 'POWRUSDT', -# 'PRAIUSDT', -# 'PRCLUSDT', -# # 'PRE05EOSUSDT', -# # 'PRE08EOSUSDT', -# 'PRIMEUSDT', -# 'PROMPTUSDT', -# 'PROMUSDT', -# 'PUFFERUSDT', -# 'PUMPUSDT', -# 'PUNDIXUSDT', -# 'PYRUSDT', -# 'PYTHUSDT', -# 'QIUSDT', -# 'QNTUSDT', -# 'QTUMUSDT', -# 'QUICKUSDT', -# 'RADUSDT', -# 'RAREUSDT', -# 'RAYDIUMUSDT', -# 'RAYUSDT', -# 'RDNTUSDT', -# 'REDUSDT', -# 'RENDERUSDT', -# 'REXUSDT', -# 'REZUSDT', -# 'RFCUSDT', -# 'RIFUSDT', -# 'RLCUSDT', -# 'ROAMUSDT', -# 'RONINUSDT', -# 'ROSEUSDT', -# 'RPLUSDT', -# 'RSRUSDT', -# 'RUNEUSDT', -# 'SAGAUSDT', -# 'SANDUSDT', -# 'SCAUSDT', -# 'SDUSDT', -# 'SENDUSDT', -# 'SERAPHUSDT', -# 'SFPUSDT', -# 'SHELLUSDT', -# 'SHIB1000PERP', -# 'SHIB1000USDT', -# 'SIGNUSDT', -# 'SIRENUSDT', -# 'SKLUSDT', -# 'SKYAIUSDT', -# 'SLERFUSDT', -# 'SLFUSDT', -# 'SLPUSDT', -# 'SNTUSDT', -# 'SNXUSDT', -# 'SOLAYERUSDT', -# 'SOLOUSDT', -# 'SOLPERP', -# 'SOLUSDT', -# 'SOLVUSDT', -# 'SONICUSDT', -# 'SPECUSDT', -# 'SPXUSDT', -# 'SSVUSDT', -# 'STEEMUSDT', -# 'STGUSDT', -# 'STORJUSDT', -# 'STOUSDT', -# 'STRKPERP', -# 'STRKUSDT', -# 'SUIPERP', -# 'SUIUSDT', -# 'SUNDOGUSDT', -# 'SUNUSDT', -# 'SUPERUSDT', -# 'SUSDT', -# 'SUSHIUSDT', -# 'SWARMSUSDT', -# 'SWEATUSDT', -# 'SWELLUSDT', -# 'SXPUSDT', -# 'SYNUSDT', -# 'SYRUPUSDT', -# 'SYSUSDT', -# 'TAIKOUSDT', -# 'TAIUSDT', -# 'TAOUSDT', -# 'THETAUSDT', -# 'THEUSDT', -# 'TIAPERP', -# 'TIAUSDT', -# 'TLMUSDT', -# 'TOKENUSDT', -# 'TONPERP', -# 'TRBUSDT', -# 'TRUMPPERP', -# 'TRUMPUSDT', -# 'TRUUSDT', -# 'TRXUSDT', -# 'TSTBSCUSDT', -# 'TUSDT', -# 'TUTUSDT', -# 'TWTUSDT', -# 'UMAUSDT', -# 'UNIPERP', -# 'UNIUSDT', -# 'USDCUSDT', -# 'USDEUSDT', -# 'USTCUSDT', -# 'USUALUSDT', -# 'UXLINKUSDT', -# 'VANAUSDT' -# ] \ No newline at end of file diff --git a/src/lib_python_md/python_md_providers/pyrightconfig.json b/src/lib_python_md/python_md_providers/pyrightconfig.json deleted file mode 100644 index d92bc4f70..000000000 --- a/src/lib_python_md/python_md_providers/pyrightconfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "include": [ - ".", - ], - "exclude": [ - "**/__pycache__" - ], - "reportMissingImports": true, - "reportMissingTypeStubs": false, - "stubPath": "../../../build/sparkd-debug-clang/src/lib_python_md/stubs", - "extraPaths": [ - "../../lib_python_core/python_core_plugins", - "../../../build/sparkd-debug-clang/src/lib_python_core/stubs", - ], - "pythonVersion": "3.10", - "pythonPlatform": "Linux", - "typeCheckingMode": "basic", - "venvPath": "../../../", - "venv": ".venv" -} diff --git a/src/lib_spark/spark/orderman/account_name.h b/src/lib_spark/spark/orderman/account_name.h index 3f45dcdd7..923f85bb2 100644 --- a/src/lib_spark/spark/orderman/account_name.h +++ b/src/lib_spark/spark/orderman/account_name.h @@ -9,6 +9,14 @@ namespace bts::orderman { using AccountName = utils::ValueWrapper>; +inline void to_json(nlohmann::json &j, const AccountName &acc) { + j = acc.value(); +} + +inline void from_json(const nlohmann::json &j, AccountName &acc) { + j.get_to(acc.value()); +} + } // namespace bts::orderman FMT_IMPL(bts::orderman::AccountName, a, "{}", a->to_string_view());