From e9bcdd0c65ceaea51fa5ef271eb2db4098dd77d8 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 12 Nov 2024 10:35:51 -0500 Subject: [PATCH 01/30] fix build: Update Newtonsoft.Json to 13.0.3 in Spark.csproj --- src/client/spark/Spark.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/Spark.csproj b/src/client/spark/Spark.csproj index 5ee8ed0dc..a1203d145 100644 --- a/src/client/spark/Spark.csproj +++ b/src/client/spark/Spark.csproj @@ -37,7 +37,7 @@ - + From 5517c5872496a4b96528a6d4a0aaebae9a08e3b3 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 12 Nov 2024 11:36:44 -0500 Subject: [PATCH 02/30] fix (ui): Display trade time w/ microsecond precision Was only showing milliseconds before --- src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs | 2 +- src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs index ffca5e43f..78d90181d 100644 --- a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs +++ b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs @@ -138,7 +138,7 @@ _dispose = Disposable.Create(resources.ToArray()); static string fmt_sec(TimeAndSalesRowVM row) => row.security.ToString(); - static string fmt_time(TimeAndSalesRowVM row) => row.time.ToString("h:mm:ss tt"); + static string fmt_time(TimeAndSalesRowVM row) => row.time.ToString("h:mm:ss.ffffff"); static string fmt_price(TimeAndSalesRowVM row) => row.security.spec_or_default().format(row.price); static string fmt_aggressor(TimeAndSalesRowVM row) => row.aggressor_side.ToString(); static string fmt_volume(TimeAndSalesRowVM row) => new TickFactors(1, 100).format(row.volume); diff --git a/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs b/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs index 75849d9b0..d412d2b86 100644 --- a/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs +++ b/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs @@ -46,7 +46,7 @@ Filled = 56, Symbol = 60, Time = 112, - TimeOfDay = 64, + TimeOfDay = 84, TimeInForce = 47, User = 53, Aggressor = 80, From eb8c97a1b7295b34e9d7ad90d7ede55889b5f008 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 12 Nov 2024 16:01:01 -0500 Subject: [PATCH 03/30] fix: Detect integer ticks of all sizes (not just 1) Also don't divide by zero! Bugs (fixed): * Spark crashes when opening a ladder with integer tick size of 5. This was first observed with TAIFEX futures. Cause: The ladder throws a DivideByZero exception when calculating the number of ticks per dollar. It isn't caught until it bubbles up to the scope in which 'Dispatcher.Run' was called. The catch handler then deliberately assert crashes. --- src/client/spark_api/data/ticks/TickExtensions.cs | 12 ++++++++++-- src/client/spark_api/data/ticks/TickUtils.cs | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/client/spark_api/data/ticks/TickExtensions.cs b/src/client/spark_api/data/ticks/TickExtensions.cs index d6a88ec4b..ad564f133 100644 --- a/src/client/spark_api/data/ticks/TickExtensions.cs +++ b/src/client/spark_api/data/ticks/TickExtensions.cs @@ -69,8 +69,16 @@ namespace mbts { public static bool is_valid(this RegionalTickFactors regional_tick) => regional_tick.low_tick.is_valid() && regional_tick.high_tick.is_valid(); - public static bool is_integer(this TickFactors tick) => - tick.tick_unit == tick.tick_base; + public static bool is_integer(this TickFactors tick) { + if (tick.tick_unit == tick.tick_base) { + return true; + } + if (tick.tick_unit > tick.tick_base && tick.tick_base != 0) { + var units_per_base = (int)(tick.tick_unit / tick.tick_base); + return units_per_base * tick.tick_base == tick.tick_unit; + } + return false; + } public static TickFactors tick_factors(this RegionalTickFactors regional_tick, double value) => value > regional_tick.low_tick_inclusive_end ? diff --git a/src/client/spark_api/data/ticks/TickUtils.cs b/src/client/spark_api/data/ticks/TickUtils.cs index e3c783cce..94bae68a7 100644 --- a/src/client/spark_api/data/ticks/TickUtils.cs +++ b/src/client/spark_api/data/ticks/TickUtils.cs @@ -295,6 +295,9 @@ namespace mbts.spark { var ticks_per_dollar = (uint)(tf.low_tick.tick_base / tf.low_tick.tick_unit); // if ticks_per_row is greater than ticks_per_dollar, round to a multiple of ticks_per_dollar if (ticks_per_row > ticks_per_dollar) { + if (ticks_per_dollar == 0) { + return null; + } return (uint)(ticks_per_dollar * Math.Floor((double)(ticks_per_row / ticks_per_dollar))); } else if (ticks_per_row == ticks_per_dollar) { return ticks_per_row; From 826bbf76a9263159e220608b4e80ad49f5bc404e Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 02:53:19 +0000 Subject: [PATCH 04/30] Add: load_reference_data config for databento --- .../api/md/DatabentoMarketDataConfig.proto | 1 + .../sparkd/md/cboe/CboeMarketDataHandler.cpp | 3 ++- .../md/databento/DatabentoDataHandler.cpp | 23 +++++++++---------- .../md/databento/DatabentoDataHandler.h | 1 + 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/lib/api/md/DatabentoMarketDataConfig.proto b/src/lib/api/md/DatabentoMarketDataConfig.proto index cea25526c..184df7e46 100644 --- a/src/lib/api/md/DatabentoMarketDataConfig.proto +++ b/src/lib/api/md/DatabentoMarketDataConfig.proto @@ -14,4 +14,5 @@ message DatabentoMarketDataConfig { string key = 60; string dataset = 70; bool filter_securities_one_year= 80; + bool load_reference_data = 90; } diff --git a/src/server/sparkd/md/cboe/CboeMarketDataHandler.cpp b/src/server/sparkd/md/cboe/CboeMarketDataHandler.cpp index ee818d5a0..47071cb1f 100644 --- a/src/server/sparkd/md/cboe/CboeMarketDataHandler.cpp +++ b/src/server/sparkd/md/cboe/CboeMarketDataHandler.cpp @@ -30,7 +30,8 @@ void CboeMarketDatahandler::run( base_path + config.databento_settings().bbo_file_name(), config.databento_settings().key(), config.databento_settings().dataset(), - config.databento_settings().filter_securities_one_year()); + config.databento_settings().filter_securities_one_year(), + config.databento_settings().load_reference_data()); } } // namespace bts::md::cboe \ No newline at end of file diff --git a/src/server/sparkd/md/databento/DatabentoDataHandler.cpp b/src/server/sparkd/md/databento/DatabentoDataHandler.cpp index a546c4a03..3b61336a4 100644 --- a/src/server/sparkd/md/databento/DatabentoDataHandler.cpp +++ b/src/server/sparkd/md/databento/DatabentoDataHandler.cpp @@ -25,6 +25,7 @@ #include #include #include +#include namespace bts::md::DatabentoHandler { @@ -49,7 +50,8 @@ void DatabentoDatahandler::run( const std::string &key, const std::string &dataset, const bool filter_securities, - const std::chrono::duration sleep_duration) { + const bool load_reference_data, + const std::chrono::duration sleep_duration ) { auto sleep_duration_updated = sleep_duration; @@ -181,19 +183,16 @@ void DatabentoDatahandler::run( {} }; //hardcoding for now will update later + std::vector symbols; switch (venue) { - case api::domain::VENUE_CBOE: - client.Subscribe({"SPX.OPT","SPXW.OPT"}, databento::Schema::Definition, databento::SType::Parent,databento::UnixNanos{}); - client.Subscribe({"SPX.OPT","SPXW.OPT"}, databento::Schema::Mbp1, databento::SType::Parent); - break; - case api::domain::VENUE_GLBX: - client.Subscribe({"ES.FUT"}, databento::Schema::Definition, databento::SType::Parent); - client.Subscribe({"ES.FUT"}, databento::Schema::Mbp1, databento::SType::Parent); - break; - default: - bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW"); - + case api::domain::VENUE_CBOE: symbols= {"SPX.OPT","SPXW.OPT"}; break; + case api::domain::VENUE_GLBX: symbols= {"ES.FUT"}; break; + default:bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW"); } + if(load_reference_data) + client.Subscribe(symbols, databento::Schema::Definition, databento::SType::Parent,databento::UnixNanos{}); + else client.Subscribe(symbols, databento::Schema::Definition, databento::SType::Parent); + client.Subscribe(symbols, databento::Schema::Mbp1, databento::SType::Parent); client.Start(); while (true) { try { diff --git a/src/server/sparkd/md/databento/DatabentoDataHandler.h b/src/server/sparkd/md/databento/DatabentoDataHandler.h index 45abb5dc7..00626b4c9 100644 --- a/src/server/sparkd/md/databento/DatabentoDataHandler.h +++ b/src/server/sparkd/md/databento/DatabentoDataHandler.h @@ -15,6 +15,7 @@ class DatabentoDatahandler { const std::string &dataset="", const std::string& key = "db-Q4eCbTQfrAHaG6sAcuFUYB9YqkF6j", const bool filter_securities=false, + const bool load_reference_data = true, const std::chrono::duration sleep_duration = 0.04s); }; } // namespace bts::md::databento \ No newline at end of file From 30822fb1f15b9f1498a0ef906b0262259dc1c9be Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 03:09:07 +0000 Subject: [PATCH 05/30] Fix: move start glbx onix to seperate thread --- src/server/sparkd/md/MarketDataGateway.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/server/sparkd/md/MarketDataGateway.cpp b/src/server/sparkd/md/MarketDataGateway.cpp index 48c6bea01..025cb05a5 100644 --- a/src/server/sparkd/md/MarketDataGateway.cpp +++ b/src/server/sparkd/md/MarketDataGateway.cpp @@ -295,7 +295,10 @@ void MarketDataGateway::_start_onix_glbx( "MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. " "cfg:\n{}", os::get_thread_id(), os::get_thread_name(), cfg.DebugString()); - bts::md::run_onixs(); + bts::threading::make_thread("Onix CME", [&]() { + bts::md::run_onixs(); + }).detach(); + } @@ -400,7 +403,6 @@ void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg){ cboe); }).detach(); - UNUSED(cboe); } } From 8604d5672ce9fa42fa3d26a8ea094a9745f7ad9b Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 03:11:54 +0000 Subject: [PATCH 06/30] ADD: Comments for CME execution for later --- src/lib/api/service_helpers/AdminServiceUtils.cpp | 3 +++ src/lib/orderman/orders/GLBXNativeOrder.cpp | 8 ++++---- src/lib/orderman/orders/glbx/iLink2OrderEncoder.cpp | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/lib/api/service_helpers/AdminServiceUtils.cpp b/src/lib/api/service_helpers/AdminServiceUtils.cpp index e2dcc4bcf..bbcfa7f30 100644 --- a/src/lib/api/service_helpers/AdminServiceUtils.cpp +++ b/src/lib/api/service_helpers/AdminServiceUtils.cpp @@ -156,6 +156,8 @@ AdminServiceUtils::resolve_connection( const bts::api::admin::ExchangeConnection *rv = nullptr; switch (sec.exchange()) { case api::Venue::VENUE_GLBX: { + //CME EXECUTION NEEDS TO BE UPDATED, probably merge with VENUE_BINANCE_USDS_FUTURES + auto deployment_type = boost::to_lower_copy(config::get().type()); rv = resolve_cme_connection(data, account, sec); @@ -227,6 +229,7 @@ void AdminServiceUtils::fill_connections_from_config( connection.set_name(c.name()); switch (c.venue()) { + //CME EXECUTION NEEDS TO BE UPDATED, probably merge default case case bts::api::VENUE_GLBX: if (c.cme_market_segment_id() == 0) connection.mutable_cme_connection()->mutable_cgw_connection(); diff --git a/src/lib/orderman/orders/GLBXNativeOrder.cpp b/src/lib/orderman/orders/GLBXNativeOrder.cpp index da0537a20..e350c1190 100644 --- a/src/lib/orderman/orders/GLBXNativeOrder.cpp +++ b/src/lib/orderman/orders/GLBXNativeOrder.cpp @@ -478,7 +478,7 @@ OrderOpResult GLBXNativeOrder::on_activate( now.to_uint64(), (now - corid.app_arrival).nanoseconds()); // in sim mode, we dont need this, me->encoder->send_create(*me->services, *me->session_services); - if (me->in_sim_mode) { + if (me->in_sim_mode) { //CME EXECUTION NEEDS TO BE UPDATED CreateOrderMsg c_msg = {}; _create_create_msg(c_msg); me->services->queue_order_msg(*me->session_services, c_msg); @@ -579,7 +579,7 @@ OrderOpResult GLBXNativeOrder::on_update(api::orderman::Update const &update) { auto p = Price(update.price()); auto q = Quantity(update.quantity()); auto send_update_in_sim_mode = [&]() { - if (me->in_sim_mode) { + if (me->in_sim_mode) { //CME EXECUTION NEEDS TO BE UPDATED CancelReplaceOrderMsg cxl_repl_msg = {}; _create_create_msg(cxl_repl_msg.create_fields); Price new_p = Price(update.price()); @@ -799,7 +799,7 @@ OrderOpResult GLBXNativeOrder::on_cancel() { me->encoder->update_cancel_encoding(*me->services); me->encoder->send_cancel(*me->services, *me->session_services); // TODO: check for sim mode here - if (me->in_sim_mode) { + if (me->in_sim_mode) { //CME EXECUTION NEEDS TO BE UPDATED CancelOrderMsg cxl_msg = {}; cxl_msg.key = key(); LOG(OrderMan, Info, "GLBXNativeOrder queue_order_msg for cxl"); @@ -1381,7 +1381,7 @@ void GLBXNativeOrder::_create_create_msg(CreateOrderMsg &c) { bts::orderman::get_quantity(me->state, StateQuantityType::OrderQuantity); c.security = me->sec; c.side = me->definition.side(); - c.order_type = get_glbx_native_ord_type(me->definition); + c.order_type = get_glbx_native_ord_type(me->definition); //CME EXECUTION NEEDS TO BE UPDATED c.time_in_force = me->definition.time_in_force(); c.order_type = api::orderman::NATIVE_ORDER_TYPE_LIMIT; c.manual_order_indicator = me->definition.manual_order_indicator(); diff --git a/src/lib/orderman/orders/glbx/iLink2OrderEncoder.cpp b/src/lib/orderman/orders/glbx/iLink2OrderEncoder.cpp index 7bdbd303d..6e4aa74cb 100644 --- a/src/lib/orderman/orders/glbx/iLink2OrderEncoder.cpp +++ b/src/lib/orderman/orders/glbx/iLink2OrderEncoder.cpp @@ -277,6 +277,7 @@ void iLink2OrderEncoder::update_exchange_order_id(ExchangeOrderIDVar const &v) { auto oid = to_trim_str(std::get(v)); me->update_encoding.add(oid); me->cancel_encoding.add(oid); + //CME EXECUTION NEEDS TO BE UPDATED, can be commented out } void iLink2OrderEncoder::update_create_order_encoding( From f0eef447d3433db465d754a452c5bdd60adcb25b Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 03:18:04 +0000 Subject: [PATCH 07/30] Add: Exchange parameter added to EMNAtive Order --- .../orderman/order_factory/OrderFactory.cpp | 2 +- src/lib/orderman/orders/EMNativeOrder.cpp | 23 ++++++++++++++----- src/lib/orderman/orders/EMNativeOrder.h | 2 +- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/lib/orderman/order_factory/OrderFactory.cpp b/src/lib/orderman/order_factory/OrderFactory.cpp index 370299c5b..0812a2d5f 100644 --- a/src/lib/orderman/order_factory/OrderFactory.cpp +++ b/src/lib/orderman/order_factory/OrderFactory.cpp @@ -129,7 +129,7 @@ NativeOrderBase *OrderFactory::make_order(const api::Venue exchange) { return new ICENativeOrder(); else if (exchange == api::VENUE_GLBX || exchange == api::VENUE_SIM) return new GLBXNativeOrder(); - return new EMNativeOrder(); + return new EMNativeOrder(exchange); } NativeOrderBase *OrderFactory::make_native_order( diff --git a/src/lib/orderman/orders/EMNativeOrder.cpp b/src/lib/orderman/orders/EMNativeOrder.cpp index 9413b58f0..1b39808a2 100644 --- a/src/lib/orderman/orders/EMNativeOrder.cpp +++ b/src/lib/orderman/orders/EMNativeOrder.cpp @@ -52,10 +52,10 @@ struct InflightUpdateState { }; struct EMNativeOrder::PImpl { - PImpl() + PImpl(const api::Venue exchange) : instance_id(EMNativeOrder::count++), services(nullptr), session_services(), sec(nullptr), previous_working_position(0), - definition(), state(), exchange_order_id() {} + definition(), state(), exchange_order_id(), exchange(exchange) {} const int instance_id; bool inited = false; OrderServices *services; @@ -67,7 +67,7 @@ struct EMNativeOrder::PImpl { OrderDefinition definition; OrderState state; std::optional exchange_order_id; - + const api::Venue exchange; OrderServices::ExeReportSubID exe_report_sid; OrderServices::CancelRejectSubID cancel_reject_sid; // If a cancel is sent before the order placement is ack'ed this is set to @@ -82,7 +82,7 @@ struct EMNativeOrder::PImpl { RoutingRecord routing_record; }; -EMNativeOrder::EMNativeOrder() : me(new PImpl()) { +EMNativeOrder::EMNativeOrder(const api::Venue exchange) : me(new PImpl(exchange)) { LOG(OrderMan, Info, "EMNativeOrder::EMNativeOrder instance_id: {} t_id: {} t_name: {}", me->instance_id, os::get_thread_id(), os::get_application_name()); @@ -501,11 +501,22 @@ OrderOpResult EMNativeOrder::_on_activate( me->state.set_filled_quantity(0); me->state.set_last_price(0); me->state.set_avg_price(0); - me->session_services = &me->services->get_session_services( get_native_state(me->state).routing_record().session()); // TODO: just to get going - me->session_services = &me->services->get_session_services("deribit_1"); + std::string session_name; + switch (me->exchange) { + case api::VENUE_CBOE: + session_name = "cboe_1"; + break; + case api::VENUE_TAIFEX: + session_name="taifex_1"; + break; + default: + session_name ="deribit_1"; + + } + me->session_services = &me->services->get_session_services(session_name); // Checks for the following: // 1. Gateway connection diff --git a/src/lib/orderman/orders/EMNativeOrder.h b/src/lib/orderman/orders/EMNativeOrder.h index 4564fab95..3a77e21a2 100644 --- a/src/lib/orderman/orders/EMNativeOrder.h +++ b/src/lib/orderman/orders/EMNativeOrder.h @@ -19,7 +19,7 @@ class SessionServices; // Wrapper around Emumba's order class EMNativeOrder : public NativeOrderBase { public: - EMNativeOrder(); + EMNativeOrder(const api::Venue exchange = api::VENUE_DERIBIT); ~EMNativeOrder() override; api::orderman::OrderDefinition const &definition() const override; void on_init_new( From 2d8523ae67cdd29d9874f7e06e611a1b08cad56a Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 03:18:31 +0000 Subject: [PATCH 08/30] FIX: Order cancel now working in emnative order --- src/lib/orderman/orders/EMNativeOrder.cpp | 5 +++++ src/lib/orderman/orders/EMNativeOrder.h | 1 + 2 files changed, 6 insertions(+) diff --git a/src/lib/orderman/orders/EMNativeOrder.cpp b/src/lib/orderman/orders/EMNativeOrder.cpp index 1b39808a2..af01a0d29 100644 --- a/src/lib/orderman/orders/EMNativeOrder.cpp +++ b/src/lib/orderman/orders/EMNativeOrder.cpp @@ -5,6 +5,7 @@ #include "lib/config/protos/toggles.pb.h" #include "lib/domain/OrderStatus.h" #include "lib/domain_model/Security.h" +#include "lib/domain_model/Security.pb.h" #include "lib/domain_model/common.h" #include "lib/jetfix/codec/FIXFragment.h" #include "lib/jetfix/codec/FieldDefinitions.h" @@ -342,6 +343,10 @@ OrderOpResult EMNativeOrder::on_cancel() { case ORDER_STATUS_WORKING: { // TODO, CancelOrderMsg is not completely filled CancelOrderMsg cxl_msg = {}; + cxl_msg.side = me->definition.side(); + cxl_msg.security = me->sec; + // cxl_msg.exchange_order_id=me->exchange_order_id;//handle this + cxl_msg.routing_record = &me->routing_record; cxl_msg.key = key(); LOG(OrderMan, Info, "EMNativeOrder queue_order_msg for cxl"); me->services->queue_order_msg(*me->session_services, cxl_msg); diff --git a/src/lib/orderman/orders/EMNativeOrder.h b/src/lib/orderman/orders/EMNativeOrder.h index 3a77e21a2..55fb3b1ae 100644 --- a/src/lib/orderman/orders/EMNativeOrder.h +++ b/src/lib/orderman/orders/EMNativeOrder.h @@ -1,6 +1,7 @@ #pragma once #include "lib/api/services/OrderService.pb.h" +#include "lib/domain_model/Security.pb.h" #include "lib/domain_model/common.h" #include "lib/orderman/CreateOrderMsg.h" #include "lib/orderman/orders/NativeOrderBase.h" From 5b61b4fe69be089cbc74066a8ad20854fbf990dc Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 03:22:18 +0000 Subject: [PATCH 09/30] Add: Accounts for CBOE Execution, subaccounts dont work --- .../api/service_helpers/AdminServiceUtils.cpp | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/lib/api/service_helpers/AdminServiceUtils.cpp b/src/lib/api/service_helpers/AdminServiceUtils.cpp index bbcfa7f30..5db17b05f 100644 --- a/src/lib/api/service_helpers/AdminServiceUtils.cpp +++ b/src/lib/api/service_helpers/AdminServiceUtils.cpp @@ -239,10 +239,7 @@ void AdminServiceUtils::fill_connections_from_config( ->set_market_segment_id(c.cme_market_segment_id()); add_connection(admin_data, connection); break; - case bts::api::VENUE_CBOE: - break; - case bts::api::VENUE_TAIFEX: - break; + case bts::api::VENUE_SIM: case bts::api::VENUE_SIMICE: connection.mutable_sim_connection(); @@ -253,6 +250,8 @@ void AdminServiceUtils::fill_connections_from_config( c.sender_compid()); add_connection(admin_data, connection); break; + case bts::api::VENUE_TAIFEX: + case bts::api::VENUE_CBOE: default: connection.mutable_general_connection()->set_exchange(c.venue()); add_connection(admin_data, connection); @@ -574,6 +573,13 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record( return rv; } switch (sec.exchange()) { + case api::Venue::VENUE_CBOE:{//subaccount not added + rv.set_session("cboe_1"); + rv.set_account(account_ptr->name()); + rv.set_operator_id(user_ptr->name()); + out_result.set_is_success(true); + return rv; + } case api::Venue::VENUE_DERIBIT: { rv.set_session("deribit_1"); rv.set_account(account_ptr->name()); @@ -736,6 +742,13 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record( rv.set_customer_or_firm(sim_account_data->origin()); return rv; } break; + case api::Venue::VENUE_TAIFEX:{ + rv.set_session("taifex_1"); + rv.set_account(account_ptr->name()); + rv.set_operator_id(user_ptr->name()); + out_result.set_is_success(true); + return rv; + } case api::VENUE_FTX: // TODO case api::VENUE_BINANCE: // TODO case api::VENUE_BINANCE_US: @@ -744,8 +757,6 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record( case api::VENUE_OKX: case api::Venue::VENUE_UNSET: case api::Venue::VENUE_BTS: - case api::Venue::VENUE_CBOE: - case api::Venue::VENUE_TAIFEX: case api::Venue::VENUE_CBOECFE: case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_: case api::Venue::Venue_INT_MAX_SENTINEL_DO_NOT_USE_: From b5ef39ee4b9026b024470c82e3837e742881e95a Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 04:18:58 +0000 Subject: [PATCH 10/30] Fix: Change default tick_config for taifed --- src/lib/config/profiles/default/taifex_reference_data.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/config/profiles/default/taifex_reference_data.conf b/src/lib/config/profiles/default/taifex_reference_data.conf index d46a59daa..4cfaefada 100644 --- a/src/lib/config/profiles/default/taifex_reference_data.conf +++ b/src/lib/config/profiles/default/taifex_reference_data.conf @@ -6,8 +6,8 @@ tickConfig tick_entry {key:"TXF",value:1}, tick_entry {key:"MXF",value:1}, tick_entry {key:"TMF",value:1}, - tick_entry {key:"CDF",value:1}, - tick_entry {key:"QFF",value:1}, + tick_entry {key:"CDF",value:5}, + tick_entry {key:"QFF",value:5}, } } From 50abb1d784f0c7beaa3a6639d3b57e224020c6f6 Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 04:24:51 +0000 Subject: [PATCH 11/30] Fix: Refactored to Switch Statement instead of if and aadded cboe --- .../gateways/ExecutionGatewayFactory.cpp | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/lib/orderman/gateways/ExecutionGatewayFactory.cpp b/src/lib/orderman/gateways/ExecutionGatewayFactory.cpp index 28ae3d5ca..9fe30c154 100644 --- a/src/lib/orderman/gateways/ExecutionGatewayFactory.cpp +++ b/src/lib/orderman/gateways/ExecutionGatewayFactory.cpp @@ -4,9 +4,12 @@ #include "lib/orderman/gateways/CCAPIExecutionGateway.h" // #include "lib/orderman/gateways/EMExecutionGateway.h" #include "lib/orderman/gateways/FakeExecutionGateway.h" +#include "lib/orderman/gateways/GeneralExecutionGateway.h" #include "lib/orderman/gateways/GLBXExecutionGateway.h" #include "lib/orderman/gateways/GLBXiLink3ExecutionGateway.h" #include "lib/orderman/gateways/ICEExecutionGateway.h" +#include "lib/orderman/gateways/IFIXGenericAdapter.h" +#include "lib/orderman/gateways/OnixFIXAdapter.h" #include "lib/orderman/gateways/sim/SimExecutionGatewayFactory.h" #include "lib/orderman/gateways/sim/SimExecutionGatewayUniqueWrapper.h" #include "lib/tracing/PerfLog.h" @@ -21,23 +24,30 @@ std::unique_ptr ExecutionGatewayFactory::create_gateway( const config::JetFIX::Connection &conn, trace::PerfLog &pl, md::SecurityOracle o) { + LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway conn: {}", + utils::pb::to_string(conn)); if (SimExecutionGatewayFactory::is_sim_execution_on(conn.venue())) { return std::make_unique(conn.venue()); } - LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway conn: {}", - utils::pb::to_string(conn)); - if (conn.venue() == bts::api::VENUE_ICE || - conn.venue() == bts::api::VENUE_SIMICE) - return std::make_unique(conn, pl, o); - else if (conn.venue() == bts::api::VENUE_GLBX) { - if (conn.use_ilink3()) - return std::make_unique( - conn, pl, o); - return std::make_unique(conn, pl, o); - } else if (conn.venue() == bts::api::VENUE_DERIBIT) - // return std::make_unique(o); - return std::make_unique(conn.venue(), o); - return std::make_unique(o); + switch(conn.venue()){ + case bts::api::VENUE_ICE: + case bts::api::VENUE_SIMICE: + return std::make_unique(conn, pl, o); + case bts::api::VENUE_GLBX:{ + if (conn.use_ilink3()) + return std::make_unique( conn, pl, o); + return std::make_unique(conn, pl, o); + } + case bts::api::VENUE_DERIBIT: + return std::make_unique(conn.venue(), o); + case bts::api::VENUE_CBOE: + { + std::unique_ptr adapter = std::make_unique(o); + return std::make_unique(conn.venue(),std::move(adapter),o); + } + default: + return std::make_unique(o); + } } } // namespace bts::orderman From 924efb2c3fcc7fed5df45b91e03bf66202a397db Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 04:26:39 +0000 Subject: [PATCH 12/30] Add: Added proto for marex fix config --- .../profiles/default/marex_fix_cfg.conf | 17 +++++++++++ src/lib/config/protos/marex_fix_cfg.proto | 28 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 src/lib/config/profiles/default/marex_fix_cfg.conf create mode 100644 src/lib/config/protos/marex_fix_cfg.proto diff --git a/src/lib/config/profiles/default/marex_fix_cfg.conf b/src/lib/config/profiles/default/marex_fix_cfg.conf new file mode 100644 index 000000000..aee300492 --- /dev/null +++ b/src/lib/config/profiles/default/marex_fix_cfg.conf @@ -0,0 +1,17 @@ +MarexFixCfg { + value{ + customer_or_firm :0 + handl_inst: 2 + rule80a: 1 + heart_bt_int : 30 + counterparty_port :22754 + name: "trafix" + msg_type: "Order_Single" + protocol_version: "FIX_42" + sender_sub_id: "BTS" + target_sub_id: "TRFXMRX" + client_id: "BTS" + counterparty_host : "74.217.55.230" + open_close :"O" + } +} \ No newline at end of file diff --git a/src/lib/config/protos/marex_fix_cfg.proto b/src/lib/config/protos/marex_fix_cfg.proto new file mode 100644 index 000000000..a5ec9b16b --- /dev/null +++ b/src/lib/config/protos/marex_fix_cfg.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; +package bts.config; +message MarexFixCfg { + enum HandlInst { + UNDEFINED = 0; + DMA = 1; + ALGO = 2; + CARE = 3; + } + enum Rule80A { + UNUSED = 0; + A = 1; + P = 2; + } + string msg_type = 1; + string protocol_version = 2; + HandlInst handl_inst = 3; + string sender_sub_id = 4; + string target_sub_id = 5; + string client_id = 6; + Rule80A rule80a = 7; + string counterparty_host = 8; + int32 heart_bt_int = 9; + int32 counterparty_port = 10; + uint32 customer_or_firm =11; + string open_close =12; + string name = 13; +} \ No newline at end of file From f28d32946f4d3e003dacd1825a238c16b111db64 Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 04:39:43 +0000 Subject: [PATCH 13/30] Add: General Execution Gateway (between system and IFIXAdapter ) --- .../gateways/GeneralExecutionGateway.cpp | 86 +++++++++++++++++++ .../gateways/GeneralExecutionGateway.h | 29 +++++++ 2 files changed, 115 insertions(+) create mode 100644 src/lib/orderman/gateways/GeneralExecutionGateway.cpp create mode 100644 src/lib/orderman/gateways/GeneralExecutionGateway.h diff --git a/src/lib/orderman/gateways/GeneralExecutionGateway.cpp b/src/lib/orderman/gateways/GeneralExecutionGateway.cpp new file mode 100644 index 000000000..bb8b738ef --- /dev/null +++ b/src/lib/orderman/gateways/GeneralExecutionGateway.cpp @@ -0,0 +1,86 @@ +#include "GeneralExecutionGateway.h" +#include "lib/domain_model/Security.pb.h" +#include "lib/logging/logging.h" +#include "lib/orderman/CreateOrderMsg.h" +#include "lib/orderman/OrderMsg.h" +#include "lib/orderman/gateways/IExecutionGateway.h" +#include +#include + +//Acts as a gateway, On One side is the ordermanager and on the other onixfix listener and onixfix adapter +namespace bts::orderman { +struct GeneralExecutionGateway::PImpl { + PImpl(bts::api::Venue venue, + std::unique_ptr adapter, + md::SecurityOracle o) + : venue(venue), _adapter(std::move(adapter)), sec_oracle(o) {} + bts::api::Venue venue; + std::unique_ptr _adapter; + md::SecurityOracle sec_oracle; + OnExeReport on_exe_report; + OnGwStatusChange on_gw_status_change; + OnCancelReject on_cancel_reject; + OnExchangePositionState on_exchange_position_state; + OnExchangeOpenOrdersState on_exchange_open_orders; +}; +OnGwStatusChange &GeneralExecutionGateway::on_gw_status_change() { + return me->on_gw_status_change; +} +// Modified constructor to accept the adapter from outside +GeneralExecutionGateway::GeneralExecutionGateway(bts::api::Venue venue, + std::unique_ptr adapter, md::SecurityOracle o) + : IExecutionGateway(), me(new PImpl(venue, std::move(adapter),o)) { + LOG(OrderMan, Info, "ExecutionGateway::ExecutionGateway constructor"); +} +GeneralExecutionGateway::~GeneralExecutionGateway() {} +void GeneralExecutionGateway::start() { + me->on_gw_status_change(true); + me->_adapter->start(); + me->_adapter->register_callback(me->on_exe_report); + LOG(OrderMan,Info,"ExecutionGateway::start started" ); +} +void GeneralExecutionGateway::send_msgs( + std::vector const &msgs, trace::CorID) { + LOG(OrderMan,Info,"ExecutionGateway::send_msgs" ); + for (const auto &msg : msgs) { + std::visit( + [this](auto &&order) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + LOG(OrderMan,Warn,"ExecutionGateway::CreateOrderMsg" ); + me->_adapter->launch_new_order(order); + + } else if constexpr (std::is_same_v) { + LOG(OrderMan,Warn,"ExecutionGateway::CancelOrderMsg" ); + me->_adapter->cancel_order(order); + } else if constexpr (std::is_same_v) { + LOG(OrderMan,Warn,"ExecutionGateway::CancelReplaceOrderMsg" ); + me->_adapter->modify_order(order); + } else if constexpr (std::is_same_v) { + // Handle RFQMsg if necessary + // adapter->handle_rfq(order); // Example placeholder + } + }, + msg); + } +} +void GeneralExecutionGateway::dispatch_incoming_fix_messages() {} +void GeneralExecutionGateway::on_securities( + const mercury::Update &) {} +void GeneralExecutionGateway::on_admin_data( + const mercury::Update &) {} +void GeneralExecutionGateway::send_fix(const jetfix::OutFIXPtr &, trace::CorID) {} +OnExeReport &GeneralExecutionGateway::on_exe_report() { return me->on_exe_report; } +OnCancelReject &GeneralExecutionGateway::on_cancel_reject() { + static OnCancelReject default_cancel_reject; + return default_cancel_reject; +} +OnExchangePositionState &GeneralExecutionGateway::on_exchange_position_state() { + static OnExchangePositionState default_exchange_position_state; + return default_exchange_position_state; +} +OnExchangeOpenOrdersState &GeneralExecutionGateway::on_exchange_open_orders() { + static OnExchangeOpenOrdersState default_exchange_open_orders; + return default_exchange_open_orders; +} +} //namespace bts::orderman \ No newline at end of file diff --git a/src/lib/orderman/gateways/GeneralExecutionGateway.h b/src/lib/orderman/gateways/GeneralExecutionGateway.h new file mode 100644 index 000000000..1969b9321 --- /dev/null +++ b/src/lib/orderman/gateways/GeneralExecutionGateway.h @@ -0,0 +1,29 @@ +#pragma once +#include "lib/orderman/gateways/IExecutionGateway.h" +#include "lib/orderman/gateways/IFIXGenericAdapter.h" +namespace bts { +namespace orderman { +class GeneralExecutionGateway : public IExecutionGateway { +public: + GeneralExecutionGateway(bts::api::Venue venue,std::unique_ptr adapter, + md::SecurityOracle o); + virtual ~GeneralExecutionGateway(); + void start() override; + void send_msgs(std::vector const & msgs, trace::CorID) override; + void dispatch_incoming_fix_messages() override; + void on_securities(const mercury::Update &) override; + void on_admin_data(const mercury::Update &) override; + void send_fix(const jetfix::OutFIXPtr &, trace::CorID) override; + OnExeReport &on_exe_report() override; + OnGwStatusChange &on_gw_status_change() override; + OnCancelReject &on_cancel_reject() override; + OnExchangePositionState &on_exchange_position_state() override; + OnExchangeOpenOrdersState &on_exchange_open_orders() override; +private: + struct PImpl; + std::unique_ptr me; + + +}; +} //namespace bts +} //namespace orderman \ No newline at end of file From 96d9aae048e4415ff6a5b205ce100c3b14094b81 Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 04:45:36 +0000 Subject: [PATCH 14/30] Add: IFIXGenericAdapter --- .../orderman/gateways/IFIXGenericAdapter.cpp | 5 +- .../orderman/gateways/IFIXGenericAdapter.h | 50 +++++++------------ 2 files changed, 22 insertions(+), 33 deletions(-) diff --git a/src/lib/orderman/gateways/IFIXGenericAdapter.cpp b/src/lib/orderman/gateways/IFIXGenericAdapter.cpp index 1f3e29caf..b4f9aba5b 100644 --- a/src/lib/orderman/gateways/IFIXGenericAdapter.cpp +++ b/src/lib/orderman/gateways/IFIXGenericAdapter.cpp @@ -1,3 +1,6 @@ #include "IFIXGenericAdapter.h" -namespace bts::orderman {} \ No newline at end of file +namespace bts::orderman { + + IFIXGenericAdapter::~IFIXGenericAdapter(){} +} \ No newline at end of file diff --git a/src/lib/orderman/gateways/IFIXGenericAdapter.h b/src/lib/orderman/gateways/IFIXGenericAdapter.h index 98bfc52d7..74bdb01bb 100644 --- a/src/lib/orderman/gateways/IFIXGenericAdapter.h +++ b/src/lib/orderman/gateways/IFIXGenericAdapter.h @@ -1,22 +1,13 @@ #pragma once -#include "lib/api/services/OrderService.pb.h" -#include "lib/config/protos/jetfix.pb.h" -#include "lib/domain_model/Security.pb.h" -#include "lib/jetfix/codec/MessageDefinitions.h" -#include "lib/jetfix/common/OutFIXPtr.h" -#include "lib/md/SecurityOracle.h" -#include "lib/mercury/events/Update.h" +#include "lib/orderman/CancelOrderMsg.h" +#include "lib/orderman/CancelReplaceOrderMsg.h" #include "lib/orderman/CreateOrderMsg.h" -#include "lib/orderman/OrderMsg.h" -#include "lib/orderman/orderman_utils/ExchangeOpenOrders.h" -#include "lib/orderman/orderman_utils/ExchangePosition.h" -#include "lib/orderman/orderman_utils/orderman_common.h" -#include "lib/utils/bts_no_copy.h" +#include "lib/tracing/PerfLog.h" #include -#include -#include + + namespace bts::orderman { @@ -29,33 +20,28 @@ class NewAdminDataEvent; class ExecutionReport; class CancelReject; -struct OrderStateWithFills { - api::orderman::OrderState ordr_state; - std::vector fills = {}; -}; - -using OnOrderUpdate = std::function; +using OnExecutionReport = std::function; // fix execution gateway for a single FIX session class IFIXGenericAdapter { public: IFIXGenericAdapter() = default; - virtual ~IFIXGenericAdapter() {} + virtual ~IFIXGenericAdapter(); virtual void start() = 0; - // this callback should be added in a list, and on receving an update from the other side, it should be passed here - virtual void register_callback(const OnOrderUpdate &); - virtual void launch_new_order(const CreateOrderMsg &); - virtual void modify_order(const CancelReplaceOrderMsg &); - virtual void cancel_order(const CancelOrderMsg &); +// // this callback should be added in a list, and on receving an update from the other side, it should be passed here + virtual void register_callback(const OnExecutionReport &) =0 ; + virtual void launch_new_order(const CreateOrderMsg &) =0; + virtual void modify_order(const CancelReplaceOrderMsg &) = 0; + virtual void cancel_order(const CancelOrderMsg &) =0; // as new securities are added - virtual void update_securities( - const mercury::Update &) = 0; - virtual void update_admin_data( - const mercury::Update &) = 0; + // virtual void update_securities( + // const mercury::Update &) = 0; + // virtual void update_admin_data( + // const mercury::Update &) = 0; - private: - BTS_NO_COPY(IFIXGenericAdapter); +// private: +// BTS_NO_COPY(IFIXGenericAdapter); }; } // namespace bts::orderman From 89955597c1b1ddd9a856967b9ef329bb5ee94002 Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 06:16:58 +0000 Subject: [PATCH 15/30] Add: Onix/Marex fix adapter, listener, and state handler --- src/lib/orderman/gateways/OnixFIXAdapter.cpp | 357 ++++++++++++++++++ src/lib/orderman/gateways/OnixFIXAdapter.h | 27 ++ src/lib/orderman/gateways/OnixFIXListener.cpp | 264 +++++++++++++ src/lib/orderman/gateways/OnixFIXListener.h | 52 +++ .../gateways/TrafixOrderStateHandler.cpp | 88 +++++ .../gateways/TrafixOrderStateHandler.h | 61 +++ 6 files changed, 849 insertions(+) create mode 100644 src/lib/orderman/gateways/OnixFIXAdapter.cpp create mode 100644 src/lib/orderman/gateways/OnixFIXAdapter.h create mode 100644 src/lib/orderman/gateways/OnixFIXListener.cpp create mode 100644 src/lib/orderman/gateways/OnixFIXListener.h create mode 100644 src/lib/orderman/gateways/TrafixOrderStateHandler.cpp create mode 100644 src/lib/orderman/gateways/TrafixOrderStateHandler.h diff --git a/src/lib/orderman/gateways/OnixFIXAdapter.cpp b/src/lib/orderman/gateways/OnixFIXAdapter.cpp new file mode 100644 index 000000000..c65484558 --- /dev/null +++ b/src/lib/orderman/gateways/OnixFIXAdapter.cpp @@ -0,0 +1,357 @@ +#include "OnixFIXAdapter.h" +#include "OnixS/FIXEngine/FIX/Engine.h" +#include "lib/config/protos/marex_fix_cfg.pb.h" +#include "lib/logging/logging.h" +#include "lib/orderman/CancelOrderMsg.h" +#include "lib/orderman/CancelReplaceOrderMsg.h" +#include "lib/orderman/CreateOrderMsg.h" +#include "lib/orderman/gateways/OnixFIXListener.h" +#include "lib/orderman/gateways/TrafixOrderStateHandler.h" +#include "lib/orderman/orderman_utils/orderman_common.h" +#include "lib/utils/bts_assert.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bts::orderman { +void handleException( + const std::string &context, const std::string &error) { + bts_assert_msgf(false, "{} Exception: {}", context, error); +} +OnixFIXAdapter::OnixFIXAdapter(md::SecurityOracle sec_oracle) + : state_handler(std::make_shared(sec_oracle)), + listener(state_handler) { + try { + auto onix_cfg = config::get(); + LOG(OrderMan, Warn, "Initializing OnixFIXAdapter"); + OnixS::FIX::Engine:: + init(); // OnixS::FIX::Engine::init() needs to run before initialising session + LOG(OrderMan, Diag, "Initializing OnixFIXAdapter engine initailes trying login"); + session = std::make_unique( + onix_cfg.sender_sub_id(), onix_cfg.target_sub_id(), + OnixS::FIX::ProtocolVersion::FIX_42, &listener); + } catch (const OnixS::FIX::Exception &ex) { + const std::string err = ex.what(); + LOG(OrderMan, Error, "Initializing OnixFIXAdapter ...{}",err); + Engine::shutdown(); + handleException("OnixFIXAdapter::Construcotr init", err); + } +} +OnixFIXAdapter::~OnixFIXAdapter() { + try { + if (session) + session->logout(); + Engine::instance()->unregisterListener(); + Engine::shutdown(); + } catch (const OnixS::FIX::Exception &ex) { + const std::string err = ex.what(); + handleException("OnixFIXAdapter::Destructor ", err); + } +} +void OnixFIXAdapter::start() { + try { + LOG(OrderMan, Warn, "OnixFIXAdapter::start logging on"); + auto onix_cfg = config::get(); + const std::string counterpartyHost = onix_cfg.counterparty_host(); + const int heartBtInt = onix_cfg.heart_bt_int(); + const int counterpartyPort = onix_cfg.counterparty_port(); + LOG(OrderMan, Diag, "OnixFIXAdapter::start logging on port: {}, ip: {}, heartbt:{}",counterpartyPort,heartBtInt,counterpartyHost); + session->logonAsInitiator(counterpartyHost, counterpartyPort, heartBtInt); + LOG(OrderMan, Warn, "OnixFIXAdapter::start logging on done"); + + } catch (const OnixS::FIX::Exception &ex) { + Engine::shutdown(); + const std::string err = ex.what(); + handleException("OnixFIXAdapter::start", err); + } +} +template +uint translateEnum(T enum_value, const std::map &translationMap) { + auto it = translationMap.find(enum_value); + bts_assert(it != translationMap.end()); // Using assert instead of bts_assert + return it->second; +} + +uint translateTimeInForce(api::orderman::TimeInForce time_in_force) { + //0 = Day 1 = GTC 2 = At the Open 3 = IOC 4 = FOK 5 = GTX + static const std::map tifMap = { + {api::orderman::TimeInForce::TIME_IN_FORCE_DAY, 0}, + {api::orderman::TimeInForce::TIME_IN_FORCE_GTC, 1}, + {api::orderman::TimeInForce::TIME_IN_FORCE_IOC, 3}}; + + return translateEnum(time_in_force, tifMap); +} + +uint translateOrderType(api::orderman::NativeOrderType order_type) { + //1 = Market 2 = Limit 3 = Stop 4 = Stop Limit 5 = Market on close A = On Close B = Limit on Close + static const std::map orderTypeMap = { + {api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_MARKET, 1}, + {api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_LIMIT, 2}, + {api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_STOP, 3}, + {api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_STOP_LIMIT, 4}}; + + return translateEnum(order_type, orderTypeMap); +} +uint translateSide(api::Side side) { + //1 = Buy 2 = Sell 5 = Sell short 6 = Sell short Exempt + static const std::map sideMap = { + {api::Side::SIDE_BUY, 1}, {api::Side::SIDE_SELL, 2}}; + + return translateEnum(side, sideMap); +} + +std::string pad_string( + const std::string &str, size_t length, char padding = ' ') { + if (str.size() > length) { + return str.substr(0, length); + } + return str + std::string(length - str.size(), padding); +} + +// Function to create the OSI Symbol +static std::string create_osi_symbol( + const std::string &symbol, + const uint32_t &year, + const uint32_t &month, + const uint32_t &day, + const char &call_put_indicator, + const double &strike_price) { + if (symbol.size() > 6) { + throw std::invalid_argument("Symbol length cannot exceed 6 characters."); + } + + std::ostringstream osi_stream; + + // Symbol: 6 bytes, padded with spaces + osi_stream << pad_string(symbol, 6); + + // Year: last 2 digits of the year + osi_stream << std::setw(2) << std::setfill('0') << (year % 100); + + // Month: 2 digits + osi_stream << std::setw(2) << std::setfill('0') << month; + + // Day: 2 digits + osi_stream << std::setw(2) << std::setfill('0') << day; + + // Call/Put Indicator: 1 byte + osi_stream << call_put_indicator; + + // Strike Price: 5 bytes for dollars, 3 bytes for decimal part + int strike_dollars = static_cast(strike_price); + int strike_decimals = + static_cast((strike_price - strike_dollars) * 1000); + osi_stream << std::setw(5) << std::setfill('0') << strike_dollars; + osi_stream << std::setw(3) << std::setfill('0') << strike_decimals; + + return osi_stream.str(); +} + +void parse_option(Message &order, const api::Security &sec) { + auto osi = create_osi_symbol( + sec.symbol(), sec.option().expiration_year(), + sec.option().expiration_month(), sec.option().expiration_day(), + sec.option().call_put() == bts::api::PutOrCall::PUTORCALL_CALL ? 'c' + : 'p', + sec.option().strike()); + std::ostringstream month_stream; + month_stream << std::setw(2) << std::setfill('0') + << sec.option().expiration_month(); + std::string month = month_stream.str(); + LOG(OrderMan, Warn, "OnixFIXAdapter::parse_option parsing osi:{}", osi); + order + .set( + OnixS::FIX::FIX42::Tags::PutOrCall, + sec.option().call_put() == bts::api::PutOrCall::PUTORCALL_CALL ? "1" + : "0") + .set( + OnixS::FIX::FIX42::Tags::StrikePrice, + std::to_string(sec.option().strike())) + .set( + OnixS::FIX::FIX42::Tags::MaturityMonthYear, + std::to_string(sec.option().expiration_year()) + month) + .set( + OnixS::FIX::FIX42::Tags::MaturityDay, + std::to_string(sec.option().expiration_day())) + .set(OnixS::FIX::FIX42::Tags::SecurityType, "OPT") + .set(OnixS::FIX::FIX42::Tags::Symbol, sec.symbol()); +} + +void add_cfg_fields(Message &order, const config::MarexFixCfg &cfg) { + order + .set( + OnixS::FIX::FIX42::Tags::HandlInst, + cfg.handl_inst()) // 1: DMA 2: ALGO 3:CARE + .set( + OnixS::FIX::FIX42::Tags::Rule80A, + cfg.Rule80A_Name(cfg.rule80a())) //A = AGENCY P = PRINCIPAL + .set(OnixS::FIX::FIX42::Tags::SenderSubID, cfg.sender_sub_id()) + .set(OnixS::FIX::FIX42::Tags::TargetSubID, cfg.target_sub_id()) + .set(OnixS::FIX::FIX42::Tags::ClientID, cfg.client_id()) + .set(OnixS::FIX::FIX42::Tags::CustomerOrFirm, cfg.customer_or_firm()) + .set(OnixS::FIX::FIX42::Tags::OpenClose, cfg.open_close()); +} +void populate_common_order_fields(Message &order, const CreateOrderMsg &msg) { + order.set(OnixS::FIX::FIX42::Tags::Account, msg.routing_record->account()) + .set( + OnixS::FIX::FIX42::Tags::TimeInForce, + translateTimeInForce(msg.time_in_force)) + .set( + OnixS::FIX::FIX42::Tags::OrdType, translateOrderType(msg.order_type)) + .set(OnixS::FIX::FIX42::Tags::Side, translateSide(msg.side)) + .set( + OnixS::FIX::FIX42::Tags::Price, + static_cast(msg.price.value())) + .set(OnixS::FIX::FIX42::Tags::OrderQty, msg.quantity.value()); + if (msg.security->has_option()) { + parse_option(order, *msg.security); + } else { + order.set(OnixS::FIX::FIX42::Tags::Symbol, msg.security->symbol()) + .set(OnixS::FIX::FIX42::Tags::SecurityType, "FUT") //need to fix this + .set(OnixS::FIX::FIX42::Tags::Currency, msg.security->currency()); + } +} +void OnixFIXAdapter::modify_order( + const CancelReplaceOrderMsg &msg) { //confirm will have everything + try { + + //use this fileOnixS.FixEngineCpp-Ubuntu2204-gcc1210-c++17-x64-4.10.1/OnixS.FixEngineCpp-Ubuntu2204-gcc1210-c++17-x64-4.10.1/doc/html/group__double-decimal-conversion.html + + LOG(OrderMan, Warn, "OnixFIXAdapter::modify_order modifying order{}", + msg); + auto order_cfg = config::get(); + auto id = + state_handler->get_marex_key(OrderKey(msg.create_fields.key.value())); + + if (id == "") { + LOG(OrderMan, Error, + "OnixFIXAdapter::modify_order could not find clorid"); + return; + } + auto new_id = state_handler->new_id(id); + + Message order( + OnixS::FIX::FIX42::Values::MsgType::Order_Cancel_Replace_Request, + OnixS::FIX::ProtocolVersion::FIX_42); + order.set(OnixS::FIX::FIX42::Tags::OrigClOrdID, id) + .set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id) // ID for order + .set( + OnixS::FIX::FIX42::Tags::TransactTime, + OnixS::FIX::Timestamp::utc(), + OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec); + + add_cfg_fields(order, order_cfg); + populate_common_order_fields(order, msg.create_fields); + order.validate(); + LOG(OrderMan, Warn, "order Validated"); + order.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART"); + std::string order_string = order.toString('|'); + state_handler->modifyOrder(msg, new_id); + session->send(&order); + LOG(OrderMan, Warn, "Order update sent :{}", order_string); + } catch (const OnixS::FIX::Exception &ex) { + const std::string err = ex.what(); + handleException("Modify order", err); + } +} + +void OnixFIXAdapter::launch_new_order(const CreateOrderMsg &msg) { + LOG(OrderMan, Warn, "launch order {}", msg); + + try { + LOG(OrderMan, Warn, "OnixFIXAdapter::launch_new_order creating order"); + auto order_cfg = config::get(); + + auto new_id = + state_handler->new_id(":" + std::to_string(msg.key.value())); + LOG(OrderMan, Warn, "OnixFIXAdapter::launch_new_order new id:'{}'", + new_id); + std::string symbol = msg.security->symbol(); + Message order( + OnixS::FIX::FIX42::Values::MsgType::Order_Single, + OnixS::FIX::ProtocolVersion::FIX_42); + order + .set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id) // ID for order + .set( + OnixS::FIX::FIX42::Tags::TransactTime, + OnixS::FIX::Timestamp::utc(), + OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec); + add_cfg_fields(order, order_cfg); + populate_common_order_fields(order, msg); + + std::string order_string = order.toString('|'); + std::string order_json = order.toJson(); + order.validate(); + LOG(OrderMan, Warn, "order Validated"); + order.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART"); + state_handler->launchOrder(msg, new_id); + // listener.fire_dummy_report(msg); + session->send(&order); + LOG(OrderMan, Warn, "Order sent :{} json:{}", order_string, order_json); + + } + // Shut down all reactors if they exist. + catch (const OnixS::FIX::Exception &ex) { + const std::string err = ex.what(); + handleException("Launch order", err); + } +} +void OnixFIXAdapter::register_callback(const OnExecutionReport &cb) { + LOG(OrderMan, Warn, "OnixFIXAdapter::register_callback callback registered"); + listener.setCallback(cb); +} +void OnixFIXAdapter::cancel_order(const CancelOrderMsg &msg) { + LOG(OrderMan, Warn, "cancel order {}", msg); + try { + auto id = state_handler->get_marex_key(OrderKey(msg.key.value())); + auto order_cfg = config::get(); + + if (id == "") { + LOG(OrderMan, Error, + "OnixFIXAdapter::modify_order could not find clorid"); + return; + } + auto new_id = state_handler->new_id(id); + Message order_cancel( + OnixS::FIX::FIX42::Values::MsgType::Order_Cancel_Request, + OnixS::FIX::ProtocolVersion::FIX_42); + order_cancel.set( + OnixS::FIX::FIX42::Tags::HandlInst, order_cfg.handl_inst()); + order_cancel.set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id); + order_cancel.set(OnixS::FIX::FIX42::Tags::OrigClOrdID, id) + .set(OnixS::FIX::FIX42::Tags::Side, translateSide(msg.side)) + .set( + OnixS::FIX::FIX42::Tags::TransactTime, + OnixS::FIX::Timestamp::utc(), + OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec); + if (msg.security->has_option()) { + parse_option(order_cancel, *msg.security); + } else { + order_cancel + .set(OnixS::FIX::FIX42::Tags::Symbol, msg.security->symbol()) + .set(OnixS::FIX::FIX42::Tags::Currency, msg.security->currency()); + } + std::string order_str = order_cancel.toString('|'); + LOG(OrderMan, Warn, "order_cancel Created: {}", order_str); + // order_cancel.validate(); fails becasuse of HandlInst + LOG(OrderMan, Warn, "order Validated"); + order_cancel.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART"); + state_handler->cancelOrder(msg, new_id); + session->send(&order_cancel); + LOG(OrderMan, Warn, "order cancel sent"); + + } + // Shut down all reactors if they exist. + catch (const OnixS::FIX::Exception &ex) { + const std::string err = ex.what(); + handleException("Cacnel order", err); + } +} +} // namespace bts::orderman \ No newline at end of file diff --git a/src/lib/orderman/gateways/OnixFIXAdapter.h b/src/lib/orderman/gateways/OnixFIXAdapter.h new file mode 100644 index 000000000..06e1a234f --- /dev/null +++ b/src/lib/orderman/gateways/OnixFIXAdapter.h @@ -0,0 +1,27 @@ +#pragma once + +#include "IFIXGenericAdapter.h" +#include "lib/orderman/gateways/OnixFIXListener.h" +#include "lib/orderman/gateways/TrafixOrderStateHandler.h" +#include + +namespace bts::orderman { + +class OnixFIXAdapter : public IFIXGenericAdapter { + static constexpr int FIX_TAG_SMART_ROUTING = 847; + public: + OnixFIXAdapter(md::SecurityOracle sec_oracle); + ~OnixFIXAdapter() override; + void register_callback(const OnExecutionReport &report) override; + void launch_new_order(const CreateOrderMsg &msg) override; + void cancel_order(const CancelOrderMsg &msg) override; + void modify_order(const CancelReplaceOrderMsg &) override; + void start() override; + + private: + std::shared_ptr state_handler; + OnixFixListener listener; + std::unique_ptr session; +}; + +} // namespace bts::orderman \ No newline at end of file diff --git a/src/lib/orderman/gateways/OnixFIXListener.cpp b/src/lib/orderman/gateways/OnixFIXListener.cpp new file mode 100644 index 000000000..ea47c956a --- /dev/null +++ b/src/lib/orderman/gateways/OnixFIXListener.cpp @@ -0,0 +1,264 @@ +#include "OnixFIXListener.h" + +#include "lib/api/services/OrderService.pb.h" +#include "lib/domain_model/Security.pb.h" +#include "lib/domain_model/common.h" +#include "lib/logging/logging.h" +#include "lib/orderman/gateways/IExecutionGateway.h" +#include "lib/orderman/gateways/IFIXGenericAdapter.h" +#include "lib/orderman/orderman_utils/ExecutionReport.h" +#include "lib/orderman/orderman_utils/orderman_common.h" +#include "lib/utils/Uuid.h" +#include "lib/utils/bts_assert.h" +#include +#include +#include + +namespace bts::orderman { +ExecutionReport OnixFixListener::fire_dummy_report(CreateOrderMsg msg) { + ExecutionReport *er_p = exe_report_pool.construct(); + ExecutionReport &er = *er_p; + std::string account_name = "BTSIM"; + std::array an_array = {}; + size_t i = 0; + for (const char c : account_name) { + an_array[i++] = c; + } + er.account_name = AccountName(an_array); + to_array(msg.routing_record->account(), &er.account_name.value()); + to_array( + msg.routing_record->operator_id(), &er.operator_id.value()); + er.order_key = msg.key; + er.msg_seq_num = 0; + er.security = msg.security; + er.leg_security = nullptr; + er.side = msg.side; + er.time_in_force = msg.time_in_force; + er.order_type = msg.order_type; + er.exec_type = api::orderman::EXEC_TYPE_NEW; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_NEW; + er.manual_order_indicator = api::orderman::MANUAL_ORDER_INDICATOR_NO; + auto s = std::to_string(msg.key.value()); + std::array idAsArray{}; + std::copy_n(s.begin(), std::min(s.size()-1, idAsArray.size() - 1), idAsArray.begin()); + er.exchange_order_id = ExchangeOrderID{idAsArray}; + er.price = msg.price; + er.quantity = msg.quantity; + er.filled_quantity = Quantity(0); + er.last_shares = Quantity(0); + er.msg = "Message Placed"; + er.exchange = msg.security->exchange(); + // Set transaction time + ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now()); + er.transact_time = now; + cb(er,{}); + return er; +} + +namespace { + ExecutionReport createDefaultExecutionReport() { + ExecutionReport er; + er.exchange = api::VENUE_CBOE; + std::string account_name = "BTSIM"; + std::string ops_id = "op_id_bts"; + + std::array account_array = {}; + std::copy_n(account_name.begin(), std::min(account_name.size(), account_array.size()), account_array.begin()); + er.account_name = AccountName(account_array); + + to_array(ops_id, &er.operator_id.value()); + return er; + } + + void populateExecTypeAndOrderStatus(ExecutionReport &er, int exec_type_value) { + switch (exec_type_value) { + case 0: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_NEW; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_NEW; + break; + case 1: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_PARTIAL_FILL; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_PARTIALLY_FILLED; + break; + case 2: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_FILL; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_FILLED; + break; + case 4: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_CANCELED; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_CANCELED; + break; + case 5: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_REPLACED; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_REPLACED; + break; + case 8: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_REJECTED; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED; + break; + default: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_UNDEFINED; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_UNDEFINED; + break; + } + } + + void populateCreateOrderFields(ExecutionReport &er, const CreateOrderMsg &create_order_msg) { + er.security = create_order_msg.security; + er.order_type = create_order_msg.order_type; + er.time_in_force = create_order_msg.time_in_force; + er.manual_order_indicator = create_order_msg.manual_order_indicator; + + to_array(create_order_msg.routing_record->account(), &er.account_name.value()); + to_array(create_order_msg.routing_record->operator_id(), &er.operator_id.value()); + er.exchange = create_order_msg.security->exchange(); + } + + std::array stringToFixedArray(const std::string &s) { + std::array idAsArray{}; + std::copy_n(s.begin(), std::min(s.size(), idAsArray.size() - 1), idAsArray.begin()); + return idAsArray; + } +} +ExecutionReport OnixFixListener::get_exe_report(Message msg) { + ExecutionReport *er_p = exe_report_pool.construct(); + ExecutionReport &er = *er_p; + + auto msg_str = msg.toString(); + LOG(OrderMan,Info, "OnixFixListener::get_exe_report {}", msg_str); + + // Extract order key and create order message + er.order_key = state_handler->getOrderKey(msg.get(FIX42::Tags::ClOrdID).toString()); //OrderKey(extractUInt(msg.get(FIX42::Tags::ClOrdID).toString())); + auto m = state_handler->getCreateOrderMsg(er.order_key); + + if(m.has_value()){ + populateCreateOrderFields(er, m.value()); + }else{ //handle this later + LOG(OrderMan,Error, "OnixFixListener::get_exe_report {} could not find create order message", msg_str); + er = createDefaultExecutionReport(); + } + // Populate fields from the message + + int tmp; + msg.get(FIX42::Tags::MsgSeqNum).toNumber(tmp); + er.msg_seq_num = tmp; + + er.leg_security = nullptr; + + er.side = msg.get(FIX42::Tags::Side).toString()=="1"?api::Side::SIDE_BUY:api::Side::SIDE_SELL; + + int exec_type_value = 9; // Default to undefined + msg.get(FIX42::Tags::ExecType).toNumber(exec_type_value); + populateExecTypeAndOrderStatus(er, exec_type_value); + + auto price_str = msg.get(FIX42::Tags::Price); + er.price = Price(price_str.valid() ? std::stod(price_str.toString()) : 0); + + auto order_qty_str = msg.get(FIX42::Tags::OrderQty); + er.quantity = Quantity(order_qty_str.valid() ? std::stoi(order_qty_str.toString()) : 0); + + auto leaves_qty_str = msg.get(FIX42::Tags::LeavesQty); + er.filled_quantity = leaves_qty_str.valid() ? er.quantity - Quantity(std::stoi(leaves_qty_str.toString())) : er.quantity; + + auto last_shares_str = msg.get(FIX42::Tags::LastShares); + er.last_shares = Quantity(last_shares_str.valid() ? std::stoi(last_shares_str.toString()) : 0); + + auto avg_px_str = msg.get(FIX42::Tags::AvgPx); + er.avg_price = Price(avg_px_str.valid() ? std::stod(avg_px_str.toString()) : 0); + + auto text_str = msg.get(FIX42::Tags::Text).toString(); + er.msg = text_str.substr(0, std::min(text_str.length(), size_t(15))); + + auto exec_id_str = msg.get(FIX42::Tags::ExecID); + std::string fill_id = exec_id_str.valid() ? exec_id_str.toString() : bts::utils::Uuid().to_text_string(); + to_array(fill_id, &er.exchange_fill_id.value()); + + auto last_px_str = msg.get(FIX42::Tags::LastPx); + er.last_price = Price(last_px_str.valid() ? std::stod(last_px_str.toString()) : 0); + + + er.exchange_order_id = ExchangeOrderID{stringToFixedArray(msg.get(FIX42::Tags::OrderID).toString())}; + + + // er.transact_time = ExchangeTimeStamp64(datetime::unix_nano_now()); + //exchgane gives this "TransactTime": "20241111-16:36:08.243", + // er.transact_time = + // Set transaction time + ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now()); + er.transact_time = now; + + return er; +} + +void OnixFixListener::onInboundApplicationMsg(Message &msg, Session * /*sn*/) { + std::string msg_str = msg.toString(); + std::string msg_json = msg.toJson(); + + FieldValueRef t = msg.type(); + char message_type; + if(!t.toChar(message_type)){ + LOG(OrderMan, Warn, + "no message type message: {} As json {}", + msg_str, msg_json); + return; + } + LOG(OrderMan, Warn, + "Incoming application-level message: {} As json {}, type: {}", + msg_str, msg_json,message_type); + + switch (message_type) { + case '8': + { + //exection report + auto er = get_exe_report(msg); + if(er.security==nullptr){ + LOG(OrderMan, Error,"DID not have sec"); + return; + } + LOG(OrderMan, Error,"msg {}",er.msg); + cb(er,{}); + } + } + + // LOG(OrderMan, Warn, + // "onInboundApplicationMsg::inbound firing er: {}", + // er->to_string()); + + // cb(*er, {}); +} + + +void OnixFixListener::onInboundSessionMsg(Message & msg, Session * /*sn*/) +{ + std::string msg_str = msg.toString(); + std::string msg_json = msg.toJson(); + LOG( OrderMan, Warn, + "Incoming session-level message:\n {} As json {}", + msg_str, + msg_json); +} + +void OnixFixListener::onStateChange(SessionState::Enum newState, SessionState::Enum prevState, + Session * /*sn*/) +{ + std::string prevst = SessionState::toString(prevState); + std::string newst = SessionState::toString(newState); + LOG(OrderMan, Warn, + "Session's state is changed, prevState = {} , newState= {}", + prevst, + newst); +} + +void OnixFixListener::onError(ErrorReason::Enum /*reason*/, const std::string & description, Session * /*sn*/) +{ + LOG( OrderMan, Error, "Session-level error: {}", description); +} + +void OnixFixListener::onWarning(WarningReason::Enum /*reason*/, const std::string & description, + Session * /*sn*/) +{ + LOG( OrderMan, Error, "Session-level warning: {}", description); +} + + +} // namespace bts::orderman \ No newline at end of file diff --git a/src/lib/orderman/gateways/OnixFIXListener.h b/src/lib/orderman/gateways/OnixFIXListener.h new file mode 100644 index 000000000..cc0f8755e --- /dev/null +++ b/src/lib/orderman/gateways/OnixFIXListener.h @@ -0,0 +1,52 @@ +#pragma once + +#include "lib/orderman/CreateOrderMsg.h" +#include "lib/orderman/gateways/IExecutionGateway.h" +#include "lib/orderman/gateways/TrafixOrderStateHandler.h" +#include "lib/orderman/orderman_utils/ExecutionReport.h" +#include +#include +#include + +namespace bts::orderman { +class TrafixOrderStateHandler; +using namespace OnixS::FIX; +class OnixFixListener : public OnixS::FIX::ISessionListener { + public: + OnixFixListener( std::shared_ptr state_handler) : state_handler(state_handler),cb(dummy) { + LOG(OrderMan, Info, "OnixFixListener::cb set callback registered"); + } + std::shared_ptr state_handler; + unsigned long exchange_oid = 0; + /// Is called when the application-level message is received from the counterparty. + void onInboundApplicationMsg(Message &msg, Session *sn) + ONIXS_FIXENGINE_FINAL; + // Set the real callback function + void setCallback(const OnExeReport &callback) { + LOG(OrderMan, Info, "OnixFixListener::setCallback callback registered"); + cb = callback; + } + /// Is called when the session-level message is received from the counterparty. + void onInboundSessionMsg(Message &msg, Session *sn) ONIXS_FIXENGINE_FINAL; + void onStateChange( + SessionState::Enum newState, + SessionState::Enum prevState, + Session *sn) ONIXS_FIXENGINE_FINAL; + OnExeReport cb; + ExecutionReport fire_dummy_report(CreateOrderMsg msg); + ExecutionReport get_exe_report(Message msg); + static void dummy(const ExecutionReport &, const trace::CorID &) { + LOG(OrderMan, Error, "OnixFixListener::Dummy fired"); + } + void onError( + ErrorReason::Enum reason, + const std::string &description, + Session *sn) ONIXS_FIXENGINE_FINAL; + void onWarning( + WarningReason::Enum reason, + const std::string &description, + Session *sn) ONIXS_FIXENGINE_FINAL; + boost::object_pool exe_report_pool; +}; + +} // namespace bts::orderman \ No newline at end of file diff --git a/src/lib/orderman/gateways/TrafixOrderStateHandler.cpp b/src/lib/orderman/gateways/TrafixOrderStateHandler.cpp new file mode 100644 index 000000000..0bb702e38 --- /dev/null +++ b/src/lib/orderman/gateways/TrafixOrderStateHandler.cpp @@ -0,0 +1,88 @@ +#include "TrafixOrderStateHandler.h" +#include "lib/logging/logging.h" +#include "lib/orderman/CreateOrderMsg.h" +#include "lib/orderman/order_factory/IOrderFactory.h" +#include "lib/orderman/orderman_utils/orderman_common.h" +#include +#include +#include +#include + +namespace bts::orderman { + +TrafixOrderStateHandler::TrafixOrderStateHandler(md::SecurityOracle sec_oracle) + : pimpl_(std::make_unique(sec_oracle)) {} + +TrafixOrderStateHandler::~TrafixOrderStateHandler() = default; + +TrafixOrderStateHandler::OrderInfo* TrafixOrderStateHandler::get_order_info(const OrderKey&key) const { + auto it = pimpl_->orders.find(key); + if (it != pimpl_->orders.end()) { + return &it->second; + } + return nullptr; +} + + +OrderKey TrafixOrderStateHandler::launchOrder(const CreateOrderMsg& msg, const std::string& clor_id) { + std::lock_guard lock(pimpl_->orders_mutex); + pimpl_->orders.emplace(msg.key, OrderInfo(msg, clor_id)); + return msg.key; +} +OrderKey TrafixOrderStateHandler::modifyOrder(const CancelReplaceOrderMsg& msg, const std::string& clor_id) { + std::lock_guard lock(pimpl_->orders_mutex); + LOG(md,Info,"Modifying Order: {}, id:{}",msg,clor_id); + auto v = get_order_info(msg.create_fields.key); + if(v==nullptr) return null_order_key(); + LOG(md,Info,"Modifying found order info Order: {}, id:{}",msg,clor_id); + v->update(msg, clor_id); + return msg.create_fields.key; +} +OrderKey TrafixOrderStateHandler::cancelOrder(const CancelOrderMsg& msg, const std::string& clor_id) { + std::lock_guard lock(pimpl_->orders_mutex); + LOG(md,Info,"canceling Order: {}, id:{}",msg,clor_id); + auto v = get_order_info(msg.key); + if(v==nullptr) return null_order_key(); + std::string cancel_id = clor_id; + v->cancel(msg, cancel_id); + return msg.key; +} +//static +std::string TrafixOrderStateHandler::new_id(const std::string& current_id) { + LOG(OrderMan, Warn, "new_id {}", current_id); + int update_number = getUpdateNumber(current_id)+1; + LOG(OrderMan, Warn, "new_id {}, update number {}", current_id,update_number); + + return std::to_string(update_number) + ":" + std::to_string(getOrderKey(current_id).value()); +} +OrderKey TrafixOrderStateHandler::getOrderKey(const std::string& id) { + std::size_t pos = id.find(':'); + if (pos == std::string::npos) return null_order_key(); + return OrderKey(std::stoi(id.substr(pos + 1))); +} +int TrafixOrderStateHandler::getUpdateNumber(const std::string& id) { + std::size_t pos = id.find(':'); + if(pos==std::string::npos || pos ==0){ + return 0; + } + std::string prefix = id.substr(0, pos); + bts_assert_msgf(std::all_of(prefix.begin(), prefix.end(), ::isdigit), "Failed to get update number id:{}",id); + return std::stoi(prefix); +} + +std::optional TrafixOrderStateHandler::getCreateOrderMsg(const OrderKey& key) const { + std::lock_guard lock(pimpl_->orders_mutex); + auto v = get_order_info(key); + if(v!=nullptr) return v->order_msg; + LOG(md, Error, "Order with key {} not found.", key); + return std::nullopt; +} +std::string TrafixOrderStateHandler::get_marex_key(const OrderKey &key) { + std::lock_guard lock(pimpl_->orders_mutex); + + auto v = get_order_info(key); + if(v!=nullptr) return v->current_id; + return ""; +} + +} // namespace b diff --git a/src/lib/orderman/gateways/TrafixOrderStateHandler.h b/src/lib/orderman/gateways/TrafixOrderStateHandler.h new file mode 100644 index 000000000..ea058065e --- /dev/null +++ b/src/lib/orderman/gateways/TrafixOrderStateHandler.h @@ -0,0 +1,61 @@ +#pragma once +#include "IFIXGenericAdapter.h" +#include "lib/orderman/CancelOrderMsg.h" +#include "lib/orderman/CreateOrderMsg.h" +#include "lib/orderman/gateways/OnixFIXListener.h" +#include "lib/orderman/order_factory/IOrderFactory.h" +#include "lib/orderman/orderman_utils/orderman_common.h" +#include "lib/utils/bts_assert.h" +#include +#include +#include +#include + +namespace bts::orderman { + +class TrafixOrderStateHandler { +public: + explicit TrafixOrderStateHandler(md::SecurityOracle sec_oracle); + ~TrafixOrderStateHandler(); + + OrderKey launchOrder(const CreateOrderMsg& msg, const std::string& clor_id); + std::optional getCreateOrderMsg(const OrderKey& key) const; + std::string get_marex_key(const OrderKey &key) ; + OrderKey modifyOrder(const CancelReplaceOrderMsg& msg, const std::string& clor_id); + OrderKey cancelOrder(const CancelOrderMsg& msg, const std::string& clor_id); + static std::string new_id(const std::string& current_id); + static OrderKey getOrderKey(const std::string& id); + static int getUpdateNumber(const std::string& id); + +private: + struct OrderInfo { + explicit OrderInfo(const CreateOrderMsg& msg, const std::string& id) + : current_id(id), order_msg(msg) {} + void update(const CancelReplaceOrderMsg& msg, const std::string& new_id) { + LOG(md,Info,"OrderInfo:update Order: {}, id:{}",msg,new_id); + + upd_msg = msg; + current_id = new_id; + } + + void cancel(const CancelOrderMsg& msg, const std::string& new_id) { + cancel_msg = msg; + current_id = new_id; + } + std::string current_id; + CreateOrderMsg order_msg; + CancelReplaceOrderMsg upd_msg; + CancelOrderMsg cancel_msg; + }; + OrderInfo* get_order_info(const OrderKey&key) const; + struct PImpl { + explicit PImpl(md::SecurityOracle sec_oracle) : sec_oracle(sec_oracle) {} + mutable std::mutex orders_mutex;//need to use this every time orders are accesed + //can be improved later on + std::unordered_map orders; + md::SecurityOracle sec_oracle; + }; + + std::unique_ptr pimpl_; +}; +} \ No newline at end of file From 715f7f7c70d4ecb2b96ce9c558cce265df42c36c Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 06:17:28 +0000 Subject: [PATCH 16/30] Fix: Race condition causes crash not sure if fixed --- src/server/sparkd/md/MarketDataGateway.cpp | 26 +++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/server/sparkd/md/MarketDataGateway.cpp b/src/server/sparkd/md/MarketDataGateway.cpp index 025cb05a5..3f8dd836c 100644 --- a/src/server/sparkd/md/MarketDataGateway.cpp +++ b/src/server/sparkd/md/MarketDataGateway.cpp @@ -247,10 +247,9 @@ void MarketDataGateway::unsubscribe(MarketDataListenerID id) { void MarketDataGateway::start_glbx(const bts::api::md::MarketDataConfig &cfg){ for (const auto &glbx : cfg.glbx()) { if(glbx.use_databento()){ - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_GLBX)); std::string tid_name = "glbx"; bts::threading::make_thread(tid_name, [&]() { + auto p = std::make_shared(bts::api::VENUE_GLBX); api::md::glbx::GlbxMarketDatahandler::run( p->listener, me_->sec_id_store, @@ -377,13 +376,10 @@ void MarketDataGateway::_start_replays( void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) { // taifex - int counter = 0; for (const auto &taifex : cfg.taifex()) { - counter++; - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_TAIFEX)); - std::string tid_name = "taifex" +std::to_string(counter); + std::string tid_name = "taifex"; bts::threading::make_thread(tid_name, [&]() { + auto p= std::make_shared(bts::api::VENUE_TAIFEX); bts::md::taifex::TaifexDataGateway::run( taifex, p->listener, me_->sec_id_store); }).detach(); @@ -392,8 +388,11 @@ void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) } void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg){ // cboe + // this has a race condition + //replace code in this function with the commented code to potentially see it system should crash under clangd release + /* for (const api::md::CboeMarketDataConfig &cboe : cfg.cboe()) { - auto p = me_->venue_listeners.emplace_back( + auto p = me_->venue_listeners.emplace_back( std::make_shared(bts::api::VENUE_CBOE)); std::string tid_name = "CBOE"; bts::threading::make_thread(tid_name, [&]() { @@ -402,6 +401,17 @@ void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg){ me_->sec_id_store, cboe); }).detach(); + } + */ + for (const api::md::CboeMarketDataConfig &cboe : cfg.cboe()) { + std::string tid_name = "CBOE"; + bts::threading::make_thread(tid_name, [&]() { + auto p = std::make_shared(bts::api::VENUE_CBOE); + api::md::cboe::CboeMarketDatahandler::run( + p->listener, + me_->sec_id_store, + cboe); + }).detach(); } } From 011372135569c489cad346c48edf6c23e8024008 Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 06:29:41 +0000 Subject: [PATCH 17/30] Add: Config for marex reconnect attemps --- .../profiles/default/marex_fix_cfg.conf | 2 + src/lib/config/protos/marex_fix_cfg.proto | 2 + src/lib/orderman/gateways/OnixFIXAdapter.cpp | 42 ++++++++++++------- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/lib/config/profiles/default/marex_fix_cfg.conf b/src/lib/config/profiles/default/marex_fix_cfg.conf index aee300492..c8863cc38 100644 --- a/src/lib/config/profiles/default/marex_fix_cfg.conf +++ b/src/lib/config/profiles/default/marex_fix_cfg.conf @@ -13,5 +13,7 @@ MarexFixCfg { client_id: "BTS" counterparty_host : "74.217.55.230" open_close :"O" + max_retries: 500 + retry_interval: 600 } } \ No newline at end of file diff --git a/src/lib/config/protos/marex_fix_cfg.proto b/src/lib/config/protos/marex_fix_cfg.proto index a5ec9b16b..7647e7492 100644 --- a/src/lib/config/protos/marex_fix_cfg.proto +++ b/src/lib/config/protos/marex_fix_cfg.proto @@ -25,4 +25,6 @@ message MarexFixCfg { uint32 customer_or_firm =11; string open_close =12; string name = 13; + int32 max_retries = 14; + uint32 retry_interval = 15; } \ No newline at end of file diff --git a/src/lib/orderman/gateways/OnixFIXAdapter.cpp b/src/lib/orderman/gateways/OnixFIXAdapter.cpp index c65484558..5e34b8571 100644 --- a/src/lib/orderman/gateways/OnixFIXAdapter.cpp +++ b/src/lib/orderman/gateways/OnixFIXAdapter.cpp @@ -56,21 +56,35 @@ OnixFIXAdapter::~OnixFIXAdapter() { } } void OnixFIXAdapter::start() { - try { - LOG(OrderMan, Warn, "OnixFIXAdapter::start logging on"); - auto onix_cfg = config::get(); - const std::string counterpartyHost = onix_cfg.counterparty_host(); - const int heartBtInt = onix_cfg.heart_bt_int(); - const int counterpartyPort = onix_cfg.counterparty_port(); - LOG(OrderMan, Diag, "OnixFIXAdapter::start logging on port: {}, ip: {}, heartbt:{}",counterpartyPort,heartBtInt,counterpartyHost); - session->logonAsInitiator(counterpartyHost, counterpartyPort, heartBtInt); - LOG(OrderMan, Warn, "OnixFIXAdapter::start logging on done"); + auto cfg = config::get(); + const std::string counterpartyHost = cfg.counterparty_host(); + const int heartBtInt = cfg.heart_bt_int(); + const int counterpartyPort = cfg.counterparty_port(); + const int retryInterval = cfg.retry_interval(); // Add this field in MarexFixCfg + const int maxRetries = cfg.max_retries(); // Optional: Set a limit on retries - } catch (const OnixS::FIX::Exception &ex) { - Engine::shutdown(); - const std::string err = ex.what(); - handleException("OnixFIXAdapter::start", err); - } + int attempt = 0; + + while (true) { + try { + LOG(OrderMan, Warn, "OnixFIXAdapter::start Attempting login (Attempt: {})", attempt + 1); + session->logonAsInitiator(counterpartyHost, counterpartyPort, heartBtInt); + LOG(OrderMan, Warn, "OnixFIXAdapter::start Login successful"); + break; // Exit loop on success + } catch (const OnixS::FIX::Exception &ex) { + const std::string err = ex.what(); + LOG(OrderMan, Error, "OnixFIXAdapter::start Login failed: {}. Retrying in {} seconds.", err, retryInterval); + + if (maxRetries > 0 && ++attempt >= maxRetries) { + Engine::shutdown(); + LOG(OrderMan, Error, "OnixFIXAdapter::start Exceeded max retries ({}). Aborting.", maxRetries); + handleException("OnixFIXAdapter::start", err); + break; // Exit if retries are exhausted + } + + sleep(fmin(60,retryInterval)); // Pause before retry + } + } } template uint translateEnum(T enum_value, const std::map &translationMap) { From b67d748ba18c4f8c73f6a01685b1f857cbb95378 Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 06:58:14 +0000 Subject: [PATCH 18/30] removed: moved onix glbx to main controler thread --- src/server/sparkd/md/MarketDataGateway.cpp | 27 ++++++++++++---------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/server/sparkd/md/MarketDataGateway.cpp b/src/server/sparkd/md/MarketDataGateway.cpp index 3f8dd836c..7fef3b888 100644 --- a/src/server/sparkd/md/MarketDataGateway.cpp +++ b/src/server/sparkd/md/MarketDataGateway.cpp @@ -139,10 +139,10 @@ void MarketDataGateway::run() { bts_assert(me_->md_config.emumba_deribit().size() == 1); _start_emumba_deribit(me_->md_config.emumba_deribit().at(0)); } - start_glbx(me_->md_config); start_taifex(me_->md_config); start_cboe(me_->md_config); _start_replays(me_->md_config); + start_glbx(me_->md_config); while (true) { std::this_thread::sleep_for(std::chrono::seconds(1)); } @@ -275,17 +275,20 @@ static void run_onixs() { main->start({cme_symbols.begin(), cme_symbols.end()}, mbo, tssd); platforms.emplace_back(main); } + while (true) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } // deribit // auto *d_platform = new bts::md::EmMDListener() // Timers - auto &disp = utils::Dispatcher::current(); - utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() { - for (auto &p : platforms) - p->send_heartbeats(); - }); - heartbeat_timer.start(); +// auto &disp = utils::Dispatcher::current(); +// utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() { +// for (auto &p : platforms) +// p->send_heartbeats(); +// }); +// heartbeat_timer.start(); - disp.run(); +// disp.run(); } void MarketDataGateway::_start_onix_glbx( @@ -294,10 +297,10 @@ void MarketDataGateway::_start_onix_glbx( "MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. " "cfg:\n{}", os::get_thread_id(), os::get_thread_name(), cfg.DebugString()); - bts::threading::make_thread("Onix CME", [&]() { - bts::md::run_onixs(); - }).detach(); - + // bts::threading::make_thread("Onix CME", [&]() { + + // }).detach(); + bts::md::run_onixs(); } From 76f18f34432091df2dbcf9275f9f14003ed599ae Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 20:17:29 +0000 Subject: [PATCH 19/30] Fix: typo in fixed securities --- src/lib/md/Shm/FixedSecurity.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index e8b5c3091..179ad5a76 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -374,8 +374,8 @@ void convert_security( dest.cme_market_segment_id = src.glbx().market_segment_id(); break; case Security::ExchangeDataCase::kCboe: - dest.numeric_id = src.deribit().exchange_id(); - dest.security_desc(src.deribit().security_desc()); + dest.numeric_id = src.cboe().exchange_id(); + dest.security_desc(src.cboe().security_desc()); dest.cboe_class_key = src.cboe().class_key(); break; case Security::ExchangeDataCase::kTaifex: From 98f8e58f3fe8bf4b06561eb7996beeab70df6d97 Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 22:12:06 +0000 Subject: [PATCH 20/30] Add optional tags to config --- src/lib/config/profiles/default/marex_fix_cfg.conf | 3 +++ src/lib/config/protos/marex_fix_cfg.proto | 3 +++ src/lib/orderman/gateways/OnixFIXAdapter.cpp | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/src/lib/config/profiles/default/marex_fix_cfg.conf b/src/lib/config/profiles/default/marex_fix_cfg.conf index c8863cc38..6e6aff44d 100644 --- a/src/lib/config/profiles/default/marex_fix_cfg.conf +++ b/src/lib/config/profiles/default/marex_fix_cfg.conf @@ -15,5 +15,8 @@ MarexFixCfg { open_close :"O" max_retries: 500 retry_interval: 600 + ex_destination : "W" + security_exchange : "CBOE" + currency : "USD" } } \ No newline at end of file diff --git a/src/lib/config/protos/marex_fix_cfg.proto b/src/lib/config/protos/marex_fix_cfg.proto index 7647e7492..9635aafb0 100644 --- a/src/lib/config/protos/marex_fix_cfg.proto +++ b/src/lib/config/protos/marex_fix_cfg.proto @@ -27,4 +27,7 @@ message MarexFixCfg { string name = 13; int32 max_retries = 14; uint32 retry_interval = 15; + string ex_destination = 16; + string security_exchange = 17; + string currency = 18; } \ No newline at end of file diff --git a/src/lib/orderman/gateways/OnixFIXAdapter.cpp b/src/lib/orderman/gateways/OnixFIXAdapter.cpp index 5e34b8571..3072342eb 100644 --- a/src/lib/orderman/gateways/OnixFIXAdapter.cpp +++ b/src/lib/orderman/gateways/OnixFIXAdapter.cpp @@ -1,6 +1,7 @@ #include "OnixFIXAdapter.h" #include "OnixS/FIXEngine/FIX/Engine.h" #include "lib/config/protos/marex_fix_cfg.pb.h" +#include "lib/domain_model/Security.pb.h" #include "lib/logging/logging.h" #include "lib/orderman/CancelOrderMsg.h" #include "lib/orderman/CancelReplaceOrderMsg.h" @@ -9,6 +10,7 @@ #include "lib/orderman/gateways/TrafixOrderStateHandler.h" #include "lib/orderman/orderman_utils/orderman_common.h" #include "lib/utils/bts_assert.h" +#include "lib/utils/helper_macros.h" #include #include #include @@ -211,6 +213,9 @@ void add_cfg_fields(Message &order, const config::MarexFixCfg &cfg) { .set(OnixS::FIX::FIX42::Tags::ClientID, cfg.client_id()) .set(OnixS::FIX::FIX42::Tags::CustomerOrFirm, cfg.customer_or_firm()) .set(OnixS::FIX::FIX42::Tags::OpenClose, cfg.open_close()); + order.set(FIX42::Tags::Currency,cfg.currency()); + order.set(FIX42::Tags::SecurityExchange,cfg.security_exchange()); + order.set(FIX42::Tags::ExDestination,cfg.ex_destination()); } void populate_common_order_fields(Message &order, const CreateOrderMsg &msg) { order.set(OnixS::FIX::FIX42::Tags::Account, msg.routing_record->account()) From bca8df6d9943df5c202b24cd3160fcd668114a1e Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 22:14:48 +0000 Subject: [PATCH 21/30] Fix: Release Crash fixed still has timer --- src/server/sparkd/md/MarketDataGateway.cpp | 102 ++++++++++----------- 1 file changed, 48 insertions(+), 54 deletions(-) diff --git a/src/server/sparkd/md/MarketDataGateway.cpp b/src/server/sparkd/md/MarketDataGateway.cpp index 7fef3b888..930a31672 100644 --- a/src/server/sparkd/md/MarketDataGateway.cpp +++ b/src/server/sparkd/md/MarketDataGateway.cpp @@ -139,10 +139,11 @@ void MarketDataGateway::run() { bts_assert(me_->md_config.emumba_deribit().size() == 1); _start_emumba_deribit(me_->md_config.emumba_deribit().at(0)); } + start_glbx(me_->md_config); + std::this_thread::sleep_for(std::chrono::seconds(5)); start_taifex(me_->md_config); start_cboe(me_->md_config); _start_replays(me_->md_config); - start_glbx(me_->md_config); while (true) { std::this_thread::sleep_for(std::chrono::seconds(1)); } @@ -246,20 +247,23 @@ void MarketDataGateway::unsubscribe(MarketDataListenerID id) { ////////////////////////////////////////////////////////////////////////////// void MarketDataGateway::start_glbx(const bts::api::md::MarketDataConfig &cfg){ for (const auto &glbx : cfg.glbx()) { - if(glbx.use_databento()){ + if(!glbx.use_databento()){ + _start_onix_glbx(glbx); + return; + } + auto p = me_->venue_listeners.emplace_back( + std::make_shared(bts::api::VENUE_GLBX)); + // Capture `p->listener` and `me_->sec_id_store` by value + auto listener = p->listener; + auto sec_id_store = me_->sec_id_store; std::string tid_name = "glbx"; - bts::threading::make_thread(tid_name, [&]() { - auto p = std::make_shared(bts::api::VENUE_GLBX); + bts::threading::make_thread(tid_name, [listener, sec_id_store, glbx](){ api::md::glbx::GlbxMarketDatahandler::run( - p->listener, - me_->sec_id_store, + listener, + sec_id_store, glbx); }).detach(); - } - else{ - _start_onix_glbx(glbx); - } - } + } } static void run_onixs() { LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name", @@ -275,20 +279,17 @@ static void run_onixs() { main->start({cme_symbols.begin(), cme_symbols.end()}, mbo, tssd); platforms.emplace_back(main); } - while (true) { - std::this_thread::sleep_for(std::chrono::seconds(1)); - } // deribit // auto *d_platform = new bts::md::EmMDListener() // Timers -// auto &disp = utils::Dispatcher::current(); -// utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() { -// for (auto &p : platforms) -// p->send_heartbeats(); -// }); -// heartbeat_timer.start(); + auto &disp = utils::Dispatcher::current(); + utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() { + for (auto &p : platforms) + p->send_heartbeats(); + }); + heartbeat_timer.start(); -// disp.run(); + disp.run(); } void MarketDataGateway::_start_onix_glbx( @@ -297,10 +298,10 @@ void MarketDataGateway::_start_onix_glbx( "MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. " "cfg:\n{}", os::get_thread_id(), os::get_thread_name(), cfg.DebugString()); - // bts::threading::make_thread("Onix CME", [&]() { - - // }).detach(); - bts::md::run_onixs(); + bts::threading::make_thread("Onix CME", [&]() { + bts::md::run_onixs(); + }).detach(); + } @@ -380,44 +381,37 @@ void MarketDataGateway::_start_replays( void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) { // taifex for (const auto &taifex : cfg.taifex()) { + auto p = me_->venue_listeners.emplace_back( + std::make_shared(bts::api::VENUE_TAIFEX)); + // Capture `p->listener` and `me_->sec_id_store` by value + auto listener = p->listener; + auto sec_id_store = me_->sec_id_store; std::string tid_name = "taifex"; - bts::threading::make_thread(tid_name, [&]() { - auto p= std::make_shared(bts::api::VENUE_TAIFEX); + bts::threading::make_thread(tid_name, [listener, sec_id_store, taifex]() { bts::md::taifex::TaifexDataGateway::run( - taifex, p->listener, me_->sec_id_store); + taifex, listener, sec_id_store); }).detach(); } } -void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg){ - // cboe - // this has a race condition - //replace code in this function with the commented code to potentially see it system should crash under clangd release - /* +void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg) { for (const api::md::CboeMarketDataConfig &cboe : cfg.cboe()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_CBOE)); - std::string tid_name = "CBOE"; - bts::threading::make_thread(tid_name, [&]() { - api::md::cboe::CboeMarketDatahandler::run( - p->listener, - me_->sec_id_store, - cboe); - }).detach(); - } - */ - for (const api::md::CboeMarketDataConfig &cboe : cfg.cboe()) { - std::string tid_name = "CBOE"; - bts::threading::make_thread(tid_name, [&]() { - auto p = std::make_shared(bts::api::VENUE_CBOE); - api::md::cboe::CboeMarketDatahandler::run( - p->listener, - me_->sec_id_store, - cboe); - }).detach(); + auto p = me_->venue_listeners.emplace_back( + std::make_shared(bts::api::VENUE_CBOE)); + std::string tid_name = "CBOE"; + + // Capture `p->listener` and `me_->sec_id_store` by value + auto listener = p->listener; + auto sec_id_store = me_->sec_id_store; + + bts::threading::make_thread(tid_name, [listener, sec_id_store, cboe]() { + api::md::cboe::CboeMarketDatahandler::run( + listener, + sec_id_store, + cboe); + }).detach(); } } - } // namespace bts::md From 84c0684b63d6cd3b5a1590d4ef9e5dd92c377420 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sat, 28 Sep 2024 16:24:45 +0000 Subject: [PATCH 22/30] refactor: Normalize sec key indexing for new (domain) md sources This is part of a larger effort to unify the way we index securities in order to generate unique ids. Older md sources (e.g., MDP3Listener) use SecurityIDRecorder to generate ids, which uses a different database and different key-string formatting than the IDGenerator used by the newer md sources (e.g., GeneralMarketDataListener and various ccapi-based sources). --- src/lib/md/GeneralMarketDataListener.cpp | 27 +++-- src/lib/persistence/IDGenerator.cpp | 23 +++- src/lib/persistence/IDGenerator.h | 13 ++- .../binance/BinancePriceStatisticsAdaptor.cpp | 10 +- .../binance/BinancePriceStatisticsAdaptor.h | 7 +- .../md/binance/BinanceSecurityRequestor.cpp | 4 +- .../sparkd/md/binance/BinanceSubscription.cpp | 56 +++++---- .../BinanceUsPriceStatisticsAdaptor.cpp | 10 +- .../BinanceUsPriceStatisticsAdaptor.h | 7 +- .../binance_us/BinanceUsSecurityRequestor.cpp | 3 +- .../md/binance_us/BinanceUsSubscription.cpp | 55 +++++---- .../BinanceUsdsFuturesSecurityRequestor.cpp | 4 +- .../BinanceUsdsFuturesSubscription.cpp | 36 +++--- .../md/coinbase/CoinbaseSecurityRequestor.cpp | 3 +- .../md/coinbase/CoinbaseSubscription.cpp | 36 ++++-- .../md/databento/DatabentoDataHandler.cpp | 108 +++++++++++------- .../md/deribit/DeribitInstrumentStore.cpp | 21 ++-- .../md/deribit/DeribitInstrumentStore.h | 5 +- .../sparkd/md/deribit/DeribitSubscription.cpp | 31 +++-- .../sparkd/md/ftx/FTXSecurityRequestor.cpp | 4 +- src/server/sparkd/md/ftx/FTXSubscription.cpp | 31 +++-- .../sparkd/md/okx/OkxInstrumentStore.cpp | 11 +- src/server/sparkd/md/okx/OkxInstrumentStore.h | 3 +- src/server/sparkd/md/okx/OkxSubscription.cpp | 21 ++-- .../md/platforms/emumba/EmMDListener.cpp | 26 ++++- .../sparkd/md/taifex/TaifexDataProcessors.h | 19 ++- 26 files changed, 366 insertions(+), 208 deletions(-) diff --git a/src/lib/md/GeneralMarketDataListener.cpp b/src/lib/md/GeneralMarketDataListener.cpp index 54058c5e4..e9aaea151 100644 --- a/src/lib/md/GeneralMarketDataListener.cpp +++ b/src/lib/md/GeneralMarketDataListener.cpp @@ -23,12 +23,12 @@ #include "lib/domain_model/Security.pb.h" #include "lib/domain_model/common.h" #include "lib/logging/logging.h" +#include "lib/md/GeneralSecurityIndex.h" #include "lib/md/IMarketDataListener.h" #include "lib/md/IntPrice.h" #include "lib/md/MDProducer.h" #include "lib/md/MDProducerHandle.h" #include "lib/md/PriceSize.h" -#include "lib/md/SecurityIDRecorder.h" #include "lib/md/SecurityIndex.h" #include "lib/md/Shm/FixedSecurity.h" #include "lib/md/Shm/MDStructure.h" @@ -274,10 +274,11 @@ bool update_incremental(const IMarketDataListener::BookOp &op) { }; struct GeneralMarketDataListener::PImpl { + const api::domain::Venue venue; // if this is set to true, then the security key passed on in the // on_securities function is used. If this is not set to true, then a new key // is generated - const bool use_given_sec_key = true; + const bool use_given_sec_key = false; const std::string cc_name_prefix = "cc_security"; size_t n_bk_updates = 0; bts::persistence::IDGenerator &id_gen = @@ -296,12 +297,15 @@ struct GeneralMarketDataListener::PImpl { MDArray asks = {}; IncrementalBookData inc_book_data = {}; explicit PImpl( + api::domain::Venue venue, GeneralSecurityIndex &index, MDProducer &md_producer, std::shared_ptr sim_ex_gwy) - : index(index), md_producer(md_producer), sim_ex_gwy(sim_ex_gwy) { - if (sim_ex_gwy != nullptr) { - sim_ex_gwy->set_producers(&index); + : venue(venue), index(index), + md_producer(md_producer), + sim_ex_gwy(std::move(sim_ex_gwy)) { + if (this->sim_ex_gwy != nullptr) { + this->sim_ex_gwy->set_producers(&this->index); } } }; @@ -363,6 +367,7 @@ void copy_books( GeneralMarketDataListener::GeneralMarketDataListener( GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue) : me(std::make_unique( + domain::DomainSecurityUtils::convert(venue), index, md_producer, bts::orderman::SimExecutionGatewayFactory::get()->get_getayeway( @@ -379,7 +384,7 @@ void GeneralMarketDataListener::on_securities( LOG(md, Info, "GeneralMarketDataListener::on_securities n_secs: {}", complete_security_list.size()); for (auto s_ptr : complete_security_list) { - domain::Security sec = *s_ptr; + const domain::Security &sec = *s_ptr; const auto &s_comm = sec.security_common; CCSecSearchResult search_result = _find_sec(sec, true); LOG(md, Info, @@ -503,11 +508,7 @@ CCSecSearchResult GeneralMarketDataListener::_find_sec( if (add_if_absent) { uint64_t bts_id = cc_id.value(); if (!me->use_given_sec_key) { - auto logical_key = get_logical_key(sec); - auto new_key_opt = - me->id_gen.get_id(me->cc_name_prefix, logical_key, true); - bts_assert(new_key_opt.has_value()); - bts_id = new_key_opt.value(); + bts_id = me->id_gen.get_or_add(me->venue, sec.type(), sec.desc()); } me->cc_sec_id_map.insert({cc_id, {bts_id}}); LOG(md, Info, @@ -654,9 +655,7 @@ uint64_t GeneralMarketDataListener::_generate_security_key( auto logical_key = get_logical_key(sec); // if (me->use_given_sec_key) // return sec.security_common.id.value(); - auto rv = me->id_gen.get_id(me->cc_name_prefix, logical_key, true); - bts_assert(rv.has_value()); - return rv.value(); + return me->id_gen.get_or_add(me->venue, sec.type(), sec.desc()); } void GeneralMarketDataListener::_possibly_match_with_sim_orders( diff --git a/src/lib/persistence/IDGenerator.cpp b/src/lib/persistence/IDGenerator.cpp index 569181eef..2ee855258 100644 --- a/src/lib/persistence/IDGenerator.cpp +++ b/src/lib/persistence/IDGenerator.cpp @@ -9,6 +9,8 @@ namespace bts::persistence { +using bts::api::domain::SecurityType; +using bts::api::domain::SecurityType_Name; using bts::api::domain::Venue; using bts::api::domain::Venue_Name; using std::string; @@ -52,8 +54,14 @@ uint64_t get_next_id( } std::optional IDGenerator::get_id( - const Venue &venue, const string &name, bool add_if_new) { - return get_id(Venue_Name(venue), name, add_if_new); + const Venue &venue, + SecurityType type, + const string &name, + bool add_if_new) { + std::string ns = fmt::format( + "{}.{}", Venue_Name(venue).substr(6), + SecurityType_Name(type).substr(12)); + return get_id(ns, name, add_if_new); } std::optional IDGenerator::get_id( @@ -80,10 +88,6 @@ std::optional IDGenerator::get_id( } } -std::uint64_t IDGenerator::get_or_add(const Venue &venue, const string &name) { - return get_or_add(Venue_Name(venue), name); -} - std::uint64_t IDGenerator::get_or_add( const std::string &ns, const std::string &key) { auto id = get_id(ns, key, true); @@ -91,6 +95,13 @@ std::uint64_t IDGenerator::get_or_add( return *id; } +std::uint64_t IDGenerator::get_or_add( + api::domain::Venue venue, + api::domain::SecurityType type, + const std::string &name) { + return *get_id(venue, type, name, true); +} + IDGenerator::~IDGenerator() {} } // namespace bts::persistence diff --git a/src/lib/persistence/IDGenerator.h b/src/lib/persistence/IDGenerator.h index 364b3c369..908997aee 100644 --- a/src/lib/persistence/IDGenerator.h +++ b/src/lib/persistence/IDGenerator.h @@ -1,6 +1,8 @@ #pragma once +#include "lib/api/domain/SecurityType.pb.h" #include "lib/api/domain/Venue.pb.h" + #include #include #include @@ -21,6 +23,7 @@ class IDGenerator { std::optional get_id( const api::domain::Venue &venue, + api::domain::SecurityType type, const std::string &name, bool add_if_new = false); @@ -28,9 +31,9 @@ class IDGenerator { const std::string &ns, const std::string &key, bool add_if_new); std::uint64_t get_or_add( - const api::domain::Venue &venue, const std::string &name); - - std::uint64_t get_or_add(const std::string &ns, const std::string &key); + api::domain::Venue venue, + api::domain::SecurityType type, + const std::string &name); private: static std::shared_ptr instance_ptr_; @@ -38,6 +41,10 @@ class IDGenerator { struct PImpl; std::unique_ptr me; + + // These methods are private to force callers to use the [public] overloads with venue, type and + // name params, to ensure consistent formatting. + std::uint64_t get_or_add(const std::string &ns, const std::string &key); }; } // namespace bts::persistence diff --git a/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.cpp b/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.cpp index 3561d8247..d69533a2f 100644 --- a/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.cpp +++ b/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.cpp @@ -10,20 +10,21 @@ namespace bts::md::binance { +using bts::api::domain::SecurityType; using bts::domain::SecurityID; using bts::domain::SecurityMarketDataStatistics; -using std::pair; using std::string; +using std::tuple; using std::vector; #define VENUE_NAME (Venue_Name(Venue::BINANCE)) struct BinancePriceStatisticsAdaptor::PImpl { - vector> securities; + vector> securities; IMarketDataListener *listener; SecurityID get_sec_id(const string &sec_desc) const { - for (const auto &[desc, id] : securities) { + for (const auto &[type, desc, id] : securities) { if (desc == sec_desc) { return id; } @@ -36,7 +37,8 @@ struct BinancePriceStatisticsAdaptor::PImpl { }; BinancePriceStatisticsAdaptor::BinancePriceStatisticsAdaptor( - vector> securities, IMarketDataListener *listener) + vector> securities, + IMarketDataListener *listener) : me_(std::make_unique()) { bts_assert(listener != nullptr); me_->securities = std::move(securities); diff --git a/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.h b/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.h index 3b3694fc6..280deaf34 100644 --- a/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.h +++ b/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.h @@ -1,9 +1,11 @@ #pragma once +#include "lib/api/domain/SecurityType.pb.h" #include "lib/domain/SecurityID.h" #include "server/sparkd/md/ccapi/ICCApiMessageHandler.h" #include +#include #include #include @@ -20,7 +22,10 @@ namespace bts::md::binance { class BinancePriceStatisticsAdaptor : public ICCApiMessageHandler { public: BinancePriceStatisticsAdaptor( - std::vector> securities, + std::vector> securities, IMarketDataListener *listener); ~BinancePriceStatisticsAdaptor(); diff --git a/src/server/sparkd/md/binance/BinanceSecurityRequestor.cpp b/src/server/sparkd/md/binance/BinanceSecurityRequestor.cpp index 38c3c9a85..d9483fb86 100644 --- a/src/server/sparkd/md/binance/BinanceSecurityRequestor.cpp +++ b/src/server/sparkd/md/binance/BinanceSecurityRequestor.cpp @@ -49,6 +49,7 @@ using std::vector; #define _WARN(...) _LOG(Warn, __VA_ARGS__) #define _ERROR(...) _LOG(Error, __VA_ARGS__) +#define VENUE Venue::VENUE_BINANCE #define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE #define BINANCE_MSG_TYPE_NAME(x) \ @@ -156,7 +157,8 @@ struct BinanceSecurityRequestor::PImpl { for (auto &sec : added_secs) { // Since the ids haven't been set yet, we should expect them to be 0 bts_assert(sec.id().value() == 0); - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + SecurityID sec_id{ + sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())}; sec.security_common.id = sec_id; } } diff --git a/src/server/sparkd/md/binance/BinanceSubscription.cpp b/src/server/sparkd/md/binance/BinanceSubscription.cpp index 25a01ec32..bead58867 100644 --- a/src/server/sparkd/md/binance/BinanceSubscription.cpp +++ b/src/server/sparkd/md/binance/BinanceSubscription.cpp @@ -34,6 +34,7 @@ namespace bts::md::binance { +using bts::api::domain::SecurityType; using bts::api::domain::Venue; using bts::api::domain::Venue_Name; using bts::api::md::BinanceMarketDataConfig; @@ -50,6 +51,7 @@ using std::unique_ptr; using std::unordered_map; using std::vector; +#define VENUE Venue::VENUE_BINANCE #define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE @@ -123,14 +125,15 @@ static CCApiCorrelationID mk_stats_corr_id(const string &desc) { } static ccapi::Request mk_stats_req( - const vector &sec_descs, uint32_t batch_id) { + const vector> &sec_descs, + uint32_t batch_id) { const auto corr_id = mk_stats_corr_id(FMT("BATCH_{}", batch_id)); ccapi::Request req{ ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, CCAPI_VENUE_NAME}; req.setCorrelationId(corr_id.to_string()); auto quoted_sec_descs = - sec_descs | - ranges::views::transform([](const auto &s) { return "\"" + s + "\""; }); + sec_descs | ranges::views::transform( + [](const auto &s) { return "\"" + s.second + "\""; }); const string symbols_query_arg = FMT_JOIN(quoted_sec_descs, ","); req.appendParam( {{CCAPI_HTTP_PATH, BINANCE_PRICE_STATS_PATH}, @@ -209,16 +212,18 @@ struct BinanceSubscription::PImpl { sec_requestor.request(); } - DepthHandler &mk_depth_handler(const string &sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + DepthHandler &mk_depth_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = depth_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert(is_new); return it->second; } - TickerHandler &mk_ticker_handler(const string &sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + TickerHandler &mk_ticker_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = ticker_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -227,23 +232,25 @@ struct BinanceSubscription::PImpl { } StatsHandler &mk_stats_handler( - const vector &sec_descs, uint32_t hnd_id) { - vector> sec_desc_id_pairs; - sec_desc_id_pairs.reserve(sec_descs.size()); - for (const auto &sec_desc : sec_descs) { - sec_desc_id_pairs.emplace_back( - sec_desc, - SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}); + const vector> &sec_descs, + uint32_t hnd_id) { + vector> sec_desc_id_tuples; + sec_desc_id_tuples.reserve(sec_descs.size()); + for (auto [type, name] : sec_descs) { + SecurityID id{sec_id_store->get_or_add(VENUE, type, name)}; + sec_desc_id_tuples.emplace_back(type, std::move(name), id); } auto [it, is_new] = stats_handlers.try_emplace( - hnd_id, std::make_unique(sec_desc_id_pairs, listener)); + hnd_id, + std::make_unique(sec_desc_id_tuples, listener)); bts_assert_msgf( is_new, "StatsHandler already created with id '{}'", hnd_id); return *it->second; } void add_stats_handler( - const vector &sec_descs, CCApiSession::BatchRequest &reqs) { + const vector> &sec_descs, + CCApiSession::BatchRequest &reqs) { if (sec_descs.empty()) { return; } @@ -268,14 +275,14 @@ struct BinanceSubscription::PImpl { CCApiSession::BatchRequest stats_reqs; const int N_SECS_PER_STATS_REQ = 16; - vector curr_stats_sec_descs; - for (const auto &sec_desc : securities) { + vector> curr_stats_sec_descs; + for (const auto &[type, desc] : securities) { depth_subs.push_back( - {mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)}); + {mk_depth_sub(desc, depth_opts), &mk_depth_handler(type, desc)}); ticker_subs.push_back( - {mk_ticker_sub(sec_desc, ticker_opts), - &mk_ticker_handler(sec_desc)}); - curr_stats_sec_descs.push_back(sec_desc); + {mk_ticker_sub(desc, ticker_opts), + &mk_ticker_handler(type, desc)}); + curr_stats_sec_descs.push_back(std::make_pair(type, desc)); if (curr_stats_sec_descs.size() == N_SECS_PER_STATS_REQ) { add_stats_handler(curr_stats_sec_descs, stats_reqs); curr_stats_sec_descs.clear(); @@ -299,7 +306,7 @@ struct BinanceSubscription::PImpl { */ void on_securities(vector secs) { // 'sec_descs' is used to make depth subscriptions - vector sec_descs; + vector> sec_descs; sec_descs.reserve(secs.size()); // 'sec_ptrs' is passed to IMarketDataListeners vector sec_ptrs; @@ -312,7 +319,8 @@ struct BinanceSubscription::PImpl { id, std::make_unique(std::move(s))); bts_assert_msgf( is_new, "Received duplicate Binance security '{}'", desc); - sec_descs.push_back(std::move(desc)); + sec_descs.push_back( + std::make_pair(it->second->type(), std::move(desc))); // add ptr to entry in 'securities' (not the loop var, we moved that) sec_ptrs.push_back(it->second.get()); } diff --git a/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp b/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp index 301e25b1c..954106e1c 100644 --- a/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp +++ b/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp @@ -10,20 +10,21 @@ namespace bts::md::binance_us { +using bts::api::domain::SecurityType; using bts::domain::SecurityID; using bts::domain::SecurityMarketDataStatistics; -using std::pair; using std::string; +using std::tuple; using std::vector; #define VENUE_NAME (Venue_Name(Venue::BINANCE)) struct BinanceUsPriceStatisticsAdaptor::PImpl { - vector> securities; + vector> securities; IMarketDataListener *listener; SecurityID get_sec_id(const string &sec_desc) const { - for (const auto &[desc, id] : securities) { + for (const auto &[type, desc, id] : securities) { if (desc == sec_desc) { return id; } @@ -36,7 +37,8 @@ struct BinanceUsPriceStatisticsAdaptor::PImpl { }; BinanceUsPriceStatisticsAdaptor::BinanceUsPriceStatisticsAdaptor( - vector> securities, IMarketDataListener *listener) + vector> securities, + IMarketDataListener *listener) : me_(std::make_unique()) { bts_assert(listener != nullptr); me_->securities = std::move(securities); diff --git a/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.h b/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.h index 2caadce6b..a0af77602 100644 --- a/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.h +++ b/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.h @@ -1,9 +1,11 @@ #pragma once +#include "lib/api/domain/SecurityType.pb.h" #include "lib/domain/SecurityID.h" #include "server/sparkd/md/ccapi/ICCApiMessageHandler.h" #include +#include #include #include @@ -25,7 +27,10 @@ namespace bts::md::binance_us { class BinanceUsPriceStatisticsAdaptor : public ICCApiMessageHandler { public: BinanceUsPriceStatisticsAdaptor( - std::vector> securities, + std::vector> securities, IMarketDataListener *listener); ~BinanceUsPriceStatisticsAdaptor(); diff --git a/src/server/sparkd/md/binance_us/BinanceUsSecurityRequestor.cpp b/src/server/sparkd/md/binance_us/BinanceUsSecurityRequestor.cpp index 3288e7327..620b861a8 100644 --- a/src/server/sparkd/md/binance_us/BinanceUsSecurityRequestor.cpp +++ b/src/server/sparkd/md/binance_us/BinanceUsSecurityRequestor.cpp @@ -156,7 +156,8 @@ struct BinanceUsSecurityRequestor::PImpl { for (auto &sec : added_secs) { // Since the ids haven't been set yet, we should expect them to be 0 bts_assert(sec.id().value() == 0); - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + SecurityID sec_id{ + sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())}; sec.security_common.id = sec_id; } } diff --git a/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp b/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp index 6d841cdd4..a582f1eb9 100644 --- a/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp +++ b/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp @@ -36,6 +36,7 @@ namespace bts::md::binance_us { +using bts::api::domain::SecurityType; using bts::api::domain::Venue; using bts::api::domain::Venue_Name; using bts::api::md::BinanceUsMarketDataConfig; @@ -47,11 +48,13 @@ using bts::io::QueryParams; using bts::persistence::IDGenerator; using std::pair; using std::string; +using std::tuple; using std::uint32_t; using std::unique_ptr; using std::unordered_map; using std::vector; +#define VENUE Venue::VENUE_BINANCE_US #define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_US)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_US @@ -126,14 +129,14 @@ static CCApiCorrelationID mk_stats_corr_id(const string &desc) { } static ccapi::Request mk_stats_req( - const vector &sec_descs, uint32_t batch_id) { + const vector> &sec_descs, uint32_t batch_id) { const auto corr_id = mk_stats_corr_id(FMT("BATCH_{}", batch_id)); ccapi::Request req{ ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, CCAPI_VENUE_NAME}; req.setCorrelationId(corr_id.to_string()); auto quoted_sec_descs = - sec_descs | - ranges::views::transform([](const auto &s) { return "\"" + s + "\""; }); + sec_descs | ranges::views::transform( + [](const auto &s) { return "\"" + s.second + "\""; }); const string symbols_query_arg = FMT_JOIN(quoted_sec_descs, ","); req.appendParam( {{CCAPI_HTTP_PATH, BINANCE_US_PRICE_STATS_PATH}, @@ -212,16 +215,18 @@ struct BinanceUsSubscription::PImpl { sec_requestor.request(); } - DepthHandler &mk_depth_handler(const string &sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + DepthHandler &mk_depth_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = depth_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert(is_new); return it->second; } - TickerHandler &mk_ticker_handler(const string &sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + TickerHandler &mk_ticker_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = ticker_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -230,23 +235,25 @@ struct BinanceUsSubscription::PImpl { } StatsHandler &mk_stats_handler( - const vector &sec_descs, uint32_t hnd_id) { - vector> sec_desc_id_pairs; - sec_desc_id_pairs.reserve(sec_descs.size()); - for (const auto &sec_desc : sec_descs) { - sec_desc_id_pairs.emplace_back( - sec_desc, - SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}); + const vector> &sec_descs, + uint32_t hnd_id) { + vector> sec_desc_id_tuples; + sec_desc_id_tuples.reserve(sec_descs.size()); + for (auto [type, name] : sec_descs) { + SecurityID id{sec_id_store->get_or_add(VENUE, type, name)}; + sec_desc_id_tuples.emplace_back(type, std::move(name), id); } auto [it, is_new] = stats_handlers.try_emplace( - hnd_id, std::make_unique(sec_desc_id_pairs, listener)); + hnd_id, + std::make_unique(sec_desc_id_tuples, listener)); bts_assert_msgf( is_new, "StatsHandler already created with id '{}'", hnd_id); return *it->second; } void add_stats_handler( - const vector &sec_descs, CCApiSession::BatchRequest &reqs) { + const vector> &sec_descs, + CCApiSession::BatchRequest &reqs) { if (sec_descs.empty()) { return; } @@ -271,14 +278,14 @@ struct BinanceUsSubscription::PImpl { CCApiSession::BatchRequest stats_reqs; const int N_SECS_PER_STATS_REQ = 16; - vector curr_stats_sec_descs; - for (const auto &sec_desc : securities) { + vector> curr_stats_sec_descs; + for (const auto &[type, name] : securities) { depth_subs.push_back( - {mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)}); + {mk_depth_sub(name, depth_opts), &mk_depth_handler(type, name)}); ticker_subs.push_back( - {mk_ticker_sub(sec_desc, ticker_opts), - &mk_ticker_handler(sec_desc)}); - curr_stats_sec_descs.push_back(sec_desc); + {mk_ticker_sub(name, ticker_opts), + &mk_ticker_handler(type, name)}); + curr_stats_sec_descs.push_back(std::make_pair(type, name)); if (curr_stats_sec_descs.size() == N_SECS_PER_STATS_REQ) { add_stats_handler(curr_stats_sec_descs, stats_reqs); curr_stats_sec_descs.clear(); @@ -302,7 +309,7 @@ struct BinanceUsSubscription::PImpl { */ void on_securities(vector secs) { // 'sec_descs' is used to make depth subscriptions - vector sec_descs; + vector> sec_descs; sec_descs.reserve(secs.size()); // 'sec_ptrs' is passed to IMarketDataListeners vector sec_ptrs; @@ -315,7 +322,7 @@ struct BinanceUsSubscription::PImpl { id, std::make_unique(std::move(s))); bts_assert_msgf( is_new, "Received duplicate Binance US security '{}'", desc); - sec_descs.push_back(std::move(desc)); + sec_descs.push_back(std::make_pair(s.type(), std::move(desc))); // add ptr to entry in 'securities' (not the loop var, we moved that) sec_ptrs.push_back(it->second.get()); } diff --git a/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSecurityRequestor.cpp b/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSecurityRequestor.cpp index f65506e89..8f78c2f11 100644 --- a/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSecurityRequestor.cpp +++ b/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSecurityRequestor.cpp @@ -48,6 +48,7 @@ using std::vector; #define _WARN(...) _LOG(Warn, __VA_ARGS__) #define _ERROR(...) _LOG(Error, __VA_ARGS__) +#define VENUE Venue::VENUE_BINANCE_USDS_FUTURES #define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_USDS_FUTURES)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_USDS_FUTURES #define VENUE_PRICE_STATS_PATH BINANCE_USDS_FUTURES_PRICE_STATS_PATH @@ -157,7 +158,8 @@ struct BinanceUsdsFuturesSecurityRequestor::PImpl { // the ids, if it does (for whatever reason), better to assert/crash // than quietly let a potential bug slip through the cracks. bts_assert(sec.id().value() == 0); - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + SecurityID sec_id{ + sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())}; sec.security_common.id = sec_id; } } diff --git a/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp b/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp index d1fb9f47f..9dbbb247f 100644 --- a/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp +++ b/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp @@ -30,6 +30,7 @@ namespace bts::md::binance_usds_futures { +using bts::api::domain::SecurityType; using bts::api::domain::Venue; using bts::api::domain::Venue_Name; using bts::api::md::BinanceUsdsFuturesMarketDataConfig; @@ -39,12 +40,14 @@ using bts::domain::SecurityID; using bts::io::ProtoPrinter; using bts::io::QueryParams; using bts::persistence::IDGenerator; +using std::pair; using std::string; using std::uint32_t; using std::unique_ptr; using std::unordered_map; using std::vector; +#define VENUE Venue::VENUE_BINANCE_USDS_FUTURES #define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_USDS_FUTURES)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_USDS_FUTURES #define VENUE_PRICE_STATS_PATH BINANCE_USDS_FUTURES_PRICE_STATS_PATH @@ -192,8 +195,8 @@ struct BinanceUsdsFuturesSubscription::PImpl { sec_requestor.request(); } - SecurityID get_sec_id(const string &sec_desc) { - return SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + SecurityID get_sec_id(SecurityType sec_type, const string &sec_desc) { + return SecurityID{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; } /** @@ -211,8 +214,9 @@ struct BinanceUsdsFuturesSubscription::PImpl { return sec_desc; } - DepthHandler &add_depth_handler(const string &sec_desc) { - const auto sec_id = get_sec_id(sec_desc); + DepthHandler &add_depth_handler( + SecurityType sec_type, const string &sec_desc) { + const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = depth_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -220,8 +224,9 @@ struct BinanceUsdsFuturesSubscription::PImpl { return it->second; } - TickerHandler &add_ticker_handler(const string &sec_desc) { - const auto sec_id = get_sec_id(sec_desc); + TickerHandler &add_ticker_handler( + SecurityType sec_type, const string &sec_desc) { + const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = ticker_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -229,8 +234,9 @@ struct BinanceUsdsFuturesSubscription::PImpl { return it->second; } - StatsHandler &add_stats_handler(const string &sec_desc) { - const auto sec_id = get_sec_id(sec_desc); + StatsHandler &add_stats_handler( + SecurityType sec_type, const string &sec_desc) { + const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = stats_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -252,16 +258,16 @@ struct BinanceUsdsFuturesSubscription::PImpl { CCApiSession::BatchSubscription ticker_subs; CCApiSession::BatchRequest stats_reqs; - for (const auto &sec_desc : securities) { - const auto sec_desc_for_reqs = get_sec_desc_for_reqs(sec_desc); + for (auto [type, name] : securities) { + const auto sec_desc_for_reqs = get_sec_desc_for_reqs(name); depth_subs.push_back( {mk_depth_sub(sec_desc_for_reqs, depth_opts), - &add_depth_handler(sec_desc)}); + &add_depth_handler(type, name)}); ticker_subs.push_back( {mk_ticker_sub(sec_desc_for_reqs, ticker_opts), - &add_ticker_handler(sec_desc)}); + &add_ticker_handler(type, name)}); stats_reqs.push_back( - {mk_stats_req(sec_desc), &add_stats_handler(sec_desc)}); + {mk_stats_req(name), &add_stats_handler(type, name)}); } session.subscribe( @@ -278,7 +284,7 @@ struct BinanceUsdsFuturesSubscription::PImpl { */ void on_securities(vector secs) { // 'sec_descs' is used to make depth subscriptions - vector sec_descs; + vector> sec_descs; sec_descs.reserve(secs.size()); // 'sec_ptrs' is passed to IMarketDataListeners vector sec_ptrs; @@ -291,7 +297,7 @@ struct BinanceUsdsFuturesSubscription::PImpl { id, std::make_unique(std::move(s))); bts_assert_msgf( is_new, "Received duplicate Binance USDS Future '{}'", desc); - sec_descs.push_back(std::move(desc)); + sec_descs.push_back(std::make_pair(s.type(), std::move(desc))); // add ptr to entry in 'securities' (not the loop var, we moved that) sec_ptrs.push_back(it->second.get()); } diff --git a/src/server/sparkd/md/coinbase/CoinbaseSecurityRequestor.cpp b/src/server/sparkd/md/coinbase/CoinbaseSecurityRequestor.cpp index 4742a7fd3..d7e8b37c2 100644 --- a/src/server/sparkd/md/coinbase/CoinbaseSecurityRequestor.cpp +++ b/src/server/sparkd/md/coinbase/CoinbaseSecurityRequestor.cpp @@ -147,7 +147,8 @@ struct CoinbaseSecurityRequestor::PImpl { for (auto &sec : added_secs) { // Since the ids haven't been set yet, we should expect them to be 0 bts_assert(sec.id().value() == 0); - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + SecurityID sec_id{ + sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())}; sec.security_common.id = sec_id; } } diff --git a/src/server/sparkd/md/coinbase/CoinbaseSubscription.cpp b/src/server/sparkd/md/coinbase/CoinbaseSubscription.cpp index d7b311210..5fc29eb42 100644 --- a/src/server/sparkd/md/coinbase/CoinbaseSubscription.cpp +++ b/src/server/sparkd/md/coinbase/CoinbaseSubscription.cpp @@ -25,6 +25,9 @@ #include namespace bts::md::coinbase { + +using bts::api::domain::SecurityType; +using bts::api::domain::Venue; using bts::api::md::CoinbaseMarketDataConfig; using bts::domain::Security; using bts::domain::SecurityCriteria; @@ -32,10 +35,13 @@ using bts::domain::SecurityID; using bts::io::ProtoPrinter; using bts::io::QueryParams; using bts::persistence::IDGenerator; +using std::pair; using std::string; using std::unique_ptr; using std::unordered_map; using std::vector; + +#define VENUE Venue::VENUE_COINBASE #define VENUE_NAME (Venue_Name(Venue::VENUE_COINBASE)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_COINBASE @@ -183,24 +189,29 @@ struct CoinbaseSubscription::PImpl { sec_requestor.callback().bind<&PImpl::on_securities>(*this); sec_requestor.request(); } - StatsHandler &mk_stats_handler(const string &sec_desc) { - auto sec_id = SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + + StatsHandler &mk_stats_handler( + SecurityType sec_type, const string &sec_desc) { + auto sec_id = + SecurityID{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = stats_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( is_new, "StatsHandler already created with id '{}'", sec_id); return it->second; } - DepthHandler &mk_depth_handler(const string &sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + DepthHandler &mk_depth_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = depth_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert(is_new); return it->second; } - TickerHandler &mk_ticker_handler(const string &sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + TickerHandler &mk_ticker_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = ticker_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -222,14 +233,15 @@ struct CoinbaseSubscription::PImpl { CCApiSession::BatchRequest stats_reqs; vector curr_stats_sec_descs; - for (const auto &sec_desc : securities) { + for (const auto &[sec_type, sec_desc] : securities) { depth_subs.push_back( - {mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)}); + {mk_depth_sub(sec_desc, depth_opts), + &mk_depth_handler(sec_type, sec_desc)}); ticker_subs.push_back( {mk_ticker_sub(sec_desc, ticker_opts), - &mk_ticker_handler(sec_desc)}); + &mk_ticker_handler(sec_type, sec_desc)}); stats_reqs.push_back( - {mk_stats_req(sec_desc), &mk_stats_handler(sec_desc)}); + {mk_stats_req(sec_desc), &mk_stats_handler(sec_type, sec_desc)}); } session.subscribe( @@ -247,7 +259,7 @@ struct CoinbaseSubscription::PImpl { void on_securities(std::vector secs) { LOG(md, Info, "on sec {}", secs.size()); // 'sec_descs' is used to make depth subscriptions - vector sec_descs; + vector> sec_descs; sec_descs.reserve(secs.size()); // 'sec_ptrs' is passed to IMarketDataListeners vector sec_ptrs; @@ -260,7 +272,7 @@ struct CoinbaseSubscription::PImpl { id, std::make_unique(std::move(s))); bts_assert_msgf( is_new, "Received duplicate Coinbase security '{}'", desc); - sec_descs.push_back(std::move(desc)); + sec_descs.push_back(std::make_pair(s.type(), std::move(desc))); // add ptr to entry in 'securities' (not the loop var, we moved that) sec_ptrs.push_back(it->second.get()); } diff --git a/src/server/sparkd/md/databento/DatabentoDataHandler.cpp b/src/server/sparkd/md/databento/DatabentoDataHandler.cpp index 3b61336a4..665855fa7 100644 --- a/src/server/sparkd/md/databento/DatabentoDataHandler.cpp +++ b/src/server/sparkd/md/databento/DatabentoDataHandler.cpp @@ -29,15 +29,32 @@ namespace bts::md::DatabentoHandler { -std::string remove_spaces(const std::string_view& str) { - std::string result; - result.reserve(str.size()); // Reserve space for better performance - for (char c : str) { - if (!isspace(c)) { - result += c; - } - } - return result; +#define SEC_TYPE(name) bts::api::domain::SECURITY_TYPE_##name + +using bts::domain::SecurityID; +using bts::api::domain::CallPut; +using bts::api::domain::SecurityType; +using bts::api::domain::Venue; + +std::string remove_spaces(const std::string_view &str) { + std::string result; + result.reserve(str.size()); // Reserve space for better performance + for (char c : str) { + if (!isspace(c)) { + result += c; + } + } + return result; +} + +static SecurityType parse_sec_type(std::string_view str) { + if (str.find("OPT") != std::string::npos) { + return SEC_TYPE(OPTION); + } + if (str.find("FUT") != std::string::npos) { + return SEC_TYPE(FUTURE); + } + return SEC_TYPE(UNSPECIFIED); } void DatabentoDatahandler::run( @@ -53,6 +70,7 @@ void DatabentoDatahandler::run( const bool load_reference_data, const std::chrono::duration sleep_duration ) { + (void)filter_securities; auto sleep_duration_updated = sleep_duration; if (!use_file) { @@ -79,22 +97,9 @@ void DatabentoDatahandler::run( bts_assert_msg(false, "Exchange not supported on databento"); return databento::KeepGoing::Stop; } - if(filter_securities){ - const auto exp_nanos = msg.expiration; - std::string exp_string = databento::ToIso8601(exp_nanos); - const auto expiration_time = bts::core::TimeStamp::from_iso_string(exp_string); - if(expiration_time.seconds() < expiration_time.now().seconds() || expiration_time.seconds() > (expiration_time.now().seconds() + 60*60*24*365)){ - return databento::KeepGoing::Continue; - } - } - - - bts::domain::SecurityID sec_id{sec_id_store->get_or_add( - venue, std::to_string(msg.hd.instrument_id))}; bts::domain::Security sec; sec.security_common = domain::SecurityCommon(); sec.security_common.venue = venue; - sec.security_common.id = sec_id; sec.security_common.exchange_sec_id = msg.hd.instrument_id; sec.security_common.desc = msg.RawSymbol(); sec.security_common.symbol = msg.Asset(); @@ -106,38 +111,55 @@ void DatabentoDatahandler::run( sec.security_common.currency = {msg.currency.data(), msg.currency.size()}; // sec.security_common.cpp = msg.hd. - const std::string_view sec_type = {msg.security_type.data(), msg.security_type.size()}; + const SecurityType sec_type = + parse_sec_type({msg.security_type.data(), msg.security_type.size()}); const auto exp_nanos = msg.expiration; std::string exp_string = databento::ToIso8601(exp_nanos); const auto expiration_time = bts::core::TimeStamp::from_iso_string(exp_string); - if (sec_type.find("OPT") != std::string::npos) { - auto call_put = api::domain::CallPut::PUT; - if(msg.instrument_class == 'C' ){ - call_put=api::domain::CallPut::CALL; - } - else{ - bts_assert(msg.instrument_class == 'P'); - } - sec.security.emplace( - expiration_time, bts::domain::Price(msg.strike_price, -9),call_put); - } else if (sec_type.find("FUT") != std::string::npos) { - sec.security.emplace(expiration_time); - } else { - bts_assert_msg(false, "Only futures and options supported on databento"); - return databento::KeepGoing::Stop; - } + switch (sec_type) { + case SEC_TYPE(FUTURE): + sec.security.emplace(expiration_time); + break; + case SEC_TYPE(OPTION): { + auto call_put = CallPut::PUT; + if (msg.instrument_class == 'C') { + call_put = CallPut::CALL; + } else { + bts_assert(msg.instrument_class == 'P'); + } + sec.security.emplace( + expiration_time, bts::domain::Price(msg.strike_price, -9), + call_put); + break; + } + default: + bts_assert_msg( + false, "Only futures and options supported on databento"); + return databento::KeepGoing::Stop; + } + *sec.security_common.id = sec_id_store->get_or_add( + venue, sec_type, std::to_string(msg.hd.instrument_id)); listener->on_securities({&sec}); return databento::KeepGoing::Continue; }; auto process_bbo = [&](const databento::Record &record) { const auto &msg = record.Get(); - auto sec_id_opt = sec_id_store->get_id( - venue, std::to_string(msg.hd.instrument_id), false); - if (!sec_id_opt.has_value()) - return databento::KeepGoing::Continue; + // The security type to is needed to get the id. Since the sec type isn't known, + // check for an id using both types (OPTION and FUTURE). This is safe to do since + // options and futures never have common names (instrument ids). + const auto name = std::to_string(msg.hd.instrument_id); + auto sec_id_opt = + sec_id_store->get_id(venue, SEC_TYPE(OPTION), name, false); + if (!sec_id_opt.has_value()) { + sec_id_opt = + sec_id_store->get_id(venue, SEC_TYPE(FUTURE), name, false); + if (!sec_id_opt.has_value()) { + return databento::KeepGoing::Continue; + } + } sleep_for(sleep_duration_updated); bts::domain::SecurityID sec_id = bts::domain::SecurityID(sec_id_opt.value()); diff --git a/src/server/sparkd/md/deribit/DeribitInstrumentStore.cpp b/src/server/sparkd/md/deribit/DeribitInstrumentStore.cpp index f557c4044..dab42a2fe 100644 --- a/src/server/sparkd/md/deribit/DeribitInstrumentStore.cpp +++ b/src/server/sparkd/md/deribit/DeribitInstrumentStore.cpp @@ -19,6 +19,7 @@ namespace bts::md::deribit { using bts::api::domain::CallPut; +using bts::api::domain::SecurityType; using bts::domain::FutureSecurity; using bts::domain::OptionSecurity; using bts::domain::Price; @@ -106,8 +107,8 @@ struct DeribitInstrumentStore::PImpl { return &it->second; } - Security *find(const string &name) { - auto id = sec_id_store_->get_id(VENUE, name); + Security *find(SecurityType type, const string &name) { + auto id = sec_id_store_->get_id(VENUE, type, name); if (!id) { return nullptr; } @@ -147,7 +148,7 @@ struct DeribitInstrumentStore::PImpl { continue; } const SecurityID id{ - sec_id_store_->get_or_add(VENUE, di.instrument_name)}; + sec_id_store_->get_or_add(VENUE, sec.type(), di.instrument_name)}; sec.security_common.id = id; auto [it, is_new] = securities_.try_emplace(id, std::move(sec)); if (!is_new) { @@ -188,8 +189,8 @@ struct DeribitInstrumentStore::PImpl { // DeribitCombo object, which is required to resolve the leg definitions. unordered_map unresolved_combos_; - const Security *find_security_(const string &name) { - const auto id = sec_id_store_->get_id(VENUE, name); + const Security *find_security_(SecurityType type, const string &name) { + const auto id = sec_id_store_->get_id(VENUE, type, name); if (!id) { return nullptr; } @@ -202,7 +203,8 @@ struct DeribitInstrumentStore::PImpl { bool resolve_combo_(const DeribitCombo &combo, SpreadSecurity &spd) { for (const auto &leg : combo.legs) { - const auto *leg_sec = find_security_(leg.instrument_name); + const auto *leg_sec = + find_security_(SEC_TYPE(SPREAD), leg.instrument_name); if (leg_sec == nullptr) { spd.legs.clear(); return false; @@ -259,7 +261,8 @@ struct DeribitInstrumentStore::PImpl { continue; } } - SecurityID id{sec_id_store_->get_or_add(VENUE, sec.desc())}; + SecurityID id{ + sec_id_store_->get_or_add(VENUE, sec.type(), sec.desc())}; sec.security_common.id = id; auto [sec_it, is_new] = securities_.try_emplace(id, std::move(sec)); if (!is_new) { @@ -292,8 +295,8 @@ const Security *DeribitInstrumentStore::find(SecurityID id) const { return me_->find(id); } -Security *DeribitInstrumentStore::find(const string &id) { - return me_->find(id); +Security *DeribitInstrumentStore::find(SecurityType type, const string &name) { + return me_->find(type, name); } void DeribitInstrumentStore::on_instruments( diff --git a/src/server/sparkd/md/deribit/DeribitInstrumentStore.h b/src/server/sparkd/md/deribit/DeribitInstrumentStore.h index 3db3c8e26..34295c9d4 100644 --- a/src/server/sparkd/md/deribit/DeribitInstrumentStore.h +++ b/src/server/sparkd/md/deribit/DeribitInstrumentStore.h @@ -45,11 +45,12 @@ class DeribitInstrumentStore { const domain::Security *find(domain::SecurityID id) const; /** - * @brief Finds the security with the specified \p name + * @brief Finds the security with the specified \p name and \p type * @details No 'const' overload bc the fn that converts name to numeric id * isn't const */ - domain::Security *find(const std::string &name); + domain::Security *find( + api::domain::SecurityType type, const std::string &name); void on_instruments( const std::vector &instruments, diff --git a/src/server/sparkd/md/deribit/DeribitSubscription.cpp b/src/server/sparkd/md/deribit/DeribitSubscription.cpp index 5269d9681..9243f6408 100644 --- a/src/server/sparkd/md/deribit/DeribitSubscription.cpp +++ b/src/server/sparkd/md/deribit/DeribitSubscription.cpp @@ -41,6 +41,7 @@ namespace bts::md::deribit { +using bts::api::domain::SecurityType; using bts::api::md::DeribitMarketDataConfig; using bts::domain::Security; using bts::domain::SecurityCriteria; @@ -58,6 +59,7 @@ using std::unordered_map; using std::unordered_set; using std::vector; +#define VENUE bts::api::domain::VENUE_DERIBIT #define VENUE_NAME \ (bts::api::domain::Venue_Name(bts::api::domain::VENUE_DERIBIT)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_DERIBIT @@ -166,8 +168,9 @@ struct DeribitSubscription::PImpl { subscribe_instrument_state(); } - SecurityData &add_security_data(const string sec_desc) { - const SecurityID sid{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + SecurityData &add_security_data( + SecurityType sec_type, const string &sec_desc) { + const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = security_data_map.try_emplace( sec_desc, std::make_unique(sid, listener)); _ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc); @@ -199,27 +202,31 @@ struct DeribitSubscription::PImpl { void subscribe_md(SecurityNames security_names) { // Populate list of 'channels', and prepare their corresponding handlers vector channels; - for (const string &sec : security_names) { - auto &sec_data = add_security_data(sec); + for (auto [type, name] : security_names) { + auto &sec_data = add_security_data(type, name); // Depth if (subscribe_depth) { const auto &ch = channels.emplace_back( - FMT("book.{}.none.{}.100ms", sec, cfg.num_depth_levels())); + FMT("book.{}.none.{}.100ms", name, cfg.num_depth_levels())); channel_demuxer.add_handler(ch) - .bind<&DeribitDepthAdaptor::on_depths>(sec_data.depth_hnd); + .template bind<&DeribitDepthAdaptor::on_depths>( + sec_data.depth_hnd); } // Ticker if (subscribe_ticker) { const auto &ch = - channels.emplace_back(FMT("incremental_ticker.{}", sec)); + channels.emplace_back(FMT("incremental_ticker.{}", name)); channel_demuxer.add_handler(ch) - .bind<&DeribitTickerAdaptor::on_tickers>(sec_data.ticker_hnd); + .template bind<&DeribitTickerAdaptor::on_tickers>( + sec_data.ticker_hnd); } // Trades if (subscribe_trades) { - const auto &ch = channels.emplace_back(FMT("trades.{}.100ms", sec)); + const auto &ch = + channels.emplace_back(FMT("trades.{}.100ms", name)); channel_demuxer.add_handler(ch) - .bind<&DeribitTradeAdaptor::on_trades>(sec_data.trade_hnd); + .template bind<&DeribitTradeAdaptor::on_trades>( + sec_data.trade_hnd); } } // If 'channels' is empty, it's a no-op @@ -253,7 +260,9 @@ struct DeribitSubscription::PImpl { bts_assert(!CONTAINS_KEY(security_data_map, sec->desc())); } listener->on_securities(secs); - auto sec_names = TRANSFORM(secs, [](const auto *s) { return s->desc(); }); + auto sec_names = TRANSFORM(secs, [](const auto *s) { + return std::make_pair(s->type(), s->desc()); + }); batch_subscribe_md(sec_names); } diff --git a/src/server/sparkd/md/ftx/FTXSecurityRequestor.cpp b/src/server/sparkd/md/ftx/FTXSecurityRequestor.cpp index 4ddd4dfd7..cb721864f 100644 --- a/src/server/sparkd/md/ftx/FTXSecurityRequestor.cpp +++ b/src/server/sparkd/md/ftx/FTXSecurityRequestor.cpp @@ -35,6 +35,7 @@ using std::string; using std::unordered_set; using std::vector; +#define VENUE Venue::VENUE_FTX #define VENUE_NAME (Venue_Name(Venue::VENUE_FTX)) #define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x @@ -172,7 +173,8 @@ struct FTXSecurityRequestor::PImpl { for (auto &sec : added_secs) { // Since the ids haven't been set yet, we should expect them to be 0 bts_assert(sec.id().value() == 0); - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + SecurityID sec_id{ + sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())}; sec.security_common.id = sec_id; } } diff --git a/src/server/sparkd/md/ftx/FTXSubscription.cpp b/src/server/sparkd/md/ftx/FTXSubscription.cpp index 1ac286e26..3f6fd41f0 100644 --- a/src/server/sparkd/md/ftx/FTXSubscription.cpp +++ b/src/server/sparkd/md/ftx/FTXSubscription.cpp @@ -28,6 +28,7 @@ namespace bts::md::ftx { +using bts::api::domain::SecurityType; using bts::api::domain::Venue; using bts::api::domain::Venue_Name; using bts::api::md::FTXMarketDataConfig; @@ -36,11 +37,13 @@ using bts::domain::SecurityCriteria; using bts::domain::SecurityID; using bts::io::ProtoPrinter; using bts::persistence::IDGenerator; +using std::pair; using std::string; using std::unique_ptr; using std::unordered_map; using std::vector; +#define VENUE Venue::VENUE_FTX #define VENUE_NAME (Venue_Name(Venue::VENUE_FTX)) #define FTX_MSG_TYPE_NAME(x) \ @@ -130,16 +133,18 @@ struct FTXSubscription::PImpl { sec_requestor.request(); } - DepthHandler &create_depth_handler(string sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + DepthHandler &create_depth_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = depth_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert(is_new); return it->second; } - TickerHandler &create_ticker_handler(string sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + TickerHandler &create_ticker_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = ticker_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -147,8 +152,9 @@ struct FTXSubscription::PImpl { return it->second; } - StatsHandler &create_stats_handler(string sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + StatsHandler &create_stats_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = stats_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -171,15 +177,16 @@ struct FTXSubscription::PImpl { CCApiSession::BatchSubscription ticker_subs; CCApiSession::BatchRequest stats_reqs; - for (const auto &sec_desc : securities) { + for (const auto &[sec_type, sec_desc] : securities) { depth_subs.push_back( {create_depth_sub(sec_desc, depth_opts), - &create_depth_handler(sec_desc)}); + &create_depth_handler(sec_type, sec_desc)}); ticker_subs.push_back( {create_ticker_sub(sec_desc, ticker_opts), - &create_ticker_handler(sec_desc)}); + &create_ticker_handler(sec_type, sec_desc)}); stats_reqs.push_back( - {create_stats_req(sec_desc), &create_stats_handler(sec_desc)}); + {create_stats_req(sec_desc), + &create_stats_handler(sec_type, sec_desc)}); } session.subscribe( @@ -196,7 +203,7 @@ struct FTXSubscription::PImpl { */ void on_securities(vector secs) { // 'sec_descs' is used to make depth subscriptions - vector sec_descs; + vector> sec_descs; sec_descs.reserve(secs.size()); // 'sec_ptrs' is passed to IMarketDataListeners vector sec_ptrs; @@ -208,7 +215,7 @@ struct FTXSubscription::PImpl { auto [it, is_new] = securities.try_emplace( id, std::make_unique(std::move(s))); bts_assert_msgf(is_new, "Received duplicate FTX security '{}'", desc); - sec_descs.push_back(std::move(desc)); + sec_descs.push_back(std::make_pair(s.type(), std::move(desc))); // add ptr to entry in 'securities' (not the loop var, we moved that) sec_ptrs.push_back(it->second.get()); } diff --git a/src/server/sparkd/md/okx/OkxInstrumentStore.cpp b/src/server/sparkd/md/okx/OkxInstrumentStore.cpp index 38d55b160..0d5aea201 100644 --- a/src/server/sparkd/md/okx/OkxInstrumentStore.cpp +++ b/src/server/sparkd/md/okx/OkxInstrumentStore.cpp @@ -14,6 +14,7 @@ namespace bts::md::okx { using bts::api::domain::CallPut; +using bts::api::domain::SecurityType; using bts::domain::FutureSecurity; using bts::domain::OptionSecurity; using bts::domain::Price; @@ -94,8 +95,8 @@ struct OkxInstrumentStore::PImpl { return &it->second; } - Security *find(const string &name) { - auto id = sec_id_store_->get_id(VENUE, name); + Security *find(SecurityType type, const string &name) { + auto id = sec_id_store_->get_id(VENUE, type, name); if (!id) { return nullptr; } @@ -124,7 +125,7 @@ struct OkxInstrumentStore::PImpl { if (!filter_(sec)) { continue; } - SecurityID id{sec_id_store_->get_or_add(VENUE, oi.instId)}; + SecurityID id{sec_id_store_->get_or_add(VENUE, sec.type(), oi.instId)}; sec.security_common.id = id; auto [it, is_new] = securities_.try_emplace(id, std::move(sec)); if (!is_new) { @@ -152,7 +153,9 @@ const Security *OkxInstrumentStore::find(SecurityID id) const { return me_->find(id); } -Security *OkxInstrumentStore::find(const string &id) { return me_->find(id); } +Security *OkxInstrumentStore::find(SecurityType type, const string &name) { + return me_->find(type, name); +} void OkxInstrumentStore::on_instruments( const vector &instruments, diff --git a/src/server/sparkd/md/okx/OkxInstrumentStore.h b/src/server/sparkd/md/okx/OkxInstrumentStore.h index f3c8a3e0e..00a19be78 100644 --- a/src/server/sparkd/md/okx/OkxInstrumentStore.h +++ b/src/server/sparkd/md/okx/OkxInstrumentStore.h @@ -50,7 +50,8 @@ class OkxInstrumentStore { * @details No 'const' overload bc the fn that converts name to numeric id * isn't const */ - domain::Security *find(const std::string &name); + domain::Security *find( + api::domain::SecurityType type, const std::string &name); void on_instruments( const std::vector &instruments, diff --git a/src/server/sparkd/md/okx/OkxSubscription.cpp b/src/server/sparkd/md/okx/OkxSubscription.cpp index c88fd6db4..f2fc1306b 100644 --- a/src/server/sparkd/md/okx/OkxSubscription.cpp +++ b/src/server/sparkd/md/okx/OkxSubscription.cpp @@ -38,6 +38,7 @@ namespace bts::md::okx { +using bts::api::domain::SecurityType; using bts::api::md::OkxMarketDataConfig; using bts::domain::Security; using bts::domain::SecurityCriteria; @@ -60,6 +61,7 @@ using std::vector; #define _WARN(...) _LOG(Warn, __VA_ARGS__) #define _ERROR(...) _LOG(Error, __VA_ARGS__) +#define VENUE bts::api::domain::VENUE_OKX #define VENUE_NAME (bts::api::domain::Venue_Name(bts::api::domain::VENUE_OKX)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_OKX #define SEC_TYPE(t) bts::api::domain::SECURITY_TYPE_##t @@ -164,8 +166,9 @@ struct OkxSubscription::PImpl { [this](const auto &secs) { on_securities(secs); }); } - SecurityData &add_security_data(const string sec_desc) { - const SecurityID sid{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + SecurityData &add_security_data( + SecurityType sec_type, const string &sec_desc) { + const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = security_data_map.try_emplace( sec_desc, std::make_unique(sid, listener)); _ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc); @@ -182,13 +185,13 @@ struct OkxSubscription::PImpl { void subscribe_md(SecurityNames security_names) { // Populate list of 'channels', and prepare their corresponding handlers vector channels; - for (const string &sec : security_names) { - auto &sec_data = add_security_data(sec); + for (const auto &[type, name] : security_names) { + auto &sec_data = add_security_data(type, name); // Depth if (subscribe_depth) { auto &ch = channels.emplace_back(); ch.channel = "books5"; - ch.instId = sec; + ch.instId = name; channel_demuxer.add_handler(ch).bind<&OkxDepthAdaptor::on_depths>( sec_data.depth_hnd); } @@ -196,7 +199,7 @@ struct OkxSubscription::PImpl { if (subscribe_ticker) { auto &ch = channels.emplace_back(); ch.channel = "tickers"; - ch.instId = sec; + ch.instId = name; channel_demuxer.add_handler(ch).bind<&OkxTickerAdaptor::on_tickers>( sec_data.ticker_hnd); } @@ -204,7 +207,7 @@ struct OkxSubscription::PImpl { if (subscribe_trades) { auto &ch = channels.emplace_back(); ch.channel = "trades"; - ch.instId = sec; + ch.instId = name; channel_demuxer.add_handler(ch).bind<&OkxTradeAdaptor::on_trades>( sec_data.trade_hnd); } @@ -240,7 +243,9 @@ struct OkxSubscription::PImpl { bts_assert(!CONTAINS_KEY(security_data_map, sec->desc())); } listener->on_securities(secs); - auto sec_names = TRANSFORM(secs, [](const auto *s) { return s->desc(); }); + auto sec_names = TRANSFORM(secs, [](const auto *s) { + return std::make_pair(s->type(), s->desc()); + }); batch_subscribe_md(sec_names); } }; diff --git a/src/server/sparkd/md/platforms/emumba/EmMDListener.cpp b/src/server/sparkd/md/platforms/emumba/EmMDListener.cpp index 35e6a42b2..1cdad5c88 100644 --- a/src/server/sparkd/md/platforms/emumba/EmMDListener.cpp +++ b/src/server/sparkd/md/platforms/emumba/EmMDListener.cpp @@ -59,6 +59,10 @@ using namespace bts::orderman; namespace bts::md { +#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x +#define TRADING_STATUS(x) bts::co2::TRADINGSTATUS_##x +#define VENUE bts::api::domain::VENUE_DERIBIT + std::atomic EmMDListener::_order_mux = nullptr; using MDArray = std::array; using EMOrderKey = uint64_t; @@ -1144,16 +1148,32 @@ DescSearchResult EmMDListener::_find_em_sec( } } +static api::domain::SecurityType convert_sec_type( + emumba::market_data::security_type emumba_sec_type) { + switch (emumba_sec_type) { + case emumba::market_data::security_type::index: + return SEC_TYPE(SPOT); + case emumba::market_data::security_type::future: + return SEC_TYPE(FUTURE); + case emumba::market_data::security_type::spread: + return SEC_TYPE(SPREAD); + case emumba::market_data::security_type::option: + return SEC_TYPE(OPTION); + default: + bts_assert_msg(false, "Unknown sec type"); + return SEC_TYPE(UNSPECIFIED); + } +} + security_key EmMDListener::_add_sec_entry( const emumba::market_data::security &sec) { const std::string &desc = extract_security_desc(sec); // id_gen either adds a key and returns a new id, or finds one and returns // the previously added id bts_assert(me->sec_emdesc_map.find(desc) == me->sec_emdesc_map.end()); - auto new_key_opt = me->id_gen.get_id("exchange_key", desc, true); - bts_assert(new_key_opt.has_value()); + const security_key new_key = static_cast( + me->id_gen.get_or_add(VENUE, convert_sec_type(sec.sec_type()), desc)); // TODO (64 -> 32 conversion) - security_key new_key = new_key_opt.value(); me->sec_emdesc_map.insert({desc, {new_key, sec}}); bts_assert(me->sec_btsid_map.find(new_key) == me->sec_btsid_map.end()); me->sec_btsid_map.insert({new_key, desc}); diff --git a/src/server/sparkd/md/taifex/TaifexDataProcessors.h b/src/server/sparkd/md/taifex/TaifexDataProcessors.h index c0b80dd22..106b8aafa 100644 --- a/src/server/sparkd/md/taifex/TaifexDataProcessors.h +++ b/src/server/sparkd/md/taifex/TaifexDataProcessors.h @@ -201,8 +201,9 @@ class TaifexDataProcessors { const auto id = TaifexUtils::remove_spaces(msg.prod_id); const auto symbol = id.substr(0, 3); + const auto sec_type = convert_sec_type(msg.parsedProductInfo.productType); bts::domain::SecurityID sec_id{ - sec_id_store->get_or_add(api::domain::VENUE_TAIFEX, id)}; + sec_id_store->get_or_add(api::domain::VENUE_TAIFEX, sec_type, id)}; auto check = security_details->add_i010_mapping(id, sec_id, msg.decimal_locator); if (!check) { @@ -286,6 +287,20 @@ class TaifexDataProcessors { security_details->add_i011_mapping( id, 1.0, msg.getCurrencyString(), msg.contract_size); } + + private: + static api::domain::SecurityType convert_sec_type( + TaifexMessages::I010Data::ProductInfo::ProductType type) { + switch (type) { + case TaifexMessages::I010Data::ProductInfo::ProductType::FUTURES: + return api::domain::SecurityType::SECURITY_TYPE_FUTURE; + case TaifexMessages::I010Data::ProductInfo::ProductType::OPTIONS: + return api::domain::SecurityType::SECURITY_TYPE_OPTION; + default: + bts_assert_msg( + false, "Unexpected security type (must be FUTURES or OPTIONS)"); + } + } }; -} //namespace bts::md::taifex \ No newline at end of file +} //namespace bts::md::taifex From c4d08840bdf69eb30a65b93d9f5561ecb1e74dac Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sun, 29 Sep 2024 03:35:47 +0000 Subject: [PATCH 23/30] Add ostream operator for SecurityType --- src/lib/domain_model/SecurityType.h | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 src/lib/domain_model/SecurityType.h diff --git a/src/lib/domain_model/SecurityType.h b/src/lib/domain_model/SecurityType.h new file mode 100644 index 000000000..5c4b78e00 --- /dev/null +++ b/src/lib/domain_model/SecurityType.h @@ -0,0 +1,30 @@ +#pragma once + +#include "lib/domain_model/Security.pb.h" +#include "lib/core/FMT_IMPL.h" +#include "lib/core/EnumMacros.h" +#include "lib/io/JsonReader.h" +#include "lib/logging/logging.h" + +#include + +namespace bts::api { + +PB_ENUM_FROM_JSON(SecurityType) +PB_ENUM_TO_JSON(SecurityType) + +std::ostream &operator<<(std::ostream &out, const SecurityType &type) noexcept; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::ostream &operator<<( + std::ostream &out, const SecurityType &type) noexcept { + out << SecurityType_Name(type); + return out; +} + +} // namespace bts::api + +FMT_IMPL(bts::api::SecurityType, t, "{}", bts::api::SecurityType_Name(t)); From a00347ac3f22b626c2d244fa4e69e2fa292a6377 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sun, 29 Sep 2024 03:38:58 +0000 Subject: [PATCH 24/30] Add ostream operator for Venue --- src/lib/domain_model/Venue.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/lib/domain_model/Venue.h b/src/lib/domain_model/Venue.h index fe05c6c61..e69c9d6c3 100644 --- a/src/lib/domain_model/Venue.h +++ b/src/lib/domain_model/Venue.h @@ -13,6 +13,17 @@ namespace bts::api { PB_ENUM_FROM_JSON(Venue) PB_ENUM_TO_JSON(Venue) +std::ostream &operator<<(std::ostream &out, Venue venue) noexcept; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::ostream &operator<<(std::ostream &out, Venue venue) noexcept { + out << Venue_Name(venue); + return out; +} + } // namespace bts::api FMT_IMPL(bts::api::Venue, v, "{}", bts::api::Venue_Name(v)); From 4004ed2267f6744dff4774f3f27600dfa6a695a4 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sun, 29 Sep 2024 03:42:26 +0000 Subject: [PATCH 25/30] refactor: Normalize sec key indexing for old md sources Now both old and new (domain) md sources index securities by the same string format. --- src/lib/domain_model/Security.cpp | 147 +++++++++++++++++++----------- src/lib/domain_model/Security.h | 62 ++++++++++++- src/lib/md/Diag/ShmDumper.cpp | 7 +- src/lib/md/SecurityIDRecorder.cpp | 140 ++++++++++++++++------------ src/lib/md/SecurityIDRecorder.h | 7 ++ src/lib/md/Shm/FixedSecurity.cpp | 77 +--------------- src/lib/md/Shm/FixedSecurity.h | 2 - 7 files changed, 244 insertions(+), 198 deletions(-) diff --git a/src/lib/domain_model/Security.cpp b/src/lib/domain_model/Security.cpp index 815d5fcee..b79c3635c 100644 --- a/src/lib/domain_model/Security.cpp +++ b/src/lib/domain_model/Security.cpp @@ -3,6 +3,7 @@ #include "lib/config/config.h" #include "lib/config/protos/orders_fills_cleanup.pb.h" +#include "lib/md/common.h" #include "lib/utils/datetime.h" #include @@ -11,51 +12,15 @@ namespace bts { namespace securities { -std::string to_str(api::Security const &sec) { - std::string serialized; - google::protobuf::TextFormat::PrintToString(sec, &serialized); - return serialized; -} - -std::pair from_str(std::string const &str) { - api::Security result; - auto success = google::protobuf::TextFormat::ParseFromString(str, &result); - return std::make_pair(success, result); -} - -std::string descriptor(api::Security const &sec) { - switch (sec.ExchangeData_case()) { - case api::Security::kTaifex: - return sec.taifex().security_desc(); - case api::Security::kGlbx: - return sec.glbx().security_desc(); - case api::Security::kCboe: - return sec.cboe().security_desc(); - case api::Security::kCfe: - return sec.cfe().security_desc(); - case api::Security::kIce: - return sec.ice().security_desc(); - case api::Security::kDeribit: - return sec.deribit().security_desc(); - case api::Security::kFtx: - return sec.ftx().security_desc(); - case api::Security::kBinance: - return sec.binance().security_desc(); - case api::Security::kBinanceUsdsFutures: - return sec.binance_usds_futures().security_desc(); - case api::Security::kBinanceUs: - return sec.binance_us().security_desc(); - case api::Security::kCoinbase: - return sec.coinbase().security_desc(); - case api::Security::kOkx: - return sec.okx().security_desc(); - default: - return "No descriptor for non glbx / ice exchange"; - } -} +using bts::api::Security; +using bts::api::Venue; api::SecurityType security_type(api::Security const &security) { - switch (security.Contract_case()) { + return security_type(security.Contract_case()); +} + +api::SecurityType security_type(api::Security::ContractCase contract_case) { + switch (contract_case) { case api::Security::ContractCase::kFuture: return api::SecurityType::SECURITYTYPE_FUTURE; case api::Security::ContractCase::kOption: @@ -70,16 +35,6 @@ api::SecurityType security_type(api::Security const &security) { return api::SecurityType::SECURITYTYPE_UNSET; } return api::SecurityType::SECURITYTYPE_UNSET; - // if (sec.has_future()) - // return api::SecurityType::SECURITYTYPE_FUTURE; - // else if (sec.has_option()) - // return api::SecurityType::SECURITYTYPE_OPTION; - // else if (sec.has_spread()) - // return api::SecurityType::SECURITYTYPE_SPREAD; - // else if (sec.has_synthetic()) - // return api::SecurityType::SECURITYTYPE_SYNTHETIC; - // else - // return api::SecurityType::SECURITYTYPE_UNSET; } boost::gregorian::date smallest_leg( @@ -140,5 +95,91 @@ bool is_expired(api::Security const &security) { return is_expired(security, datetime::local_now()); } +void write_date_string( + uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result) { + result << std::setw(2) << std::setfill('0') << month << "/" << std::setw(2) + << std::setfill('0') << day << "/" << year - 2000; +} + +std::ostringstream &write_leg_descriptor( + std::ostringstream &result, char separator, const api::ContractLeg &leg) { + bts_assert(leg.security().id() != md::invalid_security_key); + // Keep the string small, yet readable for diagnostic purposes. + result << (leg.side() == api::SIDE_BUY ? '+' : '-') << separator + << leg.quantity() << "*(i=" << leg.security().id() + << ",m=" << leg.price_multiplier() << ",d=" << leg.leg_delta() << ")"; + return result; +} + +std::ostringstream &write_future_descriptor( + std::ostringstream &result, const api::FutureContract &future) { + write_date_string( + future.expiration_year(), future.expiration_month(), + future.expiration_day(), result); + return result; +} + +std::ostringstream &write_option_descriptor( + std::ostringstream &result, + char separator, + const api::OptionContract &option) { + write_date_string( + option.expiration_year(), option.expiration_month(), + option.expiration_day(), result); + result << separator << option.strike() + << (option.call_put() == api::PUTORCALL_CALL ? 'C' : 'P'); + return result; +} + +std::ostringstream &write_legs_descriptor( + std::ostringstream &result, + char separator, + const google::protobuf::RepeatedPtrField &legs) { + bts_assert(legs.size() > 0); + auto it = legs.begin(); + write_leg_descriptor(result, separator, *it); + while (++it != legs.end()) { + result << separator; + write_leg_descriptor(result, separator, *it); + } + return result; +} + +std::string unique_descriptor(const api::Security &security) { + const char separator = ' '; + std::ostringstream result; + result << api::Venue_Name(security.exchange()).substr(6) << separator + << api::SecurityType_Name(security_type(security)).substr(12) + << descriptor(security); + switch (security.Contract_case()) { + case Security::ContractCase::kOption: + result << separator; + write_option_descriptor(result, separator, security.option()); + break; + case Security::ContractCase::kFuture: + result << separator; + write_future_descriptor(result, security.future()); + break; + case Security::ContractCase::kSpot: + break; + case Security::ContractCase::kSpread: + result << separator; + write_legs_descriptor(result, separator, security.spread().legs()); + break; + case Security::ContractCase::kSynthetic: { + result << "pc=" + << api::SyntheticContract::PriceCalc_Name( + security.synthetic().price_calc()); + write_legs_descriptor(result, separator, security.synthetic().legs()); + break; + } + default: + bts_assert_msg( + false, "unique_descriptor failed due to invalid contract case"); + break; + } + return result.str(); +} + } // namespace securities } // namespace bts diff --git a/src/lib/domain_model/Security.h b/src/lib/domain_model/Security.h index 57bd75062..fe5a3ac32 100644 --- a/src/lib/domain_model/Security.h +++ b/src/lib/domain_model/Security.h @@ -2,6 +2,7 @@ #include "lib/core/FMT_IMPL.h" #include "lib/domain_model/Security.pb.h" +#include "lib/domain_model/SecurityType.h" #include "lib/domain_model/Side.h" #include "lib/domain_model/Venue.h" #include "lib/domain_model/common.h" @@ -25,11 +26,27 @@ class ICEExchangeData; } // namespace api namespace securities { -SecurityKey security_key(api::Security const &); -std::string to_str(api::Security const &); -std::pair from_str(std::string const &); -std::string descriptor(api::Security const &); + +/** + * Gets a descriptor string for a security (should be unique within a particular exchange). + */ +const std::string &descriptor( + const api::Security &sec, const std::string &default_rv = "UNDEFINED"); + +/** + * Gets a descriptor string for a security that is unique within the entire + * system (not just exchange). + */ +std::string unique_descriptor(const api::Security &security); + api::SecurityType security_type(api::Security const &); +api::SecurityType security_type(api::Security::ContractCase contract_case); + +void write_date_string( + std::uint32_t year, + std::uint32_t month, + std::uint32_t day, + std::ostringstream &result); inline std::string const &side_name(api::Side side) { return api::Side_Name(side); @@ -234,6 +251,41 @@ inline Price tick_size(api::Security const &sec) { return Price(sec.contract_spec().low_tick()); } +inline const std::string &descriptor( + const api::Security &sec, const std::string &default_rv) { + switch (sec.ExchangeData_case()) { + case api::Security::kGlbx: + return sec.glbx().security_desc(); + case api::Security::kCboe: + return sec.cboe().security_desc(); + case api::Security::kCfe: + return sec.cfe().security_desc(); + case api::Security::kIce: + return sec.ice().security_desc(); + case api::Security::kDeribit: + return sec.deribit().security_desc(); + case api::Security::kFtx: + return sec.ftx().security_desc(); + case api::Security::kBinance: + return sec.binance().security_desc(); + case api::Security::kBinanceUsdsFutures: + return sec.binance_usds_futures().security_desc(); + case api::Security::kBinanceUs: + return sec.binance_us().security_desc(); + case api::Security::kCoinbase: + return sec.coinbase().security_desc(); + case api::Security::kOkx: + return sec.okx().security_desc(); + case api::Security::kTaifex: + return sec.taifex().security_desc(); + default: { + if (sec.has_synthetic()) { + return sec.synthetic().name(); + } + return default_rv; + } + } +} // Looking to round md::IntPrice? Look in IntPrice.h // computes price_2 - price_1 in ticks @@ -314,3 +366,5 @@ inline bool is_spread_of_strips(const bts::api::Security &sec) { } // namespace securities } // namespace bts + +FMT_IMPL(bts::api::Security, s, "{}", bts::securities::descriptor(s)); diff --git a/src/lib/md/Diag/ShmDumper.cpp b/src/lib/md/Diag/ShmDumper.cpp index 2ab4eb7fa..4dbb10599 100644 --- a/src/lib/md/Diag/ShmDumper.cpp +++ b/src/lib/md/Diag/ShmDumper.cpp @@ -3,6 +3,7 @@ #include "ShmDumper.h" #include "../Shm/MDStructure.h" +#include "lib/domain_model/Security.h" #include "lib/io/JsonDumper.h" #include "lib/logging/logging.h" #include "lib/utils/SharedList.h" @@ -51,8 +52,6 @@ string dump_header(const ColdHeader &header) { return JsonDumper::dump(header); } -void write_date_string( - uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result); string ShmDumper::dump_security(const FixedSecurity &security) { ostringstream res; res << "{"; @@ -91,7 +90,7 @@ string ShmDumper::dump_security(const FixedSecurity &security) { case SECURITYTYPE_FUTURE: { auto *psec = static_cast(&security); res << "'expiration': '"; - write_date_string( + securities::write_date_string( psec->expiration_year, psec->expiration_month, psec->expiration_day, res); res << "' "; @@ -100,7 +99,7 @@ string ShmDumper::dump_security(const FixedSecurity &security) { case SECURITYTYPE_OPTION: { auto *psec = static_cast(&security); res << "'expiration': '"; - write_date_string( + securities::write_date_string( psec->expiration_year, psec->expiration_month, psec->expiration_day, res); res << "', "; diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 94c82ad52..3313a037f 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -1,6 +1,7 @@ -#include "lib/domain_model/Security.h" #ifndef _WIN32 #include "SecurityIDRecorder.h" +#include "lib/domain_model/Security.h" +#include #include "lib/config/config.h" #include "lib/logging/logging.h" @@ -21,19 +22,27 @@ using std::size_t; namespace bts { namespace md { +using bts::api::Security; +using bts::api::SecurityType; +using bts::api::Venue; +using bts::config::SystemConfig; + uint64_t exchange_id(const api::Security &security); -struct SecurityIDEntry { - api::Security security; - SecurityIDRecorder::Callback callback; +struct SecurityIdRequest { + Venue venue; + SecurityType type; + std::string name; + std::function + callback; }; struct SecurityIDRecorderState { SecurityIDRecorderState(SecurityIDRecorderParams &p) - : work(), pending_handle(), thread_id(), db(nullptr, sqlite3_close), - timer(), lookup_key(p.lookup_key), unresolved_legs() {} + : pending_handle(), thread_id(), db(nullptr, sqlite3_close), timer(), + lookup_key(p.lookup_key), unresolved_legs() {} - tbb::concurrent_queue> work; + tbb::concurrent_queue> id_request_queue; tbb::concurrent_queue pending_handle; std::thread::id thread_id; sqlite3_ptr db; @@ -59,10 +68,18 @@ struct SecurityIDRecorderState { void work_handles(); void work_db(); - void check_access() { bts_assert(thread_id == std::this_thread::get_id()); } void assign_key( const api::Security &security, const SecurityIDRecorder::Callback &key_assigned); + void assign_key( + api::Venue venue, + api::SecurityType type, + const std::string &name, + std::function + key_assigned); + + void check_access() { bts_assert(thread_id == std::this_thread::get_id()); } void add_unresolved_spread(MDProducerHandle &handle); security_key search(const std::string &descriptor); security_key insert(const std::string &descriptor, SecurityType); @@ -101,7 +118,7 @@ void SecurityIDRecorder::init() { sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0"); sqlite::exec_nofail( - _self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING " + _self->db, "CREATE TABLE IF NOT EXISTS store (descriptor STRING " "NOT NULL PRIMARY KEY, type int);"); } @@ -113,37 +130,42 @@ void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { } void SecurityIDRecorder::assign_key( - const api::Security &security, - const SecurityIDRecorder::Callback &callback) { - LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}", - security.DebugString()); - if (security.Contract_case() == Security::ContractCase::kSpread) { - for (const auto &leg : security.spread().legs()) { - if (leg.security().id() == invalid_security_key) { - LOG(md, Error, "Security {} has legs which will never be resolved.", - security.has_glbx() ? security.glbx().security_desc() - : security.symbol()); - // No point in trying to add this to the queue - return; - } - } - } - _self->assign_key(security, callback); + const api::Security &security, const Callback &callback) { + assign_key( + security.exchange(), bts::securities::security_type(security), + bts::securities::descriptor(security), + [&callback, msec = security]( + api::Venue, api::SecurityType, const std::string &, + uint32_t id) mutable { + msec.set_id(id); + callback(msec); + }); } void SecurityIDRecorderState::assign_key( - const api::Security &security, - const SecurityIDRecorder::Callback &callback) { - LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}", - bts::securities::descriptor(security)); - if (security.id() == invalid_security_key) { - auto work = std::make_shared(); - work->security = security; - work->callback = callback; - LOG(md, Info, "SecurityIDRecorderState::assign_key work.push d: {}", - bts::securities::descriptor(work->security)); - this->work.push(work); - } + api::Venue venue, + api::SecurityType type, + const std::string &name, + std::function< + void(api::Venue, api::SecurityType, const std::string &, std::uint32_t)> + callback) { + LOG(md, Info, "SecurityIDRecorder::assign_key {} {} {}", venue, type, name); + auto id_req = std::make_shared(); + id_req->venue = venue; + id_req->type = type; + id_req->name = name; + id_req->callback = callback; + id_request_queue.push(id_req); +} + +void SecurityIDRecorder::assign_key( + api::Venue venue, + api::SecurityType type, + const std::string &name, + std::function< + void(api::Venue, api::SecurityType, const std::string &, std::uint32_t)> + callback) { + _self->assign_key(venue, type, name, std::move(callback)); } void SecurityIDRecorder::do_work() { _self->do_work(); } @@ -220,9 +242,12 @@ void SecurityIDRecorderState::work_handles() { retry.push_back(handle); } else { convert_security(handle.security(), &security); - assign_key(security, [handle](const api::Security &security) mutable { - handle.set_key(security.id()); - }); + assign_key( + security.exchange(), bts::securities::security_type(security), + bts::securities::descriptor(security), + [handle]( + api::Venue, api::SecurityType, const std::string &, + uint32_t id) mutable { handle.set_key(id); }); } } @@ -239,17 +264,17 @@ void SecurityIDRecorderState::work_db() { // Then persist that list // Once a security has a key it is public so record it in the security // database - std::shared_ptr temp; - vector> new_entries; + std::shared_ptr temp; + vector>> new_entries; // SPARK-1446 Need to try to add existing securities to the db in case we // deleted them by hand - vector> existing_securities; - vector> defer; + vector> existing_securities; + vector> defer; vector new_descriptors; size_t count = 0; string descriptor; - while (work.try_pop(temp)) { - descriptor = security_descriptor(temp->security); + while (id_request_queue.try_pop(temp)) { + descriptor = temp->name; // If we're unable to create a descriptor try again later if (descriptor == "") { @@ -259,12 +284,11 @@ void SecurityIDRecorderState::work_db() { security_key key = search(descriptor); if (key != invalid_security_key) { - temp->security.set_id(key); - temp->callback(temp->security); + temp->callback(temp->venue, temp->type, temp->name, key); existing_securities.push_back(std::move(temp)); } else { // always push in parallel - new_entries.push_back(std::move(temp)); + new_entries.push_back(std::make_pair(key, std::move(temp))); new_descriptors.push_back(descriptor); ++count; // Limit the amount of time we spend writing to sqlite during startup @@ -277,19 +301,17 @@ void SecurityIDRecorderState::work_db() { for (auto &d : defer) { LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}", - bts::securities::descriptor(d->security)); - work.push(std::move(d)); + d->name); + id_request_queue.push(std::move(d)); } if (!new_entries.empty()) { sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION"); for (size_t x = 0; x < new_entries.size(); ++x) { - Security &security = new_entries[x]->security; - security_key key = - insert(new_descriptors[x], securitytype_of(security)); - bts_assert_d(key); - security.set_id(key); + auto &entry = new_entries[x]; + entry.first = insert(new_descriptors[x], entry.second->type); + bts_assert_d(entry.first != invalid_security_key); } sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); @@ -297,8 +319,8 @@ void SecurityIDRecorderState::work_db() { if (!new_entries.empty()) { for (size_t x = 0; x < new_entries.size(); ++x) { - Security &security = new_entries[x]->security; - new_entries[x]->callback(security); + auto &[id, req] = new_entries[x]; + req->callback(req->venue, req->type, req->name, id); } } } @@ -315,7 +337,7 @@ void SecurityIDRecorder::bulk_insert( _self->db, "INSERT INTO security_keys (rowid, descriptor, type) " "values (?, ?, ?)"); for (const api::Security *sec : data) { - descriptor = prefix + security_descriptor(*sec); + descriptor = prefix + bts::securities::unique_descriptor(*sec); auto type = securitytype_of(*sec); sqlite3_bind_int(insert_stmt.get(), 1, sec->id()); diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index 14819bd8d..862ba79b8 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -49,6 +49,13 @@ class SecurityIDRecorder { void ensure_key(MDProducerHandle handle); // Slow (copies security) - for use by services void assign_key(const api::Security &security, const Callback &key_assigned); + void assign_key( + api::Venue venue, + api::SecurityType type, + const std::string &name, + std::function + key_assigned); // If a dispatcher is not provided you must call work periodically to do // lookups and flush the reserved ids to disk. diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index e8b5c3091..252edf917 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -1,4 +1,4 @@ -#include "lib/domain_model/Security.pb.h" +#include "lib/domain_model/Security.h" #include "lib/utils/bts_assert.h" #ifndef NO_CPP0X @@ -59,81 +59,6 @@ FixedSpreadLeg::FixedSpreadLeg() : key(invalid_security_key), leg_side(Side_MIN), leg_quantity(0), leg_price(0), leg_delta(1) {} -void write_date_string( - uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result) { - result << std::setw(2) << std::setfill('0') << month << "/" << std::setw(2) - << std::setfill('0') << day << "/" << year - 2000; -} - -void write_leg_descriptor( - std::ostringstream &result, char separator, const api::ContractLeg &leg) { - result << separator << leg.security().id() << "-q" << leg.quantity() << "p" - << leg.price_multiplier() << "d" << leg.leg_delta() - << (leg.side() == SIDE_BUY ? "BUY" : "SELL"); -} - -std::string security_descriptor(const api::Security &security) { - const char separator = ' '; - std::ostringstream result; - if (security.exchange() == Venue::VENUE_ICE || - security.exchange() == Venue::VENUE_SIMICE) - result << security.ice().security_desc() << separator - << Venue_Name(security.exchange()); - else - result << security.symbol() << separator - << Venue_Name(security.exchange()); - switch (security.Contract_case()) { - case Security::ContractCase::kOption: { - result << separator; - const auto &o = security.option(); - write_date_string( - o.expiration_year(), o.expiration_month(), o.expiration_day(), - result); - result << separator << o.strike() - << (o.call_put() == PUTORCALL_CALL ? 'C' : 'P'); - break; - } - case Security::ContractCase::kFuture: { - result << separator; - const auto &f = security.future(); - write_date_string( - f.expiration_year(), f.expiration_month(), f.expiration_day(), - result); - break; - } - case Security::ContractCase::kSpot: { - break; - } - case Security::ContractCase::kSpread: { - // If the security key of a leg is invalid return an error - const auto &s = security.spread(); - bts_assert(s.legs_size() > 0); - for (const auto &leg : s.legs()) { - if (leg.security().id() == invalid_security_key) - return ""; - write_leg_descriptor(result, separator, leg); - } - break; - } - case Security::ContractCase::kSynthetic: { - const auto &s = security.synthetic(); - bts_assert(s.legs_size() > 0); - result << separator << s.name() - << " pc=" << SyntheticContract::PriceCalc_Name(s.price_calc()); - for (const auto &leg : s.legs()) { - if (leg.security().id() == invalid_security_key) - return ""; // unlikey with synthetics - write_leg_descriptor(result, separator, leg); - } - break; - } - case Security::ContractCase::CONTRACT_NOT_SET: - bts_assert( - security.Contract_case() != Security::ContractCase::CONTRACT_NOT_SET); - } - return result.str(); -} - void get_contract_spec( const FixedSecurity &source, /*out*/ api::ContractSpecification &spec) { spec.set_price_break(source.price_break); diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index b26ca753a..3afb6c79e 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -208,8 +208,6 @@ typedef FixedSpreadSecurity<16> FixedSpreadSecurityS; typedef FixedSpreadSecurity<128> FixedSpreadSecurityL; typedef FixedSpreadSecurity<512> FixedSpreadSecurityXL; -std::string security_descriptor(const api::Security &); - // Convert a fixedsecurity to a normal Security - caveat: spreads will be // normalized Security convert_security(const FixedSecurity &); From cb187e699784bd3e7762b45dbe91a16138beb1a0 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sun, 29 Sep 2024 03:44:10 +0000 Subject: [PATCH 26/30] Interpret db strng param as name or path in SqliteStore ctr --- src/lib/persistence/SqliteStore.cpp | 34 ++++++++++++++----- src/lib/persistence/SqliteStore.h | 13 ++++--- .../tests/PersistenceServiceTests.cpp | 3 +- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/lib/persistence/SqliteStore.cpp b/src/lib/persistence/SqliteStore.cpp index fb8eac840..84e741ea3 100644 --- a/src/lib/persistence/SqliteStore.cpp +++ b/src/lib/persistence/SqliteStore.cpp @@ -18,9 +18,25 @@ namespace bts { namespace persistence { +using bts::config::SystemConfig; + +static boost::filesystem::path get_db_path(const std::string &str) { + for (char c : str) { + switch (c) { + case '.': + case '/': + case '\\': + // Looks like 'str' is already a path + return {str}; + } + } + return boost::filesystem::path{config::get().data_path()} / + "databases" / str / (str + ".sqlite"); +} + struct SqliteStore::PImpl { - PImpl(std::string const &store_root, std::string const &db_name) - : db(db_name), db_path(persistence::storage_location(db, store_root)), + PImpl(const boost::filesystem::path &db_path) + : db(db_path.stem().string()), db_path(db_path), conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize), remove(nullptr, sqlite3_finalize), remove_all(nullptr, sqlite3_finalize), @@ -28,6 +44,9 @@ struct SqliteStore::PImpl { bts_assert(db.size()); } + PImpl(std::string const &store_root, std::string const &db_name) + : PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {} + std::string const db; boost::filesystem::path db_path; sqlite3_ptr conn; @@ -39,10 +58,9 @@ struct SqliteStore::PImpl { }; SqliteStore::SqliteStore(std::string const &db) - : SqliteStore(config::get().data_path(), db) {} - -SqliteStore::SqliteStore(std::string const &root_path, std::string const &db) - : me(new PImpl(root_path, db)) { + : me(std::make_unique(get_db_path(db))) { + LOG(persist, Info, "SqliteStore ctr. db: {}; db_path: {}", db, + me->db_path.string()); init_db(); } @@ -122,8 +140,8 @@ SqliteStore::Data SqliteStore::retrieve_all() { return result; } -std::pair -SqliteStore::retrieve_one(std::string const &ns, std::string const &k) { +std::pair SqliteStore::retrieve_one( + std::string const &ns, std::string const &k) { auto query_str = fmt::format( "select ns,key,value from store where ns='{}' and key='{}'", ns, k); sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str()); diff --git a/src/lib/persistence/SqliteStore.h b/src/lib/persistence/SqliteStore.h index dbc7e0836..4f0caf725 100644 --- a/src/lib/persistence/SqliteStore.h +++ b/src/lib/persistence/SqliteStore.h @@ -8,17 +8,18 @@ namespace persistence { class SqliteStore : public IStore { public: - SqliteStore(std::string const &storage_root, std::string const &db); SqliteStore(std::string const &db); ~SqliteStore(); + static SqliteStore from_path(const std::string &full_path); + typedef std::map> Data; Data retrieve_all() override; - std::pair - retrieve_one(std::string const &ns, std::string const &k) override; + std::pair retrieve_one( + std::string const &ns, std::string const &k) override; - void - remove(std::string const &ns, std::set const &keys) override; + void remove( + std::string const &ns, std::set const &keys) override; void remove_all(std::string const &ns) override; void insert( std::string const &ns, @@ -33,6 +34,8 @@ class SqliteStore : public IStore { static std::string storage_location(std::string const &db); private: + SqliteStore(); + struct PImpl; std::unique_ptr me; void init_db(); diff --git a/src/lib/persistence/tests/PersistenceServiceTests.cpp b/src/lib/persistence/tests/PersistenceServiceTests.cpp index 6a8d646e1..b9ed80a5a 100644 --- a/src/lib/persistence/tests/PersistenceServiceTests.cpp +++ b/src/lib/persistence/tests/PersistenceServiceTests.cpp @@ -16,7 +16,8 @@ namespace persistence { TEST_CASE("PersistenceService Sqlite storage", "[persistence]") { utils::TempDir db_dir; - SqliteStore st(db_dir.name(), "test_db"); + SqliteStore st( + (boost::filesystem::path{db_dir.name()} / "test_db.sqlite").string()); SECTION("initial_db_is_empty") { REQUIRE(st.retrieve_all().empty()); } From bd8c97244a23ae0714ca5d3220820f00a94a50be Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 10 Oct 2024 15:55:32 +0000 Subject: [PATCH 27/30] refactor: Return row id in IStore::retrieve_one This is a prerequisite to synchronizing security ids stored via IDGenerator (the "new" way to store ids, added when we implemented Deribit), and SecurityDB (the original way from spark days). The problem was that these 2 implementations were using separate databases. We need to unify the interfaces between these storage services to avoid breaking existing code. --- src/lib/core/TimeSpan.h | 10 + src/lib/md/Diag/ShmDumper.cpp | 77 +--- src/lib/md/SecurityDB.cpp | 22 -- src/lib/md/SecurityIDRecorder.cpp | 189 ++++------ src/lib/md/SecurityIDRecorder.h | 23 +- src/lib/md/Shm/FixedSecurity.cpp | 24 +- src/lib/md/Shm/FixedSecurity.h | 7 +- src/lib/md/SyntheticSecurityService.cpp | 5 +- src/lib/md/cme/mdp3_data_directory.h | 32 ++ src/lib/md/test/md2_tests.cpp | 29 +- src/lib/os/spark_data_directory.cpp | 45 +++ src/lib/os/spark_data_directory.h | 28 ++ src/lib/persistence/IDGenerator.cpp | 20 +- src/lib/persistence/IStore.h | 39 +- src/lib/persistence/SqliteStore.cpp | 342 ++++++++++++------ src/lib/persistence/SqliteStore.h | 13 +- src/lib/persistence/store_row.h | 20 + src/lib/utils/checked_arithmetic.h | 37 ++ .../analytics/AnalyticsServiceServer.cpp | 8 +- .../exchange_sim/ExchangeSimApplication.cpp | 5 +- .../sparkd/md/emumba/EmumbaDeribitData.h | 9 +- .../sparkd/md/platforms/emumba/EmumbaMain.cpp | 9 +- .../sparkd/md/platforms/ice_onixs/ICEMain.cpp | 135 ++++--- .../sparkd/md/platforms/ice_onixs/ICEMain.h | 36 +- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 78 ++-- .../sparkd/md/platforms/mdp3_onixs/MDP3Main.h | 20 +- .../sparkd/position/PositionServiceServer.cpp | 17 +- src/server/sparkd/user/AdminServiceDB.cpp | 6 +- 28 files changed, 719 insertions(+), 566 deletions(-) create mode 100644 src/lib/md/cme/mdp3_data_directory.h create mode 100644 src/lib/os/spark_data_directory.cpp create mode 100644 src/lib/os/spark_data_directory.h create mode 100644 src/lib/persistence/store_row.h create mode 100644 src/lib/utils/checked_arithmetic.h diff --git a/src/lib/core/TimeSpan.h b/src/lib/core/TimeSpan.h index 23970e616..5e0d14458 100644 --- a/src/lib/core/TimeSpan.h +++ b/src/lib/core/TimeSpan.h @@ -50,7 +50,9 @@ class TimeSpan { constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept; +constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept; +constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept; std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept; @@ -116,10 +118,18 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept { return lhs.nanos() < rhs.nanos(); } +constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept { + return lhs.nanos() <= rhs.nanos(); +} + constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept { return lhs.nanos() > rhs.nanos(); } +constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept { + return lhs.nanos() >= rhs.nanos(); +} + } // namespace bts::core FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string()); diff --git a/src/lib/md/Diag/ShmDumper.cpp b/src/lib/md/Diag/ShmDumper.cpp index 4dbb10599..dc04f67f7 100644 --- a/src/lib/md/Diag/ShmDumper.cpp +++ b/src/lib/md/Diag/ShmDumper.cpp @@ -53,82 +53,7 @@ string dump_header(const ColdHeader &header) { } string ShmDumper::dump_security(const FixedSecurity &security) { - ostringstream res; - res << "{"; - res << "'key': " << security.key << ", "; - res << "'symbol': '" << security.symbol() << "', "; - res << "'type': '" << api::SecurityType_Name(security.type) << "', "; - res << "'exchange': '" << api::Venue_Name(security.exchange) << "', "; - res << "'alt_symbol': '" << security.alt_symbol() << "', "; - res << "'cfi_code': '" << security.cfi_code() << "', "; - res << "'security_desc': '" << security.security_desc() << "', "; - res << "'platform_id': '" << security.platform_id() << "', "; - res << "'display_factor': " << (int)security.display_factor << ", "; - res << "'numeric_id': " << security.numeric_id << ", "; - res << "'cboe_class_key': " << security.cboe_class_key << ", "; - res << "'main_frac': " << (int)security.cme_main_fraction << ", "; - res << "'sub_fract': " << (int)security.cme_sub_fraction << ", "; - res << "'price_format': " << (int)security.cme_price_display_format << ", "; - res << "'market_segment': " << (int)security.cme_market_segment_id << ", "; - - res << "'price_break': " << security.price_break << ", "; - res << "'low_tick': " << security.low_tick << ", "; - res << "'high_tick': " << security.high_tick << ", "; - res << "'cpp': " << security.cash_per_point << ", "; - - res << "'market_description ': " << security.ice_market_description() - << ", "; - res << "'cleared_alias ': " << security.ice_cleared_alias() << ", "; - res << "'product_description ': " << security.ice_product_name() << ", "; - res << "'hub_alias ': " << security.ice_hub_alias() << ", "; - res << "'strip_name': " << security.ice_strip_name() << ", "; - res << "'mic_code': " << security.mic_code() << ", "; - - switch (security.type) { - case SECURITYTYPE_SPOT: - break; - case SECURITYTYPE_FUTURE: { - auto *psec = static_cast(&security); - res << "'expiration': '"; - securities::write_date_string( - psec->expiration_year, psec->expiration_month, psec->expiration_day, - res); - res << "' "; - break; - } - case SECURITYTYPE_OPTION: { - auto *psec = static_cast(&security); - res << "'expiration': '"; - securities::write_date_string( - psec->expiration_year, psec->expiration_month, psec->expiration_day, - res); - res << "', "; - res << "'strike': " << psec->strike << ", "; - res << "'callput': '" << api::PutOrCall_Name(psec->call_put) << "' "; - break; - } - case SECURITYTYPE_SPREAD: { - auto *psec = static_cast(&security); - res << "'legs': ["; - for (size_t x = 0; x < psec->leg_count(); ++x) { - auto &leg = psec->legs[x]; - if (x > 0) - res << ","; - res << "{ 'key': " << leg.key << ", 'exkey': " << leg.exchange_key - << ", 'quantity': " << leg.leg_quantity - << ", 'leg_price': " << leg.leg_price - << ", 'delta': " << leg.leg_delta << ", 'side': '" - << api::Side_Name(leg.leg_side) << "' }"; - } - res << "]"; - break; - } - default: - break; - } - - res << "}"; - return res.str(); + return JsonDumper::dump(security); } string dump(datetime::Date date) { diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index 27b84e073..c33e5814d 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -61,26 +61,6 @@ datetime::Date sec_expiration_cut_off() { return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day()); } -void apply_compatibilit_mutations(Security &security) { - // Migrate from old tick stuff - if (security.has_glbx()) { - if (security.has_glbx()) { - if (security.contract_spec().main_fraction() == 0) { - security.mutable_contract_spec()->set_main_fraction( - security.glbx().do_not_use_main_fraction()); - } - if (security.contract_spec().sub_fraction() == 0) { - security.mutable_contract_spec()->set_sub_fraction( - security.glbx().do_not_use_sub_fraction()); - } - if (security.contract_spec().price_display_format() == 0) { - security.mutable_contract_spec()->set_price_display_format( - security.glbx().do_not_use_price_display_format()); - } - } - } -} - size_t load_securities( std::function const &sym_filter, const std::function &callback) { @@ -104,7 +84,6 @@ size_t load_securities( if (sym_filter(us)) { auto blob = select_stmt.getColumn("security_blob"); if (security.ParseFromArray(blob, blob.getBytes())) { - apply_compatibilit_mutations(security); bts_assert(security_key == security.id()); count++; callback(security); @@ -146,7 +125,6 @@ SecurityPtr find_security(SecurityKey key) { auto result = std::make_shared(); auto blob = select.getColumn(0); if (result->ParseFromArray(blob, blob.getBytes())) { - apply_compatibilit_mutations(*result); return result; } } diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 3313a037f..f0abccdf8 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -1,19 +1,25 @@ #ifndef _WIN32 #include "SecurityIDRecorder.h" #include "lib/domain_model/Security.h" +#include "lib/persistence/IStore.h" +#include "lib/persistence/SqliteStore.h" #include #include "lib/config/config.h" #include "lib/logging/logging.h" #include "lib/utils/Sqlite_utils.h" +#include "lib/utils/checked_arithmetic.h" #include #include #include "lib/config/protos/market_data.pb.h" #include "lib/config/protos/system_config.pb.h" +#include #include +#include #include +#include using namespace std; using namespace bts::utils; @@ -26,9 +32,32 @@ using bts::api::Security; using bts::api::SecurityType; using bts::api::Venue; using bts::config::SystemConfig; +using bts::persistence::IStore; +using bts::persistence::SqliteStore; uint64_t exchange_id(const api::Security &security); +static std::unique_ptr get_default_store() { + return std::make_unique("key_ids"); +} + +static persistence::StoreRow get_row( + Venue venue, + SecurityType type, + const std::string &name, + const std::string &prefix = "") { + return persistence::StoreRow{ + fmt::format( + "{}{}.{}", std::string{prefix}, Venue_Name(venue).substr(6), + SecurityType_Name(type).substr(13)), + name, ""}; +} + +static persistence::StoreRow get_row( + const api::Security &s, const std::string &prefix = "") { + return get_row(s.exchange(), bts::securities::security_type(s), prefix); +} + struct SecurityIdRequest { Venue venue; SecurityType type; @@ -38,14 +67,16 @@ struct SecurityIdRequest { }; struct SecurityIDRecorderState { - SecurityIDRecorderState(SecurityIDRecorderParams &p) - : pending_handle(), thread_id(), db(nullptr, sqlite3_close), timer(), - lookup_key(p.lookup_key), unresolved_legs() {} + SecurityIDRecorderState( + std::unique_ptr store, KeyLookupOption lookup_key) + : store(store == nullptr ? get_default_store() : std::move(store)), + pending_handle(), thread_id(), timer(), lookup_key(lookup_key), + unresolved_legs() {} + std::unique_ptr store{nullptr}; tbb::concurrent_queue> id_request_queue; tbb::concurrent_queue pending_handle; std::thread::id thread_id; - sqlite3_ptr db; unique_ptr timer; KeyLookupOption lookup_key; @@ -81,46 +112,28 @@ struct SecurityIDRecorderState { void check_access() { bts_assert(thread_id == std::this_thread::get_id()); } void add_unresolved_spread(MDProducerHandle &handle); - security_key search(const std::string &descriptor); - security_key insert(const std::string &descriptor, SecurityType); + security_key search(const persistence::StoreRow &row); + security_key insert(const persistence::StoreRow &row); }; -SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params) - : _self(new SecurityIDRecorderState(params)) { - if (params.db) { - _self->db = std::move(params.db); - } +SecurityIDRecorder::SecurityIDRecorder( + std::unique_ptr store, + KeyLookupOption lookup_key, + utils::Dispatcher *dispatcher) + : _self(std::make_unique( + store == nullptr + ? std::make_unique("key_ids") + : std::move(store), + lookup_key)) { init(); - if (params.dispatcher) { - _self->run(*params.dispatcher); + if (dispatcher != nullptr) { + _self->run(*dispatcher); } } SecurityIDRecorder::~SecurityIDRecorder() {} -void SecurityIDRecorder::init() { - LOG(md, Info, "SecurityIDRecorder::init"); - string db_dir = - config::get().data_path() + "databases/md/"; - string db_path = db_dir + "/security_keys.sqlite"; - - if (!boost::filesystem::exists(db_dir)) { - boost::filesystem::create_directories(db_dir); - } - - if (!_self->db) { - _self->db = sqlite::open_db(db_path.c_str()); - bts_assert(_self->db); - } - - // Set a long busy timeout - sqlite3_busy_timeout(_self->db.get(), 9999); - sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); - sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0"); - sqlite::exec_nofail( - _self->db, "CREATE TABLE IF NOT EXISTS store (descriptor STRING " - "NOT NULL PRIMARY KEY, type int);"); -} +void SecurityIDRecorder::init() {} void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}", @@ -134,7 +147,7 @@ void SecurityIDRecorder::assign_key( assign_key( security.exchange(), bts::securities::security_type(security), bts::securities::descriptor(security), - [&callback, msec = security]( + [callback, msec = security]( api::Venue, api::SecurityType, const std::string &, uint32_t id) mutable { msec.set_id(id); @@ -269,27 +282,25 @@ void SecurityIDRecorderState::work_db() { // SPARK-1446 Need to try to add existing securities to the db in case we // deleted them by hand vector> existing_securities; - vector> defer; - vector new_descriptors; + vector new_rows; size_t count = 0; - string descriptor; while (id_request_queue.try_pop(temp)) { - descriptor = temp->name; - - // If we're unable to create a descriptor try again later - if (descriptor == "") { - defer.push_back(std::move(temp)); - continue; + security_key key = invalid_security_key; + auto row = get_row(temp->venue, temp->type, temp->name); + try { + key = search(row); + } catch (const std::overflow_error &ex) { + LOG(persist, Error, + "SecurityIDRecorderState::work_db. Row search threw ex: {}", + std::string{ex.what()}); } - - security_key key = search(descriptor); if (key != invalid_security_key) { temp->callback(temp->venue, temp->type, temp->name, key); existing_securities.push_back(std::move(temp)); } else { // always push in parallel new_entries.push_back(std::make_pair(key, std::move(temp))); - new_descriptors.push_back(descriptor); + new_rows.push_back(std::move(row)); ++count; // Limit the amount of time we spend writing to sqlite during startup // to avoid blocking the heartbeat @@ -299,22 +310,14 @@ void SecurityIDRecorderState::work_db() { } } - for (auto &d : defer) { - LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}", - d->name); - id_request_queue.push(std::move(d)); - } - if (!new_entries.empty()) { - sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION"); - + store->insert_many_resolve_ids(new_rows); for (size_t x = 0; x < new_entries.size(); ++x) { auto &entry = new_entries[x]; - entry.first = insert(new_descriptors[x], entry.second->type); + auto &row = new_rows[x]; + entry.first = row.id; bts_assert_d(entry.first != invalid_security_key); } - - sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } if (!new_entries.empty()) { @@ -327,65 +330,27 @@ void SecurityIDRecorderState::work_db() { void SecurityIDRecorder::bulk_insert( const std::vector &data, const char *prefix) { - LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size()); - if (data.size() == 0) - return; - sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); - std::string descriptor; - { - auto insert_stmt = sqlite::prepare_query( - _self->db, "INSERT INTO security_keys (rowid, descriptor, type) " - "values (?, ?, ?)"); - for (const api::Security *sec : data) { - descriptor = prefix + bts::securities::unique_descriptor(*sec); - auto type = securitytype_of(*sec); - - sqlite3_bind_int(insert_stmt.get(), 1, sec->id()); - sqlite3_bind_text( - insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - sqlite3_bind_int(insert_stmt.get(), 3, (int)type); - sqlite3_step(insert_stmt.get()); - sqlite3_reset(insert_stmt.get()); - } + std::vector rows; + rows.reserve(data.size()); + for (const auto *s : data) { + rows.push_back(get_row(*s, prefix)); } - sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + _self->store->insert_many_resolve_ids(rows); } -security_key SecurityIDRecorderState::search(const std::string &descriptor) { - LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor); - auto select_stmt = sqlite::prepare_query( - db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); - sqlite3_bind_text( - select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - int result = sqlite3_step(select_stmt.get()); - if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_column_int(select_stmt.get(), 0); +security_key SecurityIDRecorderState::search(const persistence::StoreRow &row) { + auto [exists, rowid, val] = store->retrieve_one(row.ns, row.key); + if (exists) { + if (!val.empty()) { + bts_assert(stol(val) == rowid); + } + return system::math::checked_ltou(rowid); } return invalid_security_key; } -security_key SecurityIDRecorderState::insert( - const std::string &descriptor, SecurityType type) { - LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor); - auto insert_stmt = sqlite::prepare_query( - db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); - sqlite3_bind_text( - insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - sqlite3_bind_int(insert_stmt.get(), 2, (int)type); - int result = sqlite3_step(insert_stmt.get()); - if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_last_insert_rowid(db.get()); - } - // Just in case our insert failed due to duplicate row - auto key = search(descriptor); - if (key == invalid_security_key) { - LOG(md, Error, "Failed to assign security key. SQLite return code: {}", - result); - } - return key; +security_key SecurityIDRecorderState::insert(const persistence::StoreRow &row) { + return store->insert(row.ns, row.key, row.value); } } // namespace md diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index 862ba79b8..2925e0c20 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -2,7 +2,7 @@ #ifndef _WIN32 #include "MDProducerHandle.h" #include "common.h" - +#include "lib/persistence/IStore.h" #include "lib/utils/Sqlite_utils.h" #include #include @@ -22,26 +22,15 @@ namespace md { typedef const std::function KeyLookup; typedef std::optional KeyLookupOption; -struct SecurityIDRecorderParams { - SecurityIDRecorderParams() - : dispatcher(), db(sqlite::null_db()), lookup_key() {} - SecurityIDRecorderParams(utils::Dispatcher &d) - : dispatcher(&d), db(sqlite::null_db()), lookup_key() {} - SecurityIDRecorderParams(utils::Dispatcher &d, KeyLookup &get_key) - : dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {} - SecurityIDRecorderParams(sqlite3_ptr &db) - : dispatcher(), db(std::move(db)), lookup_key() {} - SecurityIDRecorderParams(sqlite3_ptr &db, KeyLookup &get_key) - : dispatcher(), db(std::move(db)), lookup_key(get_key) {} - utils::Dispatcher *dispatcher; - sqlite3_ptr db; - KeyLookupOption lookup_key; -}; + struct SecurityIDRecorderState; class SecurityIDRecorder { public: - SecurityIDRecorder(SecurityIDRecorderParams); + SecurityIDRecorder( + std::unique_ptr store = nullptr, + KeyLookupOption lookup_key = std::nullopt, + utils::Dispatcher *dispatcher = nullptr); ~SecurityIDRecorder(); typedef std::function Callback; diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index 252edf917..cd2ebc9ee 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -217,26 +217,12 @@ void convert_security(const FixedSecurity &source, Security *presult) { get_contract_spec(source, *dest.mutable_contract_spec()); } -SecurityType securitytype_of_contractcase(Security::ContractCase cc) { - switch (cc) { - case Security::ContractCase::kFuture: - return SECURITYTYPE_FUTURE; - case Security::ContractCase::kOption: - return SECURITYTYPE_OPTION; - case Security::ContractCase::kSpot: - return api::SECURITYTYPE_SPOT; - case Security::ContractCase::kSpread: - return SECURITYTYPE_SPREAD; - case Security::ContractCase::kSynthetic: - return SECURITYTYPE_SYNTHETIC; - case Security::ContractCase::CONTRACT_NOT_SET: - bts_assert(cc != Security::ContractCase::CONTRACT_NOT_SET); - } - bts_assert(false); -} SecurityType securitytype_of(const Security &s) { - return securitytype_of_contractcase(s.Contract_case()); + auto type = bts::securities::security_type(s.Contract_case()); + bts_assert(type != api::SecurityType::SECURITYTYPE_UNSET); + return type; } + size_t num_legs(const Security &sec) { if (sec.Contract_case() == api::Security::ContractCase::kSpread) { return sec.spread().legs_size(); @@ -284,7 +270,7 @@ uint64_t exchange_id(const Security &security) { void convert_security( const Security &src, FixedSecurity &dest, bool set_security_key) { dest.key = set_security_key ? src.id() : invalid_security_key; - dest.type = securitytype_of_contractcase(src.Contract_case()); + dest.type = bts::securities::security_type(src.Contract_case()); dest.symbol(src.symbol()); dest.currency(src.currency()); dest.exchange = src.exchange(); diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index 3afb6c79e..4b39e6c51 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -215,10 +215,9 @@ void convert_security(const FixedSecurity &, Security *); void convert_security( const Security &src, FixedSecurity &dest, bool set_security_key = false); -SecurityType securitytype_of_contractcase(Security::ContractCase cc); -SecurityType securitytype_of(const Security &); -size_t num_legs(const Security &); -uint64_t exchange_id(const Security &security); +SecurityType securitytype_of(const api::Security &); +size_t num_legs(const api::Security &); +uint64_t exchange_id(const api::Security &security); void to_json(nlohmann::json &j, const FixedSpreadLeg &f); diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp index c0301e8f4..6daca8b4c 100644 --- a/src/lib/md/SyntheticSecurityService.cpp +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -35,8 +35,9 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher struct SyntheticSecurityService::Pimpl { Pimpl(const MetadataStatefulPublisher &fn) - : recorder(utils::Dispatcher::background()), synthetics(), on_security(), - meta(), on_metadata(), metadata_publisher(fn) {} + : recorder(nullptr, std::nullopt, &utils::Dispatcher::background()), + synthetics(), on_security(), meta(), on_metadata(), + metadata_publisher(fn) {} SecurityIDRecorder recorder; std::unordered_map synthetics; utils::Event on_security; diff --git a/src/lib/md/cme/mdp3_data_directory.h b/src/lib/md/cme/mdp3_data_directory.h new file mode 100644 index 000000000..740d46dbb --- /dev/null +++ b/src/lib/md/cme/mdp3_data_directory.h @@ -0,0 +1,32 @@ +#pragma once + +#include "lib/os/spark_data_directory.h" + +#include +#include + +namespace bts::md { + +struct Mdp3DataDirectory { + boost::filesystem::path log_dir() const { return root.log_dir() / "onixs"; } + + boost::filesystem::path data_dir() const { return root.root() / "cme_mdp3"; } + + boost::filesystem::path config_file() const { + return data_dir() / "config.xml"; + } + + boost::filesystem::path templates_file() const { + return data_dir() / "templates.xml"; + } + + void create_dirs() { + boost::filesystem::create_directories(data_dir()); + boost::filesystem::create_directories(log_dir()); + } + + private: + spark::SparkDataDirectory root{}; +}; + +} // namespace bts::md diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 13cdbb323..f4cff0acc 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -9,6 +9,7 @@ #include "lib/config/config.h" #include "lib/metrics/metrics.h" #include "lib/os/os.h" +#include "lib/persistence/SqliteStore.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/Shuffle.h" #include "lib/utils/bts_chrono.h" @@ -160,12 +161,10 @@ class Co2Tests { #ifndef _WIN32 void test_security_id_recorder(const std::string &db_name) { // Verify callbacks happen after work complete - auto db1 = sqlite::open_db(db_name.c_str()); - REQUIRE(db1); - auto db2 = sqlite::open_db(db_name.c_str()); - REQUIRE(db2); - - SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}}; + SecurityIDRecorder idsvc{ + std::make_unique(db_name)}; + SecurityIDRecorder idsvc2{ + std::make_unique(db_name)}; security_key result = invalid_security_key; Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850); Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20); @@ -215,18 +214,18 @@ class Co2Tests { std::make_unique(s1, cold_header, hot_header); // Add spread to SecurityIDRecorder and work it - auto db = sqlite::open_db(":memory:"); + auto db = std::make_unique(":memory:"); REQUIRE(db); SecurityIDRecorder idsvc{ - {db, [&](ExchangeKey k) -> SecurityKey { - for (const auto &fut : - {f1handle->producer(), f2handle->producer()}) { - if (k.value() == fut.security().numeric_id) - return SecurityKey(fut.key()); - } - return SecurityKey(invalid_security_key); - }}}; + std::move(db), [&](ExchangeKey k) -> SecurityKey { + for (const auto &fut : + {f1handle->producer(), f2handle->producer()}) { + if (k.value() == fut.security().numeric_id) + return SecurityKey(fut.key()); + } + return SecurityKey(invalid_security_key); + }}; auto workwork = [&]() { idsvc.do_work(); diff --git a/src/lib/os/spark_data_directory.cpp b/src/lib/os/spark_data_directory.cpp new file mode 100644 index 000000000..fea551a9e --- /dev/null +++ b/src/lib/os/spark_data_directory.cpp @@ -0,0 +1,45 @@ +#include "spark_data_directory.h" +#include "lib/config/config.h" +#include "lib/config/protos/system_config.pb.h" + +#include +#include +#include + +namespace bts::spark { + +SparkDataDirectory::SparkDataDirectory() + : SparkDataDirectory(config::get().data_path()) {} + +SparkDataDirectory::SparkDataDirectory(const std::string &root) : root_(root) {} + +boost::filesystem::path SparkDataDirectory::root() const { return root_; } + +boost::filesystem::path SparkDataDirectory::log_dir() const { + return root_ / "logging"; +} + +boost::filesystem::path SparkDataDirectory::log_file() const { + return log_dir() / "log"; +} + +boost::filesystem::path SparkDataDirectory::log_cache_file() const { + return log_dir() / "log_cache"; +} + +boost::filesystem::path SparkDataDirectory::database_dir( + const std::string &name) const { + return root_ / "databases" / (name + "/"); +} + +boost::filesystem::path SparkDataDirectory::database_file( + const std::string &name) const { + return root_ / "databases" / name / (name + ".sqlite"); +} + +boost::filesystem::path SparkDataDirectory::database_file( + const std::string &dir_name, const std::string &db_name) const { + return root_ / "databases" / dir_name / (db_name + ".sqlite"); +} + +} // namespace bts::spark diff --git a/src/lib/os/spark_data_directory.h b/src/lib/os/spark_data_directory.h new file mode 100644 index 000000000..e33248f85 --- /dev/null +++ b/src/lib/os/spark_data_directory.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +namespace bts::spark { + +class SparkDataDirectory { + public: + SparkDataDirectory(); + + explicit SparkDataDirectory(const std::string &root); + + boost::filesystem::path root() const; + boost::filesystem::path log_dir() const; + boost::filesystem::path log_file() const; + boost::filesystem::path log_cache_file() const; + + boost::filesystem::path database_dir(const std::string &name) const; + boost::filesystem::path database_file(const std::string &name) const; + boost::filesystem::path database_file( + const std::string &dir_name, const std::string &db_name) const; + + private: + boost::filesystem::path root_; +}; + +} // namespace bts::spark diff --git a/src/lib/persistence/IDGenerator.cpp b/src/lib/persistence/IDGenerator.cpp index 2ee855258..7cf38724e 100644 --- a/src/lib/persistence/IDGenerator.cpp +++ b/src/lib/persistence/IDGenerator.cpp @@ -60,24 +60,21 @@ std::optional IDGenerator::get_id( bool add_if_new) { std::string ns = fmt::format( "{}.{}", Venue_Name(venue).substr(6), - SecurityType_Name(type).substr(12)); + SecurityType_Name(type).substr(14)); return get_id(ns, name, add_if_new); } std::optional IDGenerator::get_id( const std::string &ns, const std::string &key, bool add_if_new) { std::lock_guard myLock(_instance_mx); - auto [exists, val] = me->kv_store->retrieve_one(ns, key); + auto [exists, rowid, val] = me->kv_store->retrieve_one(ns, key); if (exists) { - uint64_t rv = str_to_id(val); - LOG(Gen, Diag, "IDGenerator::get_id ns: {} key:{} id: {}", ns, key, rv); - return std::optional(rv); - + LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rowid); + return std::optional(rowid); } else { if (add_if_new) { auto all = me->kv_store->retrieve_all(); - uint64_t new_id = get_next_id(all); - me->kv_store->insert(ns, key, std::to_string(new_id)); + auto new_id = me->kv_store->insert(ns, key, ""); LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, new_id); return std::optional(new_id); @@ -88,13 +85,6 @@ std::optional IDGenerator::get_id( } } -std::uint64_t IDGenerator::get_or_add( - const std::string &ns, const std::string &key) { - auto id = get_id(ns, key, true); - bts_assert(id.has_value()); - return *id; -} - std::uint64_t IDGenerator::get_or_add( api::domain::Venue venue, api::domain::SecurityType type, diff --git a/src/lib/persistence/IStore.h b/src/lib/persistence/IStore.h index 1f33ddf28..29e38eb7a 100644 --- a/src/lib/persistence/IStore.h +++ b/src/lib/persistence/IStore.h @@ -1,8 +1,13 @@ #pragma once +#include "lib/persistence/store_row.h" + +#include #include #include #include +#include +#include namespace bts { namespace persistence { @@ -11,17 +16,45 @@ class IStore { public: virtual std::map> retrieve_all() = 0; - virtual std::pair - retrieve_one(std::string const &ns, std::string const &k) = 0; + + /** + * Finds the row with the specified namespace and key. + * + * @return std::tuple The first item indicates + * whether or not a row was found. For successful queries, the second and third + * items are the rowid and value, respectively. + */ + virtual std::tuple retrieve_one( + std::string const &ns, std::string const &k) = 0; virtual void remove(std::string const &ns, std::set const &keys) = 0; virtual void remove_all(std::string const &ns) = 0; - virtual void insert( + virtual std::int64_t insert( std::string const &ns, std::string const &k, std::string const &v) = 0; + + void insert_many(const std::vector &rows); + void insert_many_resolve_ids(std::vector &rows); + virtual void update( std::string const &ns, std::string const &k, std::string const &v) = 0; virtual ~IStore(); }; +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline void IStore::insert_many(const std::vector &rows) { + for (const auto &row : rows) { + insert(row.ns, row.key, row.value); + } +} + +inline void IStore::insert_many_resolve_ids(std::vector &rows) { + for (auto &row : rows) { + row.id = insert(row.ns, row.key, row.value); + } +} + } // namespace persistence } // namespace bts diff --git a/src/lib/persistence/SqliteStore.cpp b/src/lib/persistence/SqliteStore.cpp index 84e741ea3..50bf7cbf6 100644 --- a/src/lib/persistence/SqliteStore.cpp +++ b/src/lib/persistence/SqliteStore.cpp @@ -35,26 +35,28 @@ static boost::filesystem::path get_db_path(const std::string &str) { } struct SqliteStore::PImpl { + std::string const db; + boost::filesystem::path db_path; + sqlite3_ptr conn{nullptr, sqlite3_close}; + sqlite3_stmt_ptr insert{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr insert_many{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr insert_with_id{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr remove{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr remove_all{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr update{nullptr, sqlite3_finalize}; + PImpl(const boost::filesystem::path &db_path) - : db(db_path.stem().string()), db_path(db_path), - conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize), - remove(nullptr, sqlite3_finalize), - remove_all(nullptr, sqlite3_finalize), - update(nullptr, sqlite3_finalize) { + : db(db_path.stem().string()), db_path(db_path) { bts_assert(db.size()); } PImpl(std::string const &store_root, std::string const &db_name) : PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {} - std::string const db; - boost::filesystem::path db_path; - sqlite3_ptr conn; - sqlite3_stmt_ptr insert; - sqlite3_stmt_ptr remove; - sqlite3_stmt_ptr remove_all; - sqlite3_stmt_ptr update; - BTS_NO_COPY(PImpl); + explicit PImpl(sqlite3_ptr conn) : db(), db_path(), conn(std::move(conn)) {} + + PImpl(const PImpl &) = delete; + PImpl &operator=(const PImpl &) = delete; }; SqliteStore::SqliteStore(std::string const &db) @@ -64,54 +66,64 @@ SqliteStore::SqliteStore(std::string const &db) init_db(); } +SqliteStore::SqliteStore(sqlite3_ptr db) + : me(std::make_unique(std::move(db))) { + LOG(persist, Info, "SqliteStore ctr. db: "); + init_db(); +} + void SqliteStore::init_db() { - auto path = me->db_path; - auto parent = path.parent_path(); + const auto path = me->db_path; - if (!boost::filesystem::is_directory(parent)) - bts_assert(boost::filesystem::create_directories(parent)); - - if (!boost::filesystem::exists(path)) { - LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", - me->db, path.string()); - - me->conn = sqlite::open_db(path.c_str()); - bts_assert_msg(me->conn, path.c_str()); - - const char *create_schema_sql = - "create table store ( ns text, key text, value text)"; - sqlite::exec_nofail(me->conn, create_schema_sql); - - const char *key_index_sql = - "create unique index key_index on store(ns,key)"; - sqlite::exec_nofail(me->conn, key_index_sql); - - const char *ns_index_sql = "create index ns_index on store(ns)"; - sqlite::exec_nofail(me->conn, ns_index_sql); - } else { - me->conn = sqlite::open_db(path.c_str()); + if (!path.empty()) { + if (!boost::filesystem::is_directory(path.parent_path())) { + bts_assert(boost::filesystem::create_directories(path.parent_path())); + } + if (!boost::filesystem::exists(path)) { + LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", + me->db, path.string()); + } } + auto &conn = me->conn; + if (conn == nullptr) { + conn = sqlite::open_db(path.c_str()); + bts_assert(conn != nullptr); + } + sqlite::exec_nofail( + conn, + "CREATE TABLE IF NOT EXISTS store (ns TEXT, key TEXT, value TEXT)"); + sqlite::exec_nofail( + conn, "CREATE UNIQUE INDEX IF NOT EXISTS key_index on store(ns, key)"); + sqlite::exec_nofail( + conn, "CREATE INDEX IF NOT EXISTS ns_index on store(ns)"); + conn = sqlite::open_db(path.c_str()); + // Asynchronous for speed - sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;"); - sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL"); + sqlite::exec_nofail(conn, "PRAGMA synchronous = 0;"); + sqlite::exec_nofail(conn, "PRAGMA journal_mode=WAL"); // prepare insert statement const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)"; - me->insert = sqlite::prepare_query(me->conn, insert_sql); + me->insert = sqlite::prepare_query(conn, insert_sql); + + // prepare insert_with_id statement + const char *insert_with_id_sql = + "insert into store (rowid, ns, key, value) values (?,?,?,?)"; + me->insert_with_id = sqlite::prepare_query(conn, insert_with_id_sql); // prepare update statement const char *update_sql = "update store set value = ? where ns = ? and key = ?"; - me->update = sqlite::prepare_query(me->conn, update_sql); + me->update = sqlite::prepare_query(conn, update_sql); // prepare remove statement const char *remove_all_sql = "delete from store where ns = ?"; - me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql); + me->remove_all = sqlite::prepare_query(conn, remove_all_sql); // prepare remove statement const char *remove_sql = "delete from store where ns = ? and key = ?"; - me->remove = sqlite::prepare_query(me->conn, remove_sql); + me->remove = sqlite::prepare_query(conn, remove_sql); } SqliteStore::~SqliteStore() {} @@ -140,25 +152,27 @@ SqliteStore::Data SqliteStore::retrieve_all() { return result; } -std::pair SqliteStore::retrieve_one( +std::tuple SqliteStore::retrieve_one( std::string const &ns, std::string const &k) { auto query_str = fmt::format( - "select ns,key,value from store where ns='{}' and key='{}'", ns, k); + "select rowid,ns,key,value from store where ns='{}' and key='{}'", ns, + k); sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str()); - std::pair result{false, ""}; - + auto result = std::make_tuple(false, 0, ""); auto pq = query.get(); auto sqlite_result = sqlite3_step(pq); if (sqlite_result == SQLITE_ROW) { + std::int64_t rowid = sqlite3_column_int64(pq, 0); std::string ns( - (char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0)); - std::string key( (char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1)); + std::string key( + (char *)sqlite3_column_text(pq, 2), sqlite3_column_bytes(pq, 2)); std::string val( - (char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2)); - result.first = true; - result.second = std::move(val); + (char *)sqlite3_column_blob(pq, 3), sqlite3_column_bytes(pq, 3)); + std::get<0>(result) = true; + std::get<1>(result) = rowid; + std::get<2>(result) = std::move(val); } return result; } @@ -167,91 +181,185 @@ void SqliteStore::remove( std::string const &ns, std::set const &keys) { bts_assert(ns.size()); - for (auto &k : keys) { - SQLITE_CHK( - sqlite3_bind_text( - me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->remove.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not delete row in store. Will retry once. error_code: {} " - "ns:{}, k: {}", - r, ns, k); - sleep_for(2s); - r = sqlite3_step(me->remove.get()); + try { + for (auto &k : keys) { + SQLITE_CHK( + sqlite3_bind_text( + me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->remove.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to delete row. Bailing out. error_code: " - "{}", - r); - bts_assert(r == SQLITE_DONE); + "Could not delete row in store. Will retry once. error_code: " + "{} " + "ns:{}, k: {}", + r, ns, k); + sleep_for(2s); + r = sqlite3_step(me->remove.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to delete row. Bailing out. " + "error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); } - SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::remove threw. what: {}", + std::string{ex.what()}); } } void SqliteStore::remove_all(std::string const &ns) { bts_assert(ns.size()); - SQLITE_CHK( - sqlite3_bind_text( - me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->remove_all.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not delete all rows in store. Will retry once. error_code: {} " - "ns:{}", - r, ns); - sleep_for(2s); - r = sqlite3_step(me->remove_all.get()); + try { + SQLITE_CHK( + sqlite3_bind_text( + me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->remove_all.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to delete all rows. Bailing out. " - "error_code: " - "{}", - r); - bts_assert(r == SQLITE_DONE); + "Could not delete all rows in store. Will retry once. error_code: " + "{} " + "ns:{}", + r, ns); + sleep_for(2s); + r = sqlite3_step(me->remove_all.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to delete all rows. Bailing out. " + "error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::remove_all threw. what: {}", + std::string{ex.what()}); } - SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); } -void SqliteStore::insert( +std::int64_t SqliteStore::insert( std::string const &ns, std::string const &k, std::string const &v) { - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->insert.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not insert row in store. Will retry once. error_code: {} ns: " - "{}, k: {}, v: {}", - r, ns, k, v); - sleep_for(2s); - r = sqlite3_step(me->insert.get()); + try { + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->insert.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to insert row. Bailing out. error_code: {}", - r); - bts_assert(r == SQLITE_DONE); + "Could not insert row in store. Will retry once. error_code: {} " + "ns: " + "{}, k: {}, v: {}", + r, ns, k, v); + sleep_for(2s); + r = sqlite3_step(me->insert.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to insert row. Bailing out. error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + const auto rowid = sqlite3_last_insert_rowid(me->conn.get()); + SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); + return rowid; + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::insert threw. what: {}", + std::string{ex.what()}); } - SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); + return 0; +} + +void SqliteStore::insert_many(const std::vector &rows) { + LOG(md, Info, "SqliteStore::insert_many size: {}", rows.size()); + if (rows.empty()) { + return; + } + try { + sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION"); + sqlite3 *conn_rawp = me->conn.get(); + sqlite3_stmt *insert_rawp = me->insert.get(); + sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get(); + for (const auto &row : rows) { + int col_idx = 1; + sqlite3_stmt *stmt; + if (row.id > 0) { + stmt = insert_with_id_rawp; + sqlite3_bind_int(stmt, col_idx++, row.id); + } else { + stmt = insert_rawp; + } + sqlite3_bind_text( + stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC); + sqlite3_bind_text( + stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC); + int result = sqlite3_step(stmt); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + sqlite3_last_insert_rowid(conn_rawp); + } + sqlite3_reset(stmt); + } + sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION"); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::insert_many threw. what: {}", + std::string{ex.what()}); + } +} + +void SqliteStore::insert_many_resolve_ids(std::vector &rows) { + LOG(md, Info, "SqliteStore::insert_many_resolve_ids size: {}", rows.size()); + if (rows.empty()) { + return; + } + sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION"); + sqlite3 *conn_rawp = me->conn.get(); + sqlite3_stmt *insert_rawp = me->insert.get(); + sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get(); + for (auto &row : rows) { + int col_idx = 1; + sqlite3_stmt *stmt; + if (row.id > 0) { + stmt = insert_with_id_rawp; + sqlite3_bind_int(stmt, col_idx++, row.id); + } else { + stmt = insert_rawp; + } + sqlite3_bind_text( + stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC); + sqlite3_bind_text( + stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC); + int result = sqlite3_step(stmt); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + if (row.id > 0) { + bts_assert(sqlite3_last_insert_rowid(conn_rawp) == row.id); + } else { + row.id = sqlite3_last_insert_rowid(conn_rawp); + } + } + sqlite3_reset(stmt); + } + sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION"); } void SqliteStore::update( diff --git a/src/lib/persistence/SqliteStore.h b/src/lib/persistence/SqliteStore.h index 4f0caf725..6a833df80 100644 --- a/src/lib/persistence/SqliteStore.h +++ b/src/lib/persistence/SqliteStore.h @@ -1,7 +1,12 @@ #pragma once #include "IStore.h" +#include "lib/persistence/store_row.h" +#include "lib/utils/Sqlite_utils.h" + +#include #include +#include namespace bts { namespace persistence { @@ -9,22 +14,26 @@ namespace persistence { class SqliteStore : public IStore { public: SqliteStore(std::string const &db); + explicit SqliteStore(sqlite3_ptr db); ~SqliteStore(); static SqliteStore from_path(const std::string &full_path); typedef std::map> Data; Data retrieve_all() override; - std::pair retrieve_one( + + std::tuple retrieve_one( std::string const &ns, std::string const &k) override; void remove( std::string const &ns, std::set const &keys) override; void remove_all(std::string const &ns) override; - void insert( + std::int64_t insert( std::string const &ns, std::string const &k, std::string const &v) override; + void insert_many(const std::vector &rows); + void insert_many_resolve_ids(std::vector &rows); void update( std::string const &ns, std::string const &k, diff --git a/src/lib/persistence/store_row.h b/src/lib/persistence/store_row.h new file mode 100644 index 000000000..1c7b7f079 --- /dev/null +++ b/src/lib/persistence/store_row.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace bts::persistence { + +struct StoreRow { + std::string ns; + std::string key; + std::string value; + /** + * Represents the unique 'rowid' in the sqlite db. A value of 0 indicates that + * the rowid isn't known, and should either be generated by the db (for inserts), + * or resolved by finding the row with matching ns, key, value (for retrieves). + */ + std::int64_t id{0}; +}; + +} // namespace bts::persistence diff --git a/src/lib/utils/checked_arithmetic.h b/src/lib/utils/checked_arithmetic.h new file mode 100644 index 000000000..91e60a6b9 --- /dev/null +++ b/src/lib/utils/checked_arithmetic.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +namespace bts::system::math { + +static std::uint32_t checked_ltou(std::int64_t l); +static std::uint64_t checked_ltoul(std::int64_t l); + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::uint32_t checked_ltou(std::int64_t l) { + if (l < 0) { + throw std::overflow_error{ + "Failed to convert int64 to uint32 (value < 0)"}; + } + if (l > + static_cast(std::numeric_limits::max())) { + throw std::overflow_error{ + "Failed to convert int64 to uint32. value > max)"}; + } + return static_cast(l); +} + +inline std::uint64_t checked_ltoul(std::int64_t l) { + if (l < 0) { + throw std::overflow_error{ + "Failed to convert int64 to uint64 (value < 0)"}; + } + return static_cast(l); +} + +} // namespace bts::system::math diff --git a/src/server/sparkd/analytics/AnalyticsServiceServer.cpp b/src/server/sparkd/analytics/AnalyticsServiceServer.cpp index 7cd0cc84a..83ff900fd 100644 --- a/src/server/sparkd/analytics/AnalyticsServiceServer.cpp +++ b/src/server/sparkd/analytics/AnalyticsServiceServer.cpp @@ -52,7 +52,7 @@ void add_or_update_to_table( const std::string &ns, const std::string &key, const std::string &val) { - auto [exists, existing_val] = store->retrieve_one(ns, key); + auto [exists, rowid, existing_val] = store->retrieve_one(ns, key); LOG(Gen, Info, "add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns, key, val, exists, existing_val); @@ -386,7 +386,8 @@ void AnalyticsServiceServer::save_report( const api::Response<::bts::api::ResultMessage> &resp) { Report report = report_arg; std::string key = std::to_string(report.id()); - auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key); + auto [present, rowid, val] = + me->reports_store->retrieve_one(me->reports_ns, key); if (!present) { int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1; report.set_id(new_id); @@ -408,7 +409,8 @@ void AnalyticsServiceServer::delete_report( const bts::api::analytics::IntWrapper &id, const api::Response<::bts::api::ResultMessage> &resp) { std::string key = std::to_string(id.data()); - auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key); + auto [present, rowid, val] = + me->reports_store->retrieve_one(me->reports_ns, key); api::ResultMessage res; res.set_success(false); if (!present) { diff --git a/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp b/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp index 055e718b4..5acc34f92 100644 --- a/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp +++ b/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp @@ -581,10 +581,11 @@ void negate_id(api::Security *sec) { } void ExchangeSimApplication::persist_book(OrderBook const &b) { - auto existing = me->store->retrieve_one("books", b.sec_desc()); + auto [exists, rowid, oldval] = + me->store->retrieve_one("books", b.sec_desc()); auto val = utils::pb::to_string(b); - if (existing.first) + if (exists) me->store->update("books", b.sec_desc(), val); else me->store->insert("books", b.sec_desc(), val); diff --git a/src/server/sparkd/md/emumba/EmumbaDeribitData.h b/src/server/sparkd/md/emumba/EmumbaDeribitData.h index 81295af80..c35d44075 100644 --- a/src/server/sparkd/md/emumba/EmumbaDeribitData.h +++ b/src/server/sparkd/md/emumba/EmumbaDeribitData.h @@ -18,10 +18,11 @@ struct EmumbaDeribitData { EmumbaDeribitData() : sec_index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(sec_index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(sec_index.get_key(k.value())); + }, + &utils::Dispatcher::current()), md_producer(recorder), listener(sec_index, md_producer) {} }; } // namespace bts::md diff --git a/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp b/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp index 748a89251..70e8e0eee 100644 --- a/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp +++ b/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp @@ -43,10 +43,11 @@ struct EmumbaMain::PImpl { PImpl() : sec_index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(sec_index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(sec_index.get_key(k.value())); + }, + &utils::Dispatcher::current()), producer(recorder), listener(sec_index, producer) { const auto config_path = Path{config::get().config_path()} / diff --git a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp index 4bcd0c9d4..0dc491f5d 100644 --- a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp +++ b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp @@ -1,70 +1,66 @@ #include "ICEMain.h" #include "lib/config/config.h" -#include "lib/md/SecurityDB.h" -#include "lib/md/coconut2.h" -#include "lib/md/ice/ICEUtils.h" -#include "lib/md/protos/co2.pb.h" -#include "lib/md/vendor/OnixsUtils.h" -#include "lib/os/os.h" -#include "lib/utils/Dispatcher.h" -#include "lib/utils/bts_chrono.h" -#include "lib/utils/helper_macros.h" -#include "lib/utils/proto_helper.h" - +#include "lib/config/protos/deployment_type.pb.h" #include "lib/config/protos/market_data.pb.h" #include "lib/config/protos/system_config.pb.h" -#include "lib/utils/thread_manager.h" +#include "lib/md/ice/ICEUtils.h" +#include "lib/md/vendor/OnixsUtils.h" +#include #include #include -#include - -void recursive_copy( - const boost::filesystem::path &src, const boost::filesystem::path &dst) { - if (boost::filesystem::exists(dst)) - throw std::runtime_error(dst.generic_string() + " exists"); - if (boost::filesystem::is_directory(src)) { - boost::filesystem::create_directories(dst); - for (boost::filesystem::directory_entry &item : - boost::filesystem::directory_iterator(src)) - recursive_copy(item.path(), dst / item.path().filename()); - } else if (boost::filesystem::is_regular_file(src)) - boost::filesystem::copy(src, dst); - else - throw std::runtime_error(dst.generic_string() + " not dir or file"); -} using namespace std; using namespace bts::co2; using std::size_t; -namespace bts { -namespace md { -namespace ice { +namespace bts::md::ice { using bts::config::SystemConfig; using Path = boost::filesystem::path; -ICEMain::ICEMain() - : _index(), _recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(_index.get_key(k.value())); - }}), - _producer(_recorder), _handlers(), _listeners() {} +struct ICEMain::PImpl { + ICEOptions options; + std::string config_file; + ICESecurityIndex index; + SecurityIDRecorder recorder; + MDProducer producer; + std::vector>> + handlers; + std::vector> listeners; -ICEMain::~ICEMain() {} + PImpl(ICEOptions options) + : options(options), + config_file((Path{config::get().data_path()} / + "onixs_ice" / this->options.ice_config_file()) + .string()), + index(), recorder( + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(index.get_key(k.value())); + }, + &utils::Dispatcher::current()), + producer(recorder), handlers(), listeners() {} -const IceChannelConfig channel_settings_for(const ICEChannel &channel) { - const auto &config = config::get(); - for (const auto &ch : config.channel()) { - if (ch.name() == channel.name) - return ch; + IceChannelConfig channel_settings_for(const ICEChannel &channel) { + for (const auto &ch : options.channel()) { + if (ch.name() == channel.name) + return ch; + } + return {}; } - return {}; +}; + +ICEMain::ICEMain() : ICEMain(config::get()) {} + +ICEMain::ICEMain(ICEOptions options) + : me_(std::make_unique(std::move(options))) { + me_->options = std::move(options); } +ICEMain::~ICEMain() = default; + onixs_ice::MarketSubscriptions market_subcription_for_channel( const ICEChannel &chan) { onixs_ice::MarketSubscriptions subs; @@ -90,23 +86,30 @@ onixs_ice::MarketSubscriptions market_subcription_for_channel( return subs; } -void ICEMain::start(const std::vector &symbols) { - auto ice_conf_location = - Path{config::get().data_path()} / "onixs_ice/"; - if (!boost::filesystem::is_directory(ice_conf_location)) { - auto conf_path = Path{os::sparkd_wd()} / "data_files" / "onixs_ice/"; - recursive_copy(Path{conf_path}, ice_conf_location); +static void validate_config_file(const std::string &config_file) { + if (!boost::filesystem::exists(config_file)) { + const std::string config_url = + config::get().type() == "dev" + ? "https://community.theice.com/docs/DOC-19989" + : "https://drive.google.com/" + "open?id=13_eyF2pKtn3F7chgsa_75ftUdf7emMbp"; + bts_assert_msgf( + false, "Ice config file not found '{}'. Please download from {}", + config_file, config_url); } +} +void ICEMain::start(const std::vector &symbols) { await_ice_config(); + validate_config_file(me_->config_file); auto license = onixs_create_temporary_license(); const auto &channels = parse_ice_channels(symbols); for (const auto &channel : channels) { unique_ptr phandler = prepare_handler(channel, license.path.string()); - _listeners.push_back(prepare_listener(*phandler, channel)); - _handlers.push_back(std::make_pair(channel, std::move(phandler))); + me_->listeners.push_back(prepare_listener(*phandler, channel)); + me_->handlers.push_back(std::make_pair(channel, std::move(phandler))); std::stringstream symstr; for (const auto &sym : channel.symbols) @@ -115,7 +118,7 @@ void ICEMain::start(const std::vector &symbols) { symstr.str()); } - for (auto &pair : _handlers) { + for (auto &pair : me_->handlers) { LOG(md, Info, "ICE: Init Handler {}", pair.first.to_string()); sleep_for(16s); onixs_ice::MarketSubscriptions subs = @@ -132,7 +135,7 @@ void ICEMain::start(const std::vector &symbols) { } void ICEMain::send_heartbeats() { - for (const auto &listener : _listeners) + for (const auto &listener : me_->listeners) listener->heartbeat(); } @@ -140,20 +143,18 @@ void ICEMain::send_heartbeats() { // Initialize HandlerSettings and allocate the channel Handler unique_ptr ICEMain::prepare_handler( const ICEChannel &channel, const std::string &license) { - const auto &options = channel_settings_for(channel); + const auto &options = me_->channel_settings_for(channel); LOG(md, Info, "Loaded channel options for channel {}:\n{}", channel.to_string(), options.DebugString()); onixs_ice::HandlerSettings settings; settings.licenseDirectory = license; - // - // Logging settings.logLevel = (onixs_ice::LogLevels::Enum)options.onixs_log_mode(); - settings.logDirectory = ice_log_path(); - - // - // Config file - settings.connectivityConfiguration = ice_config_path(); + settings.logDirectory = + (boost::filesystem::path{config::get().data_path()} / + "logging" / "onixs_ice/") + .string(); + settings.connectivityConfiguration = me_->config_file; // // Network settings @@ -231,12 +232,12 @@ unique_ptr ICEMain::prepare_listener( // heartbeats vector producer_channels; for (auto &symbol : channel.symbols) { - producer_channels.push_back(_producer.get_channel( + producer_channels.push_back(me_->producer.get_channel( symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } unique_ptr listener(new ICEListener( - _index, _producer, /*_settlement_recorder,*/ producer_channels)); + me_->index, me_->producer, /*_settlement_recorder,*/ producer_channels)); for (auto &symbol : channel.symbols) listener->add_filter_symbol(symbol.symbol()); @@ -254,6 +255,4 @@ unique_ptr ICEMain::prepare_listener( return listener; } -} // namespace ice -} // namespace md -} // namespace bts +} // namespace bts::md::ice diff --git a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h index 1ea217a39..8c37f9476 100644 --- a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h +++ b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h @@ -1,21 +1,12 @@ #pragma once #include "ICEListener.h" -#include "ICESecurityIndex.h" -#include "lib/md/coconut2.h" -#include "lib/utils/bts_no_copy.h" -#include "onixs.h" #include "server/sparkd/md/platforms/IMDPlatform.h" -#include #include #include -namespace bts { -namespace md { -namespace ice { - -// void run_onixs_ice_server(UserSymbol* sym, size_t symcount); +namespace bts::md::ice { struct ICEChannel; @@ -28,27 +19,22 @@ struct ICEChannel; class ICEMain : public IMDPlatform { public: ICEMain(); + explicit ICEMain(ICEOptions options); + ICEMain(const ICEMain &) = delete; + ICEMain &operator=(const ICEMain &) = delete; ~ICEMain(); void start(const std::vector &symbols); void send_heartbeats() override; - std::unique_ptr - prepare_handler(const ICEChannel &channel, const std::string &license); - std::unique_ptr - prepare_listener(onixs_ice::Handler &handler, const ICEChannel &channel); + std::unique_ptr prepare_handler( + const ICEChannel &channel, const std::string &license); + std::unique_ptr prepare_listener( + onixs_ice::Handler &handler, const ICEChannel &channel); private: - BTS_NO_COPY(ICEMain); - - ICESecurityIndex _index; - SecurityIDRecorder _recorder; - MDProducer _producer; - std::vector>> - _handlers; - std::vector> _listeners; + struct PImpl; + std::unique_ptr me_; }; -} // namespace ice -} // namespace md -} // namespace bts +} // namespace bts::md::ice diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 9ad5c53ba..0a54040ad 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -1,24 +1,18 @@ #include "MDP3Main.h" - +#include "lib/api/md/GlbxMarketDataConfig.pb.h" #include "lib/config/config.h" +#include "lib/config/protos/deployment_type.pb.h" #include "lib/logging/logging.h" #include "lib/md/MBOProcessor.h" #include "lib/md/SecurityDB.h" #include "lib/md/cme/MDP3Utils.h" -#include "lib/md/coconut2.h" -#include "lib/md/protos/co2.pb.h" +#include "lib/md/cme/mdp3_data_directory.h" #include "lib/md/vendor/OnixsUtils.h" -#include "lib/os/os.h" -#include "lib/utils/Dispatcher.h" -#include "lib/utils/proto_helper.h" - -#include "lib/config/protos/market_data.pb.h" -#include "lib/config/protos/system_config.pb.h" +#include "lib/os/spark_data_directory.h" #include "lib/utils/thread_manager.h" #include #include -#include #include #include @@ -29,16 +23,33 @@ using std::size_t; namespace bts { namespace md { -MDP3Main::MDP3Main() - : _index(), _recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(_index.get_key(k.value())); - }}), - _producer(_recorder), _engines(), _handlers(), _listeners(), _recovery() { -} +#define TRADING_STATUS(x) bts::domain::protobufs::TRADINGSTATUS_##x -MDP3Main::~MDP3Main() {} +using Path = boost::filesystem::path; + +struct MDP3Main::PImpl { + Mdp3DataDirectory data_dir; + CMESecurityIndex index; + SecurityIDRecorder recorder; + MDProducer producer; + std::map> engines; + std::map> handlers; + std::vector> listeners; + std::vector> recovery; + PImpl() + : index(), recorder( + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(index.get_key(k.value())); + }, + &utils::Dispatcher::current()), + producer(recorder), engines(), handlers(), listeners(), recovery() {} +}; + +MDP3Main::MDP3Main() + : me_(std::make_unique()) {} + +MDP3Main::~MDP3Main() = default; const CmeChannelConfig channel_settings_for(const string &id) { const auto &config = config::get(); @@ -173,11 +184,11 @@ void MDP3Main::start( auto license = onixs_create_temporary_license(); initialize(license.path.string()); const auto &channels = parse_channels(symbols); - _engines = create_feed_engines(channels); + me_->engines = create_feed_engines(channels); for (const auto &channel : channels) { unique_ptr phandler = prepare_handler(channel.id, mbo); - _listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd)); - _handlers.insert(std::make_pair(channel.id, std::move(phandler))); + me_->listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd)); + me_->handlers.insert(std::make_pair(channel.id, std::move(phandler))); std::stringstream symstr; for (const auto &sym : channel.symbols) { symstr << sym.symbol() << "-" << SecurityType_Name(sym.type()) << " "; @@ -186,7 +197,7 @@ void MDP3Main::start( symstr.str()); } - for (auto &pair : _handlers) { + for (auto &pair : me_->handlers) { onixs::SessionSettings settings = session_settings_for_channel(pair.first, mbo); pair.second->start(settings); @@ -196,7 +207,7 @@ void MDP3Main::start( } void MDP3Main::send_heartbeats() { - for (const auto &listener : _listeners) { + for (const auto &listener : me_->listeners) { listener->heartbeat(); } } @@ -219,7 +230,8 @@ unique_ptr MDP3Main::prepare_handler( onixs::HandlerSettings settings; settings.channel(std::stoul(channel_id)); - settings.feeds().connectivityConfigurationFile(config_path()); + settings.feeds().connectivityConfigurationFile( + (me_->data_dir.config_file()).string()); // Market data options settings.bookManagement().directBooks().maintain(true); @@ -306,7 +318,8 @@ unique_ptr MDP3Main::prepare_handler( logger = new onixs::NullLogger(); } else { onixs::FileLoggerSettings loggerSettings; - loggerSettings.filename(log_path() + "/log_" + channel_id); + loggerSettings.filename( + (me_->data_dir.log_dir() / ("log_" + channel_id)).string()); loggerSettings.severityLevel( (onixs::LogSeverity::Enum)options.onixs_log_mode()); logger = new onixs::FileLogger(loggerSettings); @@ -338,19 +351,19 @@ unique_ptr MDP3Main::prepare_handler( try { auto handler = unique_ptr(new onixs::Handler(settings)); - auto iter = _engines.find(options.feed_engine()); - if (iter == _engines.end()) { + auto iter = me_->engines.find(options.feed_engine()); + if (iter == me_->engines.end()) { LOG(md, Fatal, "Missing feed engine definition '{}' for channel id '{}'. Add a " "feed_engine entry with name:'{}' to the CMEOptions config key.", options.feed_engine(), channel_id, options.feed_engine()); bts_assert_msg( - iter != _engines.end(), "Missing feed engine definition."); + iter != me_->engines.end(), "Missing feed engine definition."); } handler->bindFeedEngine(*(iter->second)); if (recovery) { handler->bindTcpRecovery(*recovery); - _recovery.push_back(std::move(recovery)); + me_->recovery.push_back(std::move(recovery)); } if (logger) { handler->bindLogger(*logger); @@ -375,12 +388,13 @@ unique_ptr MDP3Main::prepare_listener( // heartbeats vector producer_channels; for (auto &symbol : channel.symbols) { - producer_channels.push_back(_producer.get_channel( + producer_channels.push_back(me_->producer.get_channel( symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } unique_ptr listener(new MDP3Listener( - _index, _producer, mbo, producer_channels, trade_summary_dispatcher)); + me_->index, me_->producer, mbo, producer_channels, + trade_summary_dispatcher)); for (auto &symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index c67dc93d4..fc3043ac3 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -29,7 +29,9 @@ struct CMEChannel; */ class MDP3Main : public IMDPlatform { public: - MDP3Main(); + explicit MDP3Main(); + MDP3Main(const MDP3Main &) = delete; + MDP3Main &operator=(const MDP3Main &) = delete; ~MDP3Main(); void start( @@ -39,8 +41,8 @@ class MDP3Main : public IMDPlatform { void send_heartbeats() override; void initialize(const std::string &license_store); - std::unique_ptr - prepare_handler(const std::string &channel, MBOWriter *); + std::unique_ptr prepare_handler( + const std::string &channel, MBOWriter *); std::unique_ptr prepare_listener( onixs::Handler &handler, const CMEChannel &channel, @@ -48,16 +50,8 @@ class MDP3Main : public IMDPlatform { TradeSummarySignalDispatcher *); private: - BTS_NO_COPY(MDP3Main); - - CMESecurityIndex _index; - SecurityIDRecorder _recorder; - MDProducer _producer; - // co2::SettlementRecorder _settlement_recorder; - std::map> _engines; - std::map> _handlers; - std::vector> _listeners; - std::vector> _recovery; + struct PImpl; + std::unique_ptr me_; }; } // namespace md diff --git a/src/server/sparkd/position/PositionServiceServer.cpp b/src/server/sparkd/position/PositionServiceServer.cpp index c3e511409..26c95f74f 100644 --- a/src/server/sparkd/position/PositionServiceServer.cpp +++ b/src/server/sparkd/position/PositionServiceServer.cpp @@ -293,9 +293,9 @@ void PositionServiceServer::update_manual_fill( auto *this_resp = resp.add_response(); auto id = std::to_string(mf.record_id()); - auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id); - auto found = p.first; - if (!found) { + auto [exists, rowid, val] = + me->manual_fills_store->retrieve_one(manual_fills_ns, id); + if (!exists) { this_resp->set_ok(false); this_resp->set_msg("Could not find record"); } else { @@ -326,9 +326,9 @@ void PositionServiceServer::remove_manual_fill( auto *this_resp = resp.add_response(); auto id = std::to_string(mf.record_id()); - auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id); - auto found = p.first; - if (!found) { + auto [exists, rowid, val] = + me->manual_fills_store->retrieve_one(manual_fills_ns, id); + if (!exists) { this_resp->set_ok(false); this_resp->set_msg("Could not find record"); } else { @@ -367,10 +367,9 @@ void PositionServiceServer::update_security_parameters( req.op_params_size()); for (const auto &p : req.op_params()) { std::string key_str = std::to_string(p.security_key()); - auto db_p = + auto [exists, rowid, val] = me->security_params_store->retrieve_one(security_params_ns, key_str); - auto found = db_p.first; - if (!found) { + if (!exists) { me->security_params_store->insert( security_params_ns, key_str, utils::pb::to_string(p)); } else { diff --git a/src/server/sparkd/user/AdminServiceDB.cpp b/src/server/sparkd/user/AdminServiceDB.cpp index 755ff079d..c89407e0e 100644 --- a/src/server/sparkd/user/AdminServiceDB.cpp +++ b/src/server/sparkd/user/AdminServiceDB.cpp @@ -21,7 +21,8 @@ AdminServiceDB::AdminServiceDB() : me(new PImpl()) {} AdminServiceDB::~AdminServiceDB() {} std::optional AdminServiceDB::retrieve() { - auto [has_table, data] = me->store.retrieve_one(table_name, unique_key); + auto [has_table, rowid, data] = + me->store.retrieve_one(table_name, unique_key); if (!has_table) return {}; @@ -33,7 +34,8 @@ std::optional AdminServiceDB::retrieve() { } bool AdminServiceDB::update(api::admin::AdminData const &state) { - auto [has_table, data] = me->store.retrieve_one(table_name, unique_key); + auto [has_table, rowid, data] = + me->store.retrieve_one(table_name, unique_key); UNUSED(data); std::string ser_state; From 960149b9c56745551b86c29c438aebdfdc07eeca Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 22:49:22 +0000 Subject: [PATCH 28/30] Fix: Stupid bug on sleep time --- src/lib/orderman/gateways/OnixFIXAdapter.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/orderman/gateways/OnixFIXAdapter.cpp b/src/lib/orderman/gateways/OnixFIXAdapter.cpp index 3072342eb..a5cdfda40 100644 --- a/src/lib/orderman/gateways/OnixFIXAdapter.cpp +++ b/src/lib/orderman/gateways/OnixFIXAdapter.cpp @@ -17,8 +17,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -83,8 +85,7 @@ void OnixFIXAdapter::start() { handleException("OnixFIXAdapter::start", err); break; // Exit if retries are exhausted } - - sleep(fmin(60,retryInterval)); // Pause before retry + std::this_thread::sleep_for(std::chrono::seconds(std::max(60,retryInterval))); } } } From 0593ae032055ecaee8ae22b33ff57db17c52027e Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 23:48:24 +0000 Subject: [PATCH 29/30] Fix: Logging path was incorrect --- src/lib/md/cme/mdp3_data_directory.h | 2 +- src/lib/os/spark_data_directory.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/md/cme/mdp3_data_directory.h b/src/lib/md/cme/mdp3_data_directory.h index 740d46dbb..ced0f3457 100644 --- a/src/lib/md/cme/mdp3_data_directory.h +++ b/src/lib/md/cme/mdp3_data_directory.h @@ -8,7 +8,7 @@ namespace bts::md { struct Mdp3DataDirectory { - boost::filesystem::path log_dir() const { return root.log_dir() / "onixs"; } + boost::filesystem::path log_dir() const { return root.log_dir() / "onix"; } boost::filesystem::path data_dir() const { return root.root() / "cme_mdp3"; } diff --git a/src/lib/os/spark_data_directory.cpp b/src/lib/os/spark_data_directory.cpp index fea551a9e..edd1ced61 100644 --- a/src/lib/os/spark_data_directory.cpp +++ b/src/lib/os/spark_data_directory.cpp @@ -1,6 +1,7 @@ #include "spark_data_directory.h" #include "lib/config/config.h" #include "lib/config/protos/system_config.pb.h" +#include "lib/os/os.h" #include #include @@ -16,7 +17,7 @@ SparkDataDirectory::SparkDataDirectory(const std::string &root) : root_(root) {} boost::filesystem::path SparkDataDirectory::root() const { return root_; } boost::filesystem::path SparkDataDirectory::log_dir() const { - return root_ / "logging"; + return root_ / fmt::format("logging.{}/", os::get_application_name()); } boost::filesystem::path SparkDataDirectory::log_file() const { From 0e893b9b5d74ad4f5db2c4fe5bbed203c6e62446 Mon Sep 17 00:00:00 2001 From: Mohammad Arham Date: Wed, 20 Nov 2024 23:48:50 +0000 Subject: [PATCH 30/30] Fix: Multiple Exchange data sources work without timee --- src/server/sparkd/md/MarketDataGateway.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/server/sparkd/md/MarketDataGateway.cpp b/src/server/sparkd/md/MarketDataGateway.cpp index 930a31672..2303c7ba8 100644 --- a/src/server/sparkd/md/MarketDataGateway.cpp +++ b/src/server/sparkd/md/MarketDataGateway.cpp @@ -140,7 +140,6 @@ void MarketDataGateway::run() { _start_emumba_deribit(me_->md_config.emumba_deribit().at(0)); } start_glbx(me_->md_config); - std::this_thread::sleep_for(std::chrono::seconds(5)); start_taifex(me_->md_config); start_cboe(me_->md_config); _start_replays(me_->md_config); @@ -399,7 +398,7 @@ void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg) { auto p = me_->venue_listeners.emplace_back( std::make_shared(bts::api::VENUE_CBOE)); std::string tid_name = "CBOE"; - + // Capture `p->listener` and `me_->sec_id_store` by value auto listener = p->listener; auto sec_id_store = me_->sec_id_store;