diff --git a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs index ffca5e43f..78d90181d 100644 --- a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs +++ b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs @@ -138,7 +138,7 @@ _dispose = Disposable.Create(resources.ToArray()); static string fmt_sec(TimeAndSalesRowVM row) => row.security.ToString(); - static string fmt_time(TimeAndSalesRowVM row) => row.time.ToString("h:mm:ss tt"); + static string fmt_time(TimeAndSalesRowVM row) => row.time.ToString("h:mm:ss.ffffff"); static string fmt_price(TimeAndSalesRowVM row) => row.security.spec_or_default().format(row.price); static string fmt_aggressor(TimeAndSalesRowVM row) => row.aggressor_side.ToString(); static string fmt_volume(TimeAndSalesRowVM row) => new TickFactors(1, 100).format(row.volume); diff --git a/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs b/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs index 75849d9b0..d412d2b86 100644 --- a/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs +++ b/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs @@ -46,7 +46,7 @@ Filled = 56, Symbol = 60, Time = 112, - TimeOfDay = 64, + TimeOfDay = 84, TimeInForce = 47, User = 53, Aggressor = 80, diff --git a/src/client/spark_api/data/ticks/TickExtensions.cs b/src/client/spark_api/data/ticks/TickExtensions.cs index d6a88ec4b..ad564f133 100644 --- a/src/client/spark_api/data/ticks/TickExtensions.cs +++ b/src/client/spark_api/data/ticks/TickExtensions.cs @@ -69,8 +69,16 @@ namespace mbts { public static bool is_valid(this RegionalTickFactors regional_tick) => regional_tick.low_tick.is_valid() && regional_tick.high_tick.is_valid(); - public static bool is_integer(this TickFactors tick) => - tick.tick_unit == tick.tick_base; + public static bool is_integer(this TickFactors tick) { + if (tick.tick_unit == tick.tick_base) { + return true; + } + if (tick.tick_unit > tick.tick_base && tick.tick_base != 0) { + var units_per_base = (int)(tick.tick_unit / tick.tick_base); + return units_per_base * tick.tick_base == tick.tick_unit; + } + return false; + } public static TickFactors tick_factors(this RegionalTickFactors regional_tick, double value) => value > regional_tick.low_tick_inclusive_end ? diff --git a/src/client/spark_api/data/ticks/TickUtils.cs b/src/client/spark_api/data/ticks/TickUtils.cs index e3c783cce..94bae68a7 100644 --- a/src/client/spark_api/data/ticks/TickUtils.cs +++ b/src/client/spark_api/data/ticks/TickUtils.cs @@ -295,6 +295,9 @@ namespace mbts.spark { var ticks_per_dollar = (uint)(tf.low_tick.tick_base / tf.low_tick.tick_unit); // if ticks_per_row is greater than ticks_per_dollar, round to a multiple of ticks_per_dollar if (ticks_per_row > ticks_per_dollar) { + if (ticks_per_dollar == 0) { + return null; + } return (uint)(ticks_per_dollar * Math.Floor((double)(ticks_per_row / ticks_per_dollar))); } else if (ticks_per_row == ticks_per_dollar) { return ticks_per_row; diff --git a/src/lib/api/md/DatabentoMarketDataConfig.proto b/src/lib/api/md/DatabentoMarketDataConfig.proto index cea25526c..184df7e46 100644 --- a/src/lib/api/md/DatabentoMarketDataConfig.proto +++ b/src/lib/api/md/DatabentoMarketDataConfig.proto @@ -14,4 +14,5 @@ message DatabentoMarketDataConfig { string key = 60; string dataset = 70; bool filter_securities_one_year= 80; + bool load_reference_data = 90; } diff --git a/src/lib/api/service_helpers/AdminServiceUtils.cpp b/src/lib/api/service_helpers/AdminServiceUtils.cpp index e2dcc4bcf..5db17b05f 100644 --- a/src/lib/api/service_helpers/AdminServiceUtils.cpp +++ b/src/lib/api/service_helpers/AdminServiceUtils.cpp @@ -156,6 +156,8 @@ AdminServiceUtils::resolve_connection( const bts::api::admin::ExchangeConnection *rv = nullptr; switch (sec.exchange()) { case api::Venue::VENUE_GLBX: { + //CME EXECUTION NEEDS TO BE UPDATED, probably merge with VENUE_BINANCE_USDS_FUTURES + auto deployment_type = boost::to_lower_copy(config::get().type()); rv = resolve_cme_connection(data, account, sec); @@ -227,6 +229,7 @@ void AdminServiceUtils::fill_connections_from_config( connection.set_name(c.name()); switch (c.venue()) { + //CME EXECUTION NEEDS TO BE UPDATED, probably merge default case case bts::api::VENUE_GLBX: if (c.cme_market_segment_id() == 0) connection.mutable_cme_connection()->mutable_cgw_connection(); @@ -236,10 +239,7 @@ void AdminServiceUtils::fill_connections_from_config( ->set_market_segment_id(c.cme_market_segment_id()); add_connection(admin_data, connection); break; - case bts::api::VENUE_CBOE: - break; - case bts::api::VENUE_TAIFEX: - break; + case bts::api::VENUE_SIM: case bts::api::VENUE_SIMICE: connection.mutable_sim_connection(); @@ -250,6 +250,8 @@ void AdminServiceUtils::fill_connections_from_config( c.sender_compid()); add_connection(admin_data, connection); break; + case bts::api::VENUE_TAIFEX: + case bts::api::VENUE_CBOE: default: connection.mutable_general_connection()->set_exchange(c.venue()); add_connection(admin_data, connection); @@ -571,6 +573,13 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record( return rv; } switch (sec.exchange()) { + case api::Venue::VENUE_CBOE:{//subaccount not added + rv.set_session("cboe_1"); + rv.set_account(account_ptr->name()); + rv.set_operator_id(user_ptr->name()); + out_result.set_is_success(true); + return rv; + } case api::Venue::VENUE_DERIBIT: { rv.set_session("deribit_1"); rv.set_account(account_ptr->name()); @@ -733,6 +742,13 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record( rv.set_customer_or_firm(sim_account_data->origin()); return rv; } break; + case api::Venue::VENUE_TAIFEX:{ + rv.set_session("taifex_1"); + rv.set_account(account_ptr->name()); + rv.set_operator_id(user_ptr->name()); + out_result.set_is_success(true); + return rv; + } case api::VENUE_FTX: // TODO case api::VENUE_BINANCE: // TODO case api::VENUE_BINANCE_US: @@ -741,8 +757,6 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record( case api::VENUE_OKX: case api::Venue::VENUE_UNSET: case api::Venue::VENUE_BTS: - case api::Venue::VENUE_CBOE: - case api::Venue::VENUE_TAIFEX: case api::Venue::VENUE_CBOECFE: case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_: case api::Venue::Venue_INT_MAX_SENTINEL_DO_NOT_USE_: diff --git a/src/lib/config/profiles/default/marex_fix_cfg.conf b/src/lib/config/profiles/default/marex_fix_cfg.conf new file mode 100644 index 000000000..6e6aff44d --- /dev/null +++ b/src/lib/config/profiles/default/marex_fix_cfg.conf @@ -0,0 +1,22 @@ +MarexFixCfg { + value{ + customer_or_firm :0 + handl_inst: 2 + rule80a: 1 + heart_bt_int : 30 + counterparty_port :22754 + name: "trafix" + msg_type: "Order_Single" + protocol_version: "FIX_42" + sender_sub_id: "BTS" + target_sub_id: "TRFXMRX" + client_id: "BTS" + counterparty_host : "74.217.55.230" + open_close :"O" + max_retries: 500 + retry_interval: 600 + ex_destination : "W" + security_exchange : "CBOE" + currency : "USD" + } +} \ No newline at end of file diff --git a/src/lib/config/profiles/default/taifex_reference_data.conf b/src/lib/config/profiles/default/taifex_reference_data.conf index d46a59daa..4cfaefada 100644 --- a/src/lib/config/profiles/default/taifex_reference_data.conf +++ b/src/lib/config/profiles/default/taifex_reference_data.conf @@ -6,8 +6,8 @@ tickConfig tick_entry {key:"TXF",value:1}, tick_entry {key:"MXF",value:1}, tick_entry {key:"TMF",value:1}, - tick_entry {key:"CDF",value:1}, - tick_entry {key:"QFF",value:1}, + tick_entry {key:"CDF",value:5}, + tick_entry {key:"QFF",value:5}, } } diff --git a/src/lib/config/protos/marex_fix_cfg.proto b/src/lib/config/protos/marex_fix_cfg.proto new file mode 100644 index 000000000..9635aafb0 --- /dev/null +++ b/src/lib/config/protos/marex_fix_cfg.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; +package bts.config; +message MarexFixCfg { + enum HandlInst { + UNDEFINED = 0; + DMA = 1; + ALGO = 2; + CARE = 3; + } + enum Rule80A { + UNUSED = 0; + A = 1; + P = 2; + } + string msg_type = 1; + string protocol_version = 2; + HandlInst handl_inst = 3; + string sender_sub_id = 4; + string target_sub_id = 5; + string client_id = 6; + Rule80A rule80a = 7; + string counterparty_host = 8; + int32 heart_bt_int = 9; + int32 counterparty_port = 10; + uint32 customer_or_firm =11; + string open_close =12; + string name = 13; + int32 max_retries = 14; + uint32 retry_interval = 15; + string ex_destination = 16; + string security_exchange = 17; + string currency = 18; +} \ No newline at end of file diff --git a/src/lib/core/TimeSpan.h b/src/lib/core/TimeSpan.h index 23970e616..5e0d14458 100644 --- a/src/lib/core/TimeSpan.h +++ b/src/lib/core/TimeSpan.h @@ -50,7 +50,9 @@ class TimeSpan { constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept; +constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept; +constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept; std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept; @@ -116,10 +118,18 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept { return lhs.nanos() < rhs.nanos(); } +constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept { + return lhs.nanos() <= rhs.nanos(); +} + constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept { return lhs.nanos() > rhs.nanos(); } +constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept { + return lhs.nanos() >= rhs.nanos(); +} + } // namespace bts::core FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string()); diff --git a/src/lib/domain_model/Security.cpp b/src/lib/domain_model/Security.cpp index 815d5fcee..b79c3635c 100644 --- a/src/lib/domain_model/Security.cpp +++ b/src/lib/domain_model/Security.cpp @@ -3,6 +3,7 @@ #include "lib/config/config.h" #include "lib/config/protos/orders_fills_cleanup.pb.h" +#include "lib/md/common.h" #include "lib/utils/datetime.h" #include @@ -11,51 +12,15 @@ namespace bts { namespace securities { -std::string to_str(api::Security const &sec) { - std::string serialized; - google::protobuf::TextFormat::PrintToString(sec, &serialized); - return serialized; -} - -std::pair from_str(std::string const &str) { - api::Security result; - auto success = google::protobuf::TextFormat::ParseFromString(str, &result); - return std::make_pair(success, result); -} - -std::string descriptor(api::Security const &sec) { - switch (sec.ExchangeData_case()) { - case api::Security::kTaifex: - return sec.taifex().security_desc(); - case api::Security::kGlbx: - return sec.glbx().security_desc(); - case api::Security::kCboe: - return sec.cboe().security_desc(); - case api::Security::kCfe: - return sec.cfe().security_desc(); - case api::Security::kIce: - return sec.ice().security_desc(); - case api::Security::kDeribit: - return sec.deribit().security_desc(); - case api::Security::kFtx: - return sec.ftx().security_desc(); - case api::Security::kBinance: - return sec.binance().security_desc(); - case api::Security::kBinanceUsdsFutures: - return sec.binance_usds_futures().security_desc(); - case api::Security::kBinanceUs: - return sec.binance_us().security_desc(); - case api::Security::kCoinbase: - return sec.coinbase().security_desc(); - case api::Security::kOkx: - return sec.okx().security_desc(); - default: - return "No descriptor for non glbx / ice exchange"; - } -} +using bts::api::Security; +using bts::api::Venue; api::SecurityType security_type(api::Security const &security) { - switch (security.Contract_case()) { + return security_type(security.Contract_case()); +} + +api::SecurityType security_type(api::Security::ContractCase contract_case) { + switch (contract_case) { case api::Security::ContractCase::kFuture: return api::SecurityType::SECURITYTYPE_FUTURE; case api::Security::ContractCase::kOption: @@ -70,16 +35,6 @@ api::SecurityType security_type(api::Security const &security) { return api::SecurityType::SECURITYTYPE_UNSET; } return api::SecurityType::SECURITYTYPE_UNSET; - // if (sec.has_future()) - // return api::SecurityType::SECURITYTYPE_FUTURE; - // else if (sec.has_option()) - // return api::SecurityType::SECURITYTYPE_OPTION; - // else if (sec.has_spread()) - // return api::SecurityType::SECURITYTYPE_SPREAD; - // else if (sec.has_synthetic()) - // return api::SecurityType::SECURITYTYPE_SYNTHETIC; - // else - // return api::SecurityType::SECURITYTYPE_UNSET; } boost::gregorian::date smallest_leg( @@ -140,5 +95,91 @@ bool is_expired(api::Security const &security) { return is_expired(security, datetime::local_now()); } +void write_date_string( + uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result) { + result << std::setw(2) << std::setfill('0') << month << "/" << std::setw(2) + << std::setfill('0') << day << "/" << year - 2000; +} + +std::ostringstream &write_leg_descriptor( + std::ostringstream &result, char separator, const api::ContractLeg &leg) { + bts_assert(leg.security().id() != md::invalid_security_key); + // Keep the string small, yet readable for diagnostic purposes. + result << (leg.side() == api::SIDE_BUY ? '+' : '-') << separator + << leg.quantity() << "*(i=" << leg.security().id() + << ",m=" << leg.price_multiplier() << ",d=" << leg.leg_delta() << ")"; + return result; +} + +std::ostringstream &write_future_descriptor( + std::ostringstream &result, const api::FutureContract &future) { + write_date_string( + future.expiration_year(), future.expiration_month(), + future.expiration_day(), result); + return result; +} + +std::ostringstream &write_option_descriptor( + std::ostringstream &result, + char separator, + const api::OptionContract &option) { + write_date_string( + option.expiration_year(), option.expiration_month(), + option.expiration_day(), result); + result << separator << option.strike() + << (option.call_put() == api::PUTORCALL_CALL ? 'C' : 'P'); + return result; +} + +std::ostringstream &write_legs_descriptor( + std::ostringstream &result, + char separator, + const google::protobuf::RepeatedPtrField &legs) { + bts_assert(legs.size() > 0); + auto it = legs.begin(); + write_leg_descriptor(result, separator, *it); + while (++it != legs.end()) { + result << separator; + write_leg_descriptor(result, separator, *it); + } + return result; +} + +std::string unique_descriptor(const api::Security &security) { + const char separator = ' '; + std::ostringstream result; + result << api::Venue_Name(security.exchange()).substr(6) << separator + << api::SecurityType_Name(security_type(security)).substr(12) + << descriptor(security); + switch (security.Contract_case()) { + case Security::ContractCase::kOption: + result << separator; + write_option_descriptor(result, separator, security.option()); + break; + case Security::ContractCase::kFuture: + result << separator; + write_future_descriptor(result, security.future()); + break; + case Security::ContractCase::kSpot: + break; + case Security::ContractCase::kSpread: + result << separator; + write_legs_descriptor(result, separator, security.spread().legs()); + break; + case Security::ContractCase::kSynthetic: { + result << "pc=" + << api::SyntheticContract::PriceCalc_Name( + security.synthetic().price_calc()); + write_legs_descriptor(result, separator, security.synthetic().legs()); + break; + } + default: + bts_assert_msg( + false, "unique_descriptor failed due to invalid contract case"); + break; + } + return result.str(); +} + } // namespace securities } // namespace bts diff --git a/src/lib/domain_model/Security.h b/src/lib/domain_model/Security.h index 57bd75062..fe5a3ac32 100644 --- a/src/lib/domain_model/Security.h +++ b/src/lib/domain_model/Security.h @@ -2,6 +2,7 @@ #include "lib/core/FMT_IMPL.h" #include "lib/domain_model/Security.pb.h" +#include "lib/domain_model/SecurityType.h" #include "lib/domain_model/Side.h" #include "lib/domain_model/Venue.h" #include "lib/domain_model/common.h" @@ -25,11 +26,27 @@ class ICEExchangeData; } // namespace api namespace securities { -SecurityKey security_key(api::Security const &); -std::string to_str(api::Security const &); -std::pair from_str(std::string const &); -std::string descriptor(api::Security const &); + +/** + * Gets a descriptor string for a security (should be unique within a particular exchange). + */ +const std::string &descriptor( + const api::Security &sec, const std::string &default_rv = "UNDEFINED"); + +/** + * Gets a descriptor string for a security that is unique within the entire + * system (not just exchange). + */ +std::string unique_descriptor(const api::Security &security); + api::SecurityType security_type(api::Security const &); +api::SecurityType security_type(api::Security::ContractCase contract_case); + +void write_date_string( + std::uint32_t year, + std::uint32_t month, + std::uint32_t day, + std::ostringstream &result); inline std::string const &side_name(api::Side side) { return api::Side_Name(side); @@ -234,6 +251,41 @@ inline Price tick_size(api::Security const &sec) { return Price(sec.contract_spec().low_tick()); } +inline const std::string &descriptor( + const api::Security &sec, const std::string &default_rv) { + switch (sec.ExchangeData_case()) { + case api::Security::kGlbx: + return sec.glbx().security_desc(); + case api::Security::kCboe: + return sec.cboe().security_desc(); + case api::Security::kCfe: + return sec.cfe().security_desc(); + case api::Security::kIce: + return sec.ice().security_desc(); + case api::Security::kDeribit: + return sec.deribit().security_desc(); + case api::Security::kFtx: + return sec.ftx().security_desc(); + case api::Security::kBinance: + return sec.binance().security_desc(); + case api::Security::kBinanceUsdsFutures: + return sec.binance_usds_futures().security_desc(); + case api::Security::kBinanceUs: + return sec.binance_us().security_desc(); + case api::Security::kCoinbase: + return sec.coinbase().security_desc(); + case api::Security::kOkx: + return sec.okx().security_desc(); + case api::Security::kTaifex: + return sec.taifex().security_desc(); + default: { + if (sec.has_synthetic()) { + return sec.synthetic().name(); + } + return default_rv; + } + } +} // Looking to round md::IntPrice? Look in IntPrice.h // computes price_2 - price_1 in ticks @@ -314,3 +366,5 @@ inline bool is_spread_of_strips(const bts::api::Security &sec) { } // namespace securities } // namespace bts + +FMT_IMPL(bts::api::Security, s, "{}", bts::securities::descriptor(s)); diff --git a/src/lib/domain_model/SecurityType.h b/src/lib/domain_model/SecurityType.h new file mode 100644 index 000000000..5c4b78e00 --- /dev/null +++ b/src/lib/domain_model/SecurityType.h @@ -0,0 +1,30 @@ +#pragma once + +#include "lib/domain_model/Security.pb.h" +#include "lib/core/FMT_IMPL.h" +#include "lib/core/EnumMacros.h" +#include "lib/io/JsonReader.h" +#include "lib/logging/logging.h" + +#include + +namespace bts::api { + +PB_ENUM_FROM_JSON(SecurityType) +PB_ENUM_TO_JSON(SecurityType) + +std::ostream &operator<<(std::ostream &out, const SecurityType &type) noexcept; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::ostream &operator<<( + std::ostream &out, const SecurityType &type) noexcept { + out << SecurityType_Name(type); + return out; +} + +} // namespace bts::api + +FMT_IMPL(bts::api::SecurityType, t, "{}", bts::api::SecurityType_Name(t)); diff --git a/src/lib/domain_model/Venue.h b/src/lib/domain_model/Venue.h index fe05c6c61..e69c9d6c3 100644 --- a/src/lib/domain_model/Venue.h +++ b/src/lib/domain_model/Venue.h @@ -13,6 +13,17 @@ namespace bts::api { PB_ENUM_FROM_JSON(Venue) PB_ENUM_TO_JSON(Venue) +std::ostream &operator<<(std::ostream &out, Venue venue) noexcept; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::ostream &operator<<(std::ostream &out, Venue venue) noexcept { + out << Venue_Name(venue); + return out; +} + } // namespace bts::api FMT_IMPL(bts::api::Venue, v, "{}", bts::api::Venue_Name(v)); diff --git a/src/lib/md/Diag/ShmDumper.cpp b/src/lib/md/Diag/ShmDumper.cpp index 2ab4eb7fa..dc04f67f7 100644 --- a/src/lib/md/Diag/ShmDumper.cpp +++ b/src/lib/md/Diag/ShmDumper.cpp @@ -3,6 +3,7 @@ #include "ShmDumper.h" #include "../Shm/MDStructure.h" +#include "lib/domain_model/Security.h" #include "lib/io/JsonDumper.h" #include "lib/logging/logging.h" #include "lib/utils/SharedList.h" @@ -51,85 +52,8 @@ string dump_header(const ColdHeader &header) { return JsonDumper::dump(header); } -void write_date_string( - uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result); string ShmDumper::dump_security(const FixedSecurity &security) { - ostringstream res; - res << "{"; - res << "'key': " << security.key << ", "; - res << "'symbol': '" << security.symbol() << "', "; - res << "'type': '" << api::SecurityType_Name(security.type) << "', "; - res << "'exchange': '" << api::Venue_Name(security.exchange) << "', "; - res << "'alt_symbol': '" << security.alt_symbol() << "', "; - res << "'cfi_code': '" << security.cfi_code() << "', "; - res << "'security_desc': '" << security.security_desc() << "', "; - res << "'platform_id': '" << security.platform_id() << "', "; - res << "'display_factor': " << (int)security.display_factor << ", "; - res << "'numeric_id': " << security.numeric_id << ", "; - res << "'cboe_class_key': " << security.cboe_class_key << ", "; - res << "'main_frac': " << (int)security.cme_main_fraction << ", "; - res << "'sub_fract': " << (int)security.cme_sub_fraction << ", "; - res << "'price_format': " << (int)security.cme_price_display_format << ", "; - res << "'market_segment': " << (int)security.cme_market_segment_id << ", "; - - res << "'price_break': " << security.price_break << ", "; - res << "'low_tick': " << security.low_tick << ", "; - res << "'high_tick': " << security.high_tick << ", "; - res << "'cpp': " << security.cash_per_point << ", "; - - res << "'market_description ': " << security.ice_market_description() - << ", "; - res << "'cleared_alias ': " << security.ice_cleared_alias() << ", "; - res << "'product_description ': " << security.ice_product_name() << ", "; - res << "'hub_alias ': " << security.ice_hub_alias() << ", "; - res << "'strip_name': " << security.ice_strip_name() << ", "; - res << "'mic_code': " << security.mic_code() << ", "; - - switch (security.type) { - case SECURITYTYPE_SPOT: - break; - case SECURITYTYPE_FUTURE: { - auto *psec = static_cast(&security); - res << "'expiration': '"; - write_date_string( - psec->expiration_year, psec->expiration_month, psec->expiration_day, - res); - res << "' "; - break; - } - case SECURITYTYPE_OPTION: { - auto *psec = static_cast(&security); - res << "'expiration': '"; - write_date_string( - psec->expiration_year, psec->expiration_month, psec->expiration_day, - res); - res << "', "; - res << "'strike': " << psec->strike << ", "; - res << "'callput': '" << api::PutOrCall_Name(psec->call_put) << "' "; - break; - } - case SECURITYTYPE_SPREAD: { - auto *psec = static_cast(&security); - res << "'legs': ["; - for (size_t x = 0; x < psec->leg_count(); ++x) { - auto &leg = psec->legs[x]; - if (x > 0) - res << ","; - res << "{ 'key': " << leg.key << ", 'exkey': " << leg.exchange_key - << ", 'quantity': " << leg.leg_quantity - << ", 'leg_price': " << leg.leg_price - << ", 'delta': " << leg.leg_delta << ", 'side': '" - << api::Side_Name(leg.leg_side) << "' }"; - } - res << "]"; - break; - } - default: - break; - } - - res << "}"; - return res.str(); + return JsonDumper::dump(security); } string dump(datetime::Date date) { diff --git a/src/lib/md/GeneralMarketDataListener.cpp b/src/lib/md/GeneralMarketDataListener.cpp index 54058c5e4..e9aaea151 100644 --- a/src/lib/md/GeneralMarketDataListener.cpp +++ b/src/lib/md/GeneralMarketDataListener.cpp @@ -23,12 +23,12 @@ #include "lib/domain_model/Security.pb.h" #include "lib/domain_model/common.h" #include "lib/logging/logging.h" +#include "lib/md/GeneralSecurityIndex.h" #include "lib/md/IMarketDataListener.h" #include "lib/md/IntPrice.h" #include "lib/md/MDProducer.h" #include "lib/md/MDProducerHandle.h" #include "lib/md/PriceSize.h" -#include "lib/md/SecurityIDRecorder.h" #include "lib/md/SecurityIndex.h" #include "lib/md/Shm/FixedSecurity.h" #include "lib/md/Shm/MDStructure.h" @@ -274,10 +274,11 @@ bool update_incremental(const IMarketDataListener::BookOp &op) { }; struct GeneralMarketDataListener::PImpl { + const api::domain::Venue venue; // if this is set to true, then the security key passed on in the // on_securities function is used. If this is not set to true, then a new key // is generated - const bool use_given_sec_key = true; + const bool use_given_sec_key = false; const std::string cc_name_prefix = "cc_security"; size_t n_bk_updates = 0; bts::persistence::IDGenerator &id_gen = @@ -296,12 +297,15 @@ struct GeneralMarketDataListener::PImpl { MDArray asks = {}; IncrementalBookData inc_book_data = {}; explicit PImpl( + api::domain::Venue venue, GeneralSecurityIndex &index, MDProducer &md_producer, std::shared_ptr sim_ex_gwy) - : index(index), md_producer(md_producer), sim_ex_gwy(sim_ex_gwy) { - if (sim_ex_gwy != nullptr) { - sim_ex_gwy->set_producers(&index); + : venue(venue), index(index), + md_producer(md_producer), + sim_ex_gwy(std::move(sim_ex_gwy)) { + if (this->sim_ex_gwy != nullptr) { + this->sim_ex_gwy->set_producers(&this->index); } } }; @@ -363,6 +367,7 @@ void copy_books( GeneralMarketDataListener::GeneralMarketDataListener( GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue) : me(std::make_unique( + domain::DomainSecurityUtils::convert(venue), index, md_producer, bts::orderman::SimExecutionGatewayFactory::get()->get_getayeway( @@ -379,7 +384,7 @@ void GeneralMarketDataListener::on_securities( LOG(md, Info, "GeneralMarketDataListener::on_securities n_secs: {}", complete_security_list.size()); for (auto s_ptr : complete_security_list) { - domain::Security sec = *s_ptr; + const domain::Security &sec = *s_ptr; const auto &s_comm = sec.security_common; CCSecSearchResult search_result = _find_sec(sec, true); LOG(md, Info, @@ -503,11 +508,7 @@ CCSecSearchResult GeneralMarketDataListener::_find_sec( if (add_if_absent) { uint64_t bts_id = cc_id.value(); if (!me->use_given_sec_key) { - auto logical_key = get_logical_key(sec); - auto new_key_opt = - me->id_gen.get_id(me->cc_name_prefix, logical_key, true); - bts_assert(new_key_opt.has_value()); - bts_id = new_key_opt.value(); + bts_id = me->id_gen.get_or_add(me->venue, sec.type(), sec.desc()); } me->cc_sec_id_map.insert({cc_id, {bts_id}}); LOG(md, Info, @@ -654,9 +655,7 @@ uint64_t GeneralMarketDataListener::_generate_security_key( auto logical_key = get_logical_key(sec); // if (me->use_given_sec_key) // return sec.security_common.id.value(); - auto rv = me->id_gen.get_id(me->cc_name_prefix, logical_key, true); - bts_assert(rv.has_value()); - return rv.value(); + return me->id_gen.get_or_add(me->venue, sec.type(), sec.desc()); } void GeneralMarketDataListener::_possibly_match_with_sim_orders( diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index 27b84e073..c33e5814d 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -61,26 +61,6 @@ datetime::Date sec_expiration_cut_off() { return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day()); } -void apply_compatibilit_mutations(Security &security) { - // Migrate from old tick stuff - if (security.has_glbx()) { - if (security.has_glbx()) { - if (security.contract_spec().main_fraction() == 0) { - security.mutable_contract_spec()->set_main_fraction( - security.glbx().do_not_use_main_fraction()); - } - if (security.contract_spec().sub_fraction() == 0) { - security.mutable_contract_spec()->set_sub_fraction( - security.glbx().do_not_use_sub_fraction()); - } - if (security.contract_spec().price_display_format() == 0) { - security.mutable_contract_spec()->set_price_display_format( - security.glbx().do_not_use_price_display_format()); - } - } - } -} - size_t load_securities( std::function const &sym_filter, const std::function &callback) { @@ -104,7 +84,6 @@ size_t load_securities( if (sym_filter(us)) { auto blob = select_stmt.getColumn("security_blob"); if (security.ParseFromArray(blob, blob.getBytes())) { - apply_compatibilit_mutations(security); bts_assert(security_key == security.id()); count++; callback(security); @@ -146,7 +125,6 @@ SecurityPtr find_security(SecurityKey key) { auto result = std::make_shared(); auto blob = select.getColumn(0); if (result->ParseFromArray(blob, blob.getBytes())) { - apply_compatibilit_mutations(*result); return result; } } diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 94c82ad52..f0abccdf8 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -1,18 +1,25 @@ -#include "lib/domain_model/Security.h" #ifndef _WIN32 #include "SecurityIDRecorder.h" +#include "lib/domain_model/Security.h" +#include "lib/persistence/IStore.h" +#include "lib/persistence/SqliteStore.h" +#include #include "lib/config/config.h" #include "lib/logging/logging.h" #include "lib/utils/Sqlite_utils.h" +#include "lib/utils/checked_arithmetic.h" #include #include #include "lib/config/protos/market_data.pb.h" #include "lib/config/protos/system_config.pb.h" +#include #include +#include #include +#include using namespace std; using namespace bts::utils; @@ -21,22 +28,55 @@ using std::size_t; namespace bts { namespace md { +using bts::api::Security; +using bts::api::SecurityType; +using bts::api::Venue; +using bts::config::SystemConfig; +using bts::persistence::IStore; +using bts::persistence::SqliteStore; + uint64_t exchange_id(const api::Security &security); -struct SecurityIDEntry { - api::Security security; - SecurityIDRecorder::Callback callback; +static std::unique_ptr get_default_store() { + return std::make_unique("key_ids"); +} + +static persistence::StoreRow get_row( + Venue venue, + SecurityType type, + const std::string &name, + const std::string &prefix = "") { + return persistence::StoreRow{ + fmt::format( + "{}{}.{}", std::string{prefix}, Venue_Name(venue).substr(6), + SecurityType_Name(type).substr(13)), + name, ""}; +} + +static persistence::StoreRow get_row( + const api::Security &s, const std::string &prefix = "") { + return get_row(s.exchange(), bts::securities::security_type(s), prefix); +} + +struct SecurityIdRequest { + Venue venue; + SecurityType type; + std::string name; + std::function + callback; }; struct SecurityIDRecorderState { - SecurityIDRecorderState(SecurityIDRecorderParams &p) - : work(), pending_handle(), thread_id(), db(nullptr, sqlite3_close), - timer(), lookup_key(p.lookup_key), unresolved_legs() {} + SecurityIDRecorderState( + std::unique_ptr store, KeyLookupOption lookup_key) + : store(store == nullptr ? get_default_store() : std::move(store)), + pending_handle(), thread_id(), timer(), lookup_key(lookup_key), + unresolved_legs() {} - tbb::concurrent_queue> work; + std::unique_ptr store{nullptr}; + tbb::concurrent_queue> id_request_queue; tbb::concurrent_queue pending_handle; std::thread::id thread_id; - sqlite3_ptr db; unique_ptr timer; KeyLookupOption lookup_key; @@ -59,51 +99,41 @@ struct SecurityIDRecorderState { void work_handles(); void work_db(); - void check_access() { bts_assert(thread_id == std::this_thread::get_id()); } void assign_key( const api::Security &security, const SecurityIDRecorder::Callback &key_assigned); + void assign_key( + api::Venue venue, + api::SecurityType type, + const std::string &name, + std::function + key_assigned); + + void check_access() { bts_assert(thread_id == std::this_thread::get_id()); } void add_unresolved_spread(MDProducerHandle &handle); - security_key search(const std::string &descriptor); - security_key insert(const std::string &descriptor, SecurityType); + security_key search(const persistence::StoreRow &row); + security_key insert(const persistence::StoreRow &row); }; -SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params) - : _self(new SecurityIDRecorderState(params)) { - if (params.db) { - _self->db = std::move(params.db); - } +SecurityIDRecorder::SecurityIDRecorder( + std::unique_ptr store, + KeyLookupOption lookup_key, + utils::Dispatcher *dispatcher) + : _self(std::make_unique( + store == nullptr + ? std::make_unique("key_ids") + : std::move(store), + lookup_key)) { init(); - if (params.dispatcher) { - _self->run(*params.dispatcher); + if (dispatcher != nullptr) { + _self->run(*dispatcher); } } SecurityIDRecorder::~SecurityIDRecorder() {} -void SecurityIDRecorder::init() { - LOG(md, Info, "SecurityIDRecorder::init"); - string db_dir = - config::get().data_path() + "databases/md/"; - string db_path = db_dir + "/security_keys.sqlite"; - - if (!boost::filesystem::exists(db_dir)) { - boost::filesystem::create_directories(db_dir); - } - - if (!_self->db) { - _self->db = sqlite::open_db(db_path.c_str()); - bts_assert(_self->db); - } - - // Set a long busy timeout - sqlite3_busy_timeout(_self->db.get(), 9999); - sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); - sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0"); - sqlite::exec_nofail( - _self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING " - "NOT NULL PRIMARY KEY, type int);"); -} +void SecurityIDRecorder::init() {} void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}", @@ -113,37 +143,42 @@ void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { } void SecurityIDRecorder::assign_key( - const api::Security &security, - const SecurityIDRecorder::Callback &callback) { - LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}", - security.DebugString()); - if (security.Contract_case() == Security::ContractCase::kSpread) { - for (const auto &leg : security.spread().legs()) { - if (leg.security().id() == invalid_security_key) { - LOG(md, Error, "Security {} has legs which will never be resolved.", - security.has_glbx() ? security.glbx().security_desc() - : security.symbol()); - // No point in trying to add this to the queue - return; - } - } - } - _self->assign_key(security, callback); + const api::Security &security, const Callback &callback) { + assign_key( + security.exchange(), bts::securities::security_type(security), + bts::securities::descriptor(security), + [callback, msec = security]( + api::Venue, api::SecurityType, const std::string &, + uint32_t id) mutable { + msec.set_id(id); + callback(msec); + }); } void SecurityIDRecorderState::assign_key( - const api::Security &security, - const SecurityIDRecorder::Callback &callback) { - LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}", - bts::securities::descriptor(security)); - if (security.id() == invalid_security_key) { - auto work = std::make_shared(); - work->security = security; - work->callback = callback; - LOG(md, Info, "SecurityIDRecorderState::assign_key work.push d: {}", - bts::securities::descriptor(work->security)); - this->work.push(work); - } + api::Venue venue, + api::SecurityType type, + const std::string &name, + std::function< + void(api::Venue, api::SecurityType, const std::string &, std::uint32_t)> + callback) { + LOG(md, Info, "SecurityIDRecorder::assign_key {} {} {}", venue, type, name); + auto id_req = std::make_shared(); + id_req->venue = venue; + id_req->type = type; + id_req->name = name; + id_req->callback = callback; + id_request_queue.push(id_req); +} + +void SecurityIDRecorder::assign_key( + api::Venue venue, + api::SecurityType type, + const std::string &name, + std::function< + void(api::Venue, api::SecurityType, const std::string &, std::uint32_t)> + callback) { + _self->assign_key(venue, type, name, std::move(callback)); } void SecurityIDRecorder::do_work() { _self->do_work(); } @@ -220,9 +255,12 @@ void SecurityIDRecorderState::work_handles() { retry.push_back(handle); } else { convert_security(handle.security(), &security); - assign_key(security, [handle](const api::Security &security) mutable { - handle.set_key(security.id()); - }); + assign_key( + security.exchange(), bts::securities::security_type(security), + bts::securities::descriptor(security), + [handle]( + api::Venue, api::SecurityType, const std::string &, + uint32_t id) mutable { handle.set_key(id); }); } } @@ -239,33 +277,30 @@ void SecurityIDRecorderState::work_db() { // Then persist that list // Once a security has a key it is public so record it in the security // database - std::shared_ptr temp; - vector> new_entries; + std::shared_ptr temp; + vector>> new_entries; // SPARK-1446 Need to try to add existing securities to the db in case we // deleted them by hand - vector> existing_securities; - vector> defer; - vector new_descriptors; + vector> existing_securities; + vector new_rows; size_t count = 0; - string descriptor; - while (work.try_pop(temp)) { - descriptor = security_descriptor(temp->security); - - // If we're unable to create a descriptor try again later - if (descriptor == "") { - defer.push_back(std::move(temp)); - continue; + while (id_request_queue.try_pop(temp)) { + security_key key = invalid_security_key; + auto row = get_row(temp->venue, temp->type, temp->name); + try { + key = search(row); + } catch (const std::overflow_error &ex) { + LOG(persist, Error, + "SecurityIDRecorderState::work_db. Row search threw ex: {}", + std::string{ex.what()}); } - - security_key key = search(descriptor); if (key != invalid_security_key) { - temp->security.set_id(key); - temp->callback(temp->security); + temp->callback(temp->venue, temp->type, temp->name, key); existing_securities.push_back(std::move(temp)); } else { // always push in parallel - new_entries.push_back(std::move(temp)); - new_descriptors.push_back(descriptor); + new_entries.push_back(std::make_pair(key, std::move(temp))); + new_rows.push_back(std::move(row)); ++count; // Limit the amount of time we spend writing to sqlite during startup // to avoid blocking the heartbeat @@ -275,95 +310,47 @@ void SecurityIDRecorderState::work_db() { } } - for (auto &d : defer) { - LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}", - bts::securities::descriptor(d->security)); - work.push(std::move(d)); - } - if (!new_entries.empty()) { - sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION"); - + store->insert_many_resolve_ids(new_rows); for (size_t x = 0; x < new_entries.size(); ++x) { - Security &security = new_entries[x]->security; - security_key key = - insert(new_descriptors[x], securitytype_of(security)); - bts_assert_d(key); - security.set_id(key); + auto &entry = new_entries[x]; + auto &row = new_rows[x]; + entry.first = row.id; + bts_assert_d(entry.first != invalid_security_key); } - - sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } if (!new_entries.empty()) { for (size_t x = 0; x < new_entries.size(); ++x) { - Security &security = new_entries[x]->security; - new_entries[x]->callback(security); + auto &[id, req] = new_entries[x]; + req->callback(req->venue, req->type, req->name, id); } } } void SecurityIDRecorder::bulk_insert( const std::vector &data, const char *prefix) { - LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size()); - if (data.size() == 0) - return; - sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); - std::string descriptor; - { - auto insert_stmt = sqlite::prepare_query( - _self->db, "INSERT INTO security_keys (rowid, descriptor, type) " - "values (?, ?, ?)"); - for (const api::Security *sec : data) { - descriptor = prefix + security_descriptor(*sec); - auto type = securitytype_of(*sec); - - sqlite3_bind_int(insert_stmt.get(), 1, sec->id()); - sqlite3_bind_text( - insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - sqlite3_bind_int(insert_stmt.get(), 3, (int)type); - sqlite3_step(insert_stmt.get()); - sqlite3_reset(insert_stmt.get()); - } + std::vector rows; + rows.reserve(data.size()); + for (const auto *s : data) { + rows.push_back(get_row(*s, prefix)); } - sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + _self->store->insert_many_resolve_ids(rows); } -security_key SecurityIDRecorderState::search(const std::string &descriptor) { - LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor); - auto select_stmt = sqlite::prepare_query( - db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); - sqlite3_bind_text( - select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - int result = sqlite3_step(select_stmt.get()); - if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_column_int(select_stmt.get(), 0); +security_key SecurityIDRecorderState::search(const persistence::StoreRow &row) { + auto [exists, rowid, val] = store->retrieve_one(row.ns, row.key); + if (exists) { + if (!val.empty()) { + bts_assert(stol(val) == rowid); + } + return system::math::checked_ltou(rowid); } return invalid_security_key; } -security_key SecurityIDRecorderState::insert( - const std::string &descriptor, SecurityType type) { - LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor); - auto insert_stmt = sqlite::prepare_query( - db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); - sqlite3_bind_text( - insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - sqlite3_bind_int(insert_stmt.get(), 2, (int)type); - int result = sqlite3_step(insert_stmt.get()); - if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_last_insert_rowid(db.get()); - } - // Just in case our insert failed due to duplicate row - auto key = search(descriptor); - if (key == invalid_security_key) { - LOG(md, Error, "Failed to assign security key. SQLite return code: {}", - result); - } - return key; +security_key SecurityIDRecorderState::insert(const persistence::StoreRow &row) { + return store->insert(row.ns, row.key, row.value); } } // namespace md diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index 14819bd8d..2925e0c20 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -2,7 +2,7 @@ #ifndef _WIN32 #include "MDProducerHandle.h" #include "common.h" - +#include "lib/persistence/IStore.h" #include "lib/utils/Sqlite_utils.h" #include #include @@ -22,26 +22,15 @@ namespace md { typedef const std::function KeyLookup; typedef std::optional KeyLookupOption; -struct SecurityIDRecorderParams { - SecurityIDRecorderParams() - : dispatcher(), db(sqlite::null_db()), lookup_key() {} - SecurityIDRecorderParams(utils::Dispatcher &d) - : dispatcher(&d), db(sqlite::null_db()), lookup_key() {} - SecurityIDRecorderParams(utils::Dispatcher &d, KeyLookup &get_key) - : dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {} - SecurityIDRecorderParams(sqlite3_ptr &db) - : dispatcher(), db(std::move(db)), lookup_key() {} - SecurityIDRecorderParams(sqlite3_ptr &db, KeyLookup &get_key) - : dispatcher(), db(std::move(db)), lookup_key(get_key) {} - utils::Dispatcher *dispatcher; - sqlite3_ptr db; - KeyLookupOption lookup_key; -}; + struct SecurityIDRecorderState; class SecurityIDRecorder { public: - SecurityIDRecorder(SecurityIDRecorderParams); + SecurityIDRecorder( + std::unique_ptr store = nullptr, + KeyLookupOption lookup_key = std::nullopt, + utils::Dispatcher *dispatcher = nullptr); ~SecurityIDRecorder(); typedef std::function Callback; @@ -49,6 +38,13 @@ class SecurityIDRecorder { void ensure_key(MDProducerHandle handle); // Slow (copies security) - for use by services void assign_key(const api::Security &security, const Callback &key_assigned); + void assign_key( + api::Venue venue, + api::SecurityType type, + const std::string &name, + std::function + key_assigned); // If a dispatcher is not provided you must call work periodically to do // lookups and flush the reserved ids to disk. diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index e8b5c3091..561fa7161 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -1,4 +1,4 @@ -#include "lib/domain_model/Security.pb.h" +#include "lib/domain_model/Security.h" #include "lib/utils/bts_assert.h" #ifndef NO_CPP0X @@ -59,81 +59,6 @@ FixedSpreadLeg::FixedSpreadLeg() : key(invalid_security_key), leg_side(Side_MIN), leg_quantity(0), leg_price(0), leg_delta(1) {} -void write_date_string( - uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result) { - result << std::setw(2) << std::setfill('0') << month << "/" << std::setw(2) - << std::setfill('0') << day << "/" << year - 2000; -} - -void write_leg_descriptor( - std::ostringstream &result, char separator, const api::ContractLeg &leg) { - result << separator << leg.security().id() << "-q" << leg.quantity() << "p" - << leg.price_multiplier() << "d" << leg.leg_delta() - << (leg.side() == SIDE_BUY ? "BUY" : "SELL"); -} - -std::string security_descriptor(const api::Security &security) { - const char separator = ' '; - std::ostringstream result; - if (security.exchange() == Venue::VENUE_ICE || - security.exchange() == Venue::VENUE_SIMICE) - result << security.ice().security_desc() << separator - << Venue_Name(security.exchange()); - else - result << security.symbol() << separator - << Venue_Name(security.exchange()); - switch (security.Contract_case()) { - case Security::ContractCase::kOption: { - result << separator; - const auto &o = security.option(); - write_date_string( - o.expiration_year(), o.expiration_month(), o.expiration_day(), - result); - result << separator << o.strike() - << (o.call_put() == PUTORCALL_CALL ? 'C' : 'P'); - break; - } - case Security::ContractCase::kFuture: { - result << separator; - const auto &f = security.future(); - write_date_string( - f.expiration_year(), f.expiration_month(), f.expiration_day(), - result); - break; - } - case Security::ContractCase::kSpot: { - break; - } - case Security::ContractCase::kSpread: { - // If the security key of a leg is invalid return an error - const auto &s = security.spread(); - bts_assert(s.legs_size() > 0); - for (const auto &leg : s.legs()) { - if (leg.security().id() == invalid_security_key) - return ""; - write_leg_descriptor(result, separator, leg); - } - break; - } - case Security::ContractCase::kSynthetic: { - const auto &s = security.synthetic(); - bts_assert(s.legs_size() > 0); - result << separator << s.name() - << " pc=" << SyntheticContract::PriceCalc_Name(s.price_calc()); - for (const auto &leg : s.legs()) { - if (leg.security().id() == invalid_security_key) - return ""; // unlikey with synthetics - write_leg_descriptor(result, separator, leg); - } - break; - } - case Security::ContractCase::CONTRACT_NOT_SET: - bts_assert( - security.Contract_case() != Security::ContractCase::CONTRACT_NOT_SET); - } - return result.str(); -} - void get_contract_spec( const FixedSecurity &source, /*out*/ api::ContractSpecification &spec) { spec.set_price_break(source.price_break); @@ -292,26 +217,12 @@ void convert_security(const FixedSecurity &source, Security *presult) { get_contract_spec(source, *dest.mutable_contract_spec()); } -SecurityType securitytype_of_contractcase(Security::ContractCase cc) { - switch (cc) { - case Security::ContractCase::kFuture: - return SECURITYTYPE_FUTURE; - case Security::ContractCase::kOption: - return SECURITYTYPE_OPTION; - case Security::ContractCase::kSpot: - return api::SECURITYTYPE_SPOT; - case Security::ContractCase::kSpread: - return SECURITYTYPE_SPREAD; - case Security::ContractCase::kSynthetic: - return SECURITYTYPE_SYNTHETIC; - case Security::ContractCase::CONTRACT_NOT_SET: - bts_assert(cc != Security::ContractCase::CONTRACT_NOT_SET); - } - bts_assert(false); -} SecurityType securitytype_of(const Security &s) { - return securitytype_of_contractcase(s.Contract_case()); + auto type = bts::securities::security_type(s.Contract_case()); + bts_assert(type != api::SecurityType::SECURITYTYPE_UNSET); + return type; } + size_t num_legs(const Security &sec) { if (sec.Contract_case() == api::Security::ContractCase::kSpread) { return sec.spread().legs_size(); @@ -359,7 +270,7 @@ uint64_t exchange_id(const Security &security) { void convert_security( const Security &src, FixedSecurity &dest, bool set_security_key) { dest.key = set_security_key ? src.id() : invalid_security_key; - dest.type = securitytype_of_contractcase(src.Contract_case()); + dest.type = bts::securities::security_type(src.Contract_case()); dest.symbol(src.symbol()); dest.currency(src.currency()); dest.exchange = src.exchange(); @@ -374,8 +285,8 @@ void convert_security( dest.cme_market_segment_id = src.glbx().market_segment_id(); break; case Security::ExchangeDataCase::kCboe: - dest.numeric_id = src.deribit().exchange_id(); - dest.security_desc(src.deribit().security_desc()); + dest.numeric_id = src.cboe().exchange_id(); + dest.security_desc(src.cboe().security_desc()); dest.cboe_class_key = src.cboe().class_key(); break; case Security::ExchangeDataCase::kTaifex: diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index b26ca753a..4b39e6c51 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -208,8 +208,6 @@ typedef FixedSpreadSecurity<16> FixedSpreadSecurityS; typedef FixedSpreadSecurity<128> FixedSpreadSecurityL; typedef FixedSpreadSecurity<512> FixedSpreadSecurityXL; -std::string security_descriptor(const api::Security &); - // Convert a fixedsecurity to a normal Security - caveat: spreads will be // normalized Security convert_security(const FixedSecurity &); @@ -217,10 +215,9 @@ void convert_security(const FixedSecurity &, Security *); void convert_security( const Security &src, FixedSecurity &dest, bool set_security_key = false); -SecurityType securitytype_of_contractcase(Security::ContractCase cc); -SecurityType securitytype_of(const Security &); -size_t num_legs(const Security &); -uint64_t exchange_id(const Security &security); +SecurityType securitytype_of(const api::Security &); +size_t num_legs(const api::Security &); +uint64_t exchange_id(const api::Security &security); void to_json(nlohmann::json &j, const FixedSpreadLeg &f); diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp index c0301e8f4..6daca8b4c 100644 --- a/src/lib/md/SyntheticSecurityService.cpp +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -35,8 +35,9 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher struct SyntheticSecurityService::Pimpl { Pimpl(const MetadataStatefulPublisher &fn) - : recorder(utils::Dispatcher::background()), synthetics(), on_security(), - meta(), on_metadata(), metadata_publisher(fn) {} + : recorder(nullptr, std::nullopt, &utils::Dispatcher::background()), + synthetics(), on_security(), meta(), on_metadata(), + metadata_publisher(fn) {} SecurityIDRecorder recorder; std::unordered_map synthetics; utils::Event on_security; diff --git a/src/lib/md/cme/mdp3_data_directory.h b/src/lib/md/cme/mdp3_data_directory.h new file mode 100644 index 000000000..ced0f3457 --- /dev/null +++ b/src/lib/md/cme/mdp3_data_directory.h @@ -0,0 +1,32 @@ +#pragma once + +#include "lib/os/spark_data_directory.h" + +#include +#include + +namespace bts::md { + +struct Mdp3DataDirectory { + boost::filesystem::path log_dir() const { return root.log_dir() / "onix"; } + + boost::filesystem::path data_dir() const { return root.root() / "cme_mdp3"; } + + boost::filesystem::path config_file() const { + return data_dir() / "config.xml"; + } + + boost::filesystem::path templates_file() const { + return data_dir() / "templates.xml"; + } + + void create_dirs() { + boost::filesystem::create_directories(data_dir()); + boost::filesystem::create_directories(log_dir()); + } + + private: + spark::SparkDataDirectory root{}; +}; + +} // namespace bts::md diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 13cdbb323..f4cff0acc 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -9,6 +9,7 @@ #include "lib/config/config.h" #include "lib/metrics/metrics.h" #include "lib/os/os.h" +#include "lib/persistence/SqliteStore.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/Shuffle.h" #include "lib/utils/bts_chrono.h" @@ -160,12 +161,10 @@ class Co2Tests { #ifndef _WIN32 void test_security_id_recorder(const std::string &db_name) { // Verify callbacks happen after work complete - auto db1 = sqlite::open_db(db_name.c_str()); - REQUIRE(db1); - auto db2 = sqlite::open_db(db_name.c_str()); - REQUIRE(db2); - - SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}}; + SecurityIDRecorder idsvc{ + std::make_unique(db_name)}; + SecurityIDRecorder idsvc2{ + std::make_unique(db_name)}; security_key result = invalid_security_key; Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850); Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20); @@ -215,18 +214,18 @@ class Co2Tests { std::make_unique(s1, cold_header, hot_header); // Add spread to SecurityIDRecorder and work it - auto db = sqlite::open_db(":memory:"); + auto db = std::make_unique(":memory:"); REQUIRE(db); SecurityIDRecorder idsvc{ - {db, [&](ExchangeKey k) -> SecurityKey { - for (const auto &fut : - {f1handle->producer(), f2handle->producer()}) { - if (k.value() == fut.security().numeric_id) - return SecurityKey(fut.key()); - } - return SecurityKey(invalid_security_key); - }}}; + std::move(db), [&](ExchangeKey k) -> SecurityKey { + for (const auto &fut : + {f1handle->producer(), f2handle->producer()}) { + if (k.value() == fut.security().numeric_id) + return SecurityKey(fut.key()); + } + return SecurityKey(invalid_security_key); + }}; auto workwork = [&]() { idsvc.do_work(); diff --git a/src/lib/orderman/gateways/ExecutionGatewayFactory.cpp b/src/lib/orderman/gateways/ExecutionGatewayFactory.cpp index 28ae3d5ca..9fe30c154 100644 --- a/src/lib/orderman/gateways/ExecutionGatewayFactory.cpp +++ b/src/lib/orderman/gateways/ExecutionGatewayFactory.cpp @@ -4,9 +4,12 @@ #include "lib/orderman/gateways/CCAPIExecutionGateway.h" // #include "lib/orderman/gateways/EMExecutionGateway.h" #include "lib/orderman/gateways/FakeExecutionGateway.h" +#include "lib/orderman/gateways/GeneralExecutionGateway.h" #include "lib/orderman/gateways/GLBXExecutionGateway.h" #include "lib/orderman/gateways/GLBXiLink3ExecutionGateway.h" #include "lib/orderman/gateways/ICEExecutionGateway.h" +#include "lib/orderman/gateways/IFIXGenericAdapter.h" +#include "lib/orderman/gateways/OnixFIXAdapter.h" #include "lib/orderman/gateways/sim/SimExecutionGatewayFactory.h" #include "lib/orderman/gateways/sim/SimExecutionGatewayUniqueWrapper.h" #include "lib/tracing/PerfLog.h" @@ -21,23 +24,30 @@ std::unique_ptr ExecutionGatewayFactory::create_gateway( const config::JetFIX::Connection &conn, trace::PerfLog &pl, md::SecurityOracle o) { + LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway conn: {}", + utils::pb::to_string(conn)); if (SimExecutionGatewayFactory::is_sim_execution_on(conn.venue())) { return std::make_unique(conn.venue()); } - LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway conn: {}", - utils::pb::to_string(conn)); - if (conn.venue() == bts::api::VENUE_ICE || - conn.venue() == bts::api::VENUE_SIMICE) - return std::make_unique(conn, pl, o); - else if (conn.venue() == bts::api::VENUE_GLBX) { - if (conn.use_ilink3()) - return std::make_unique( - conn, pl, o); - return std::make_unique(conn, pl, o); - } else if (conn.venue() == bts::api::VENUE_DERIBIT) - // return std::make_unique(o); - return std::make_unique(conn.venue(), o); - return std::make_unique(o); + switch(conn.venue()){ + case bts::api::VENUE_ICE: + case bts::api::VENUE_SIMICE: + return std::make_unique(conn, pl, o); + case bts::api::VENUE_GLBX:{ + if (conn.use_ilink3()) + return std::make_unique( conn, pl, o); + return std::make_unique(conn, pl, o); + } + case bts::api::VENUE_DERIBIT: + return std::make_unique(conn.venue(), o); + case bts::api::VENUE_CBOE: + { + std::unique_ptr adapter = std::make_unique(o); + return std::make_unique(conn.venue(),std::move(adapter),o); + } + default: + return std::make_unique(o); + } } } // namespace bts::orderman diff --git a/src/lib/orderman/gateways/GeneralExecutionGateway.cpp b/src/lib/orderman/gateways/GeneralExecutionGateway.cpp new file mode 100644 index 000000000..bb8b738ef --- /dev/null +++ b/src/lib/orderman/gateways/GeneralExecutionGateway.cpp @@ -0,0 +1,86 @@ +#include "GeneralExecutionGateway.h" +#include "lib/domain_model/Security.pb.h" +#include "lib/logging/logging.h" +#include "lib/orderman/CreateOrderMsg.h" +#include "lib/orderman/OrderMsg.h" +#include "lib/orderman/gateways/IExecutionGateway.h" +#include +#include + +//Acts as a gateway, On One side is the ordermanager and on the other onixfix listener and onixfix adapter +namespace bts::orderman { +struct GeneralExecutionGateway::PImpl { + PImpl(bts::api::Venue venue, + std::unique_ptr adapter, + md::SecurityOracle o) + : venue(venue), _adapter(std::move(adapter)), sec_oracle(o) {} + bts::api::Venue venue; + std::unique_ptr _adapter; + md::SecurityOracle sec_oracle; + OnExeReport on_exe_report; + OnGwStatusChange on_gw_status_change; + OnCancelReject on_cancel_reject; + OnExchangePositionState on_exchange_position_state; + OnExchangeOpenOrdersState on_exchange_open_orders; +}; +OnGwStatusChange &GeneralExecutionGateway::on_gw_status_change() { + return me->on_gw_status_change; +} +// Modified constructor to accept the adapter from outside +GeneralExecutionGateway::GeneralExecutionGateway(bts::api::Venue venue, + std::unique_ptr adapter, md::SecurityOracle o) + : IExecutionGateway(), me(new PImpl(venue, std::move(adapter),o)) { + LOG(OrderMan, Info, "ExecutionGateway::ExecutionGateway constructor"); +} +GeneralExecutionGateway::~GeneralExecutionGateway() {} +void GeneralExecutionGateway::start() { + me->on_gw_status_change(true); + me->_adapter->start(); + me->_adapter->register_callback(me->on_exe_report); + LOG(OrderMan,Info,"ExecutionGateway::start started" ); +} +void GeneralExecutionGateway::send_msgs( + std::vector const &msgs, trace::CorID) { + LOG(OrderMan,Info,"ExecutionGateway::send_msgs" ); + for (const auto &msg : msgs) { + std::visit( + [this](auto &&order) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + LOG(OrderMan,Warn,"ExecutionGateway::CreateOrderMsg" ); + me->_adapter->launch_new_order(order); + + } else if constexpr (std::is_same_v) { + LOG(OrderMan,Warn,"ExecutionGateway::CancelOrderMsg" ); + me->_adapter->cancel_order(order); + } else if constexpr (std::is_same_v) { + LOG(OrderMan,Warn,"ExecutionGateway::CancelReplaceOrderMsg" ); + me->_adapter->modify_order(order); + } else if constexpr (std::is_same_v) { + // Handle RFQMsg if necessary + // adapter->handle_rfq(order); // Example placeholder + } + }, + msg); + } +} +void GeneralExecutionGateway::dispatch_incoming_fix_messages() {} +void GeneralExecutionGateway::on_securities( + const mercury::Update &) {} +void GeneralExecutionGateway::on_admin_data( + const mercury::Update &) {} +void GeneralExecutionGateway::send_fix(const jetfix::OutFIXPtr &, trace::CorID) {} +OnExeReport &GeneralExecutionGateway::on_exe_report() { return me->on_exe_report; } +OnCancelReject &GeneralExecutionGateway::on_cancel_reject() { + static OnCancelReject default_cancel_reject; + return default_cancel_reject; +} +OnExchangePositionState &GeneralExecutionGateway::on_exchange_position_state() { + static OnExchangePositionState default_exchange_position_state; + return default_exchange_position_state; +} +OnExchangeOpenOrdersState &GeneralExecutionGateway::on_exchange_open_orders() { + static OnExchangeOpenOrdersState default_exchange_open_orders; + return default_exchange_open_orders; +} +} //namespace bts::orderman \ No newline at end of file diff --git a/src/lib/orderman/gateways/GeneralExecutionGateway.h b/src/lib/orderman/gateways/GeneralExecutionGateway.h new file mode 100644 index 000000000..1969b9321 --- /dev/null +++ b/src/lib/orderman/gateways/GeneralExecutionGateway.h @@ -0,0 +1,29 @@ +#pragma once +#include "lib/orderman/gateways/IExecutionGateway.h" +#include "lib/orderman/gateways/IFIXGenericAdapter.h" +namespace bts { +namespace orderman { +class GeneralExecutionGateway : public IExecutionGateway { +public: + GeneralExecutionGateway(bts::api::Venue venue,std::unique_ptr adapter, + md::SecurityOracle o); + virtual ~GeneralExecutionGateway(); + void start() override; + void send_msgs(std::vector const & msgs, trace::CorID) override; + void dispatch_incoming_fix_messages() override; + void on_securities(const mercury::Update &) override; + void on_admin_data(const mercury::Update &) override; + void send_fix(const jetfix::OutFIXPtr &, trace::CorID) override; + OnExeReport &on_exe_report() override; + OnGwStatusChange &on_gw_status_change() override; + OnCancelReject &on_cancel_reject() override; + OnExchangePositionState &on_exchange_position_state() override; + OnExchangeOpenOrdersState &on_exchange_open_orders() override; +private: + struct PImpl; + std::unique_ptr me; + + +}; +} //namespace bts +} //namespace orderman \ No newline at end of file diff --git a/src/lib/orderman/gateways/IFIXGenericAdapter.cpp b/src/lib/orderman/gateways/IFIXGenericAdapter.cpp index 1f3e29caf..b4f9aba5b 100644 --- a/src/lib/orderman/gateways/IFIXGenericAdapter.cpp +++ b/src/lib/orderman/gateways/IFIXGenericAdapter.cpp @@ -1,3 +1,6 @@ #include "IFIXGenericAdapter.h" -namespace bts::orderman {} \ No newline at end of file +namespace bts::orderman { + + IFIXGenericAdapter::~IFIXGenericAdapter(){} +} \ No newline at end of file diff --git a/src/lib/orderman/gateways/IFIXGenericAdapter.h b/src/lib/orderman/gateways/IFIXGenericAdapter.h index 98bfc52d7..74bdb01bb 100644 --- a/src/lib/orderman/gateways/IFIXGenericAdapter.h +++ b/src/lib/orderman/gateways/IFIXGenericAdapter.h @@ -1,22 +1,13 @@ #pragma once -#include "lib/api/services/OrderService.pb.h" -#include "lib/config/protos/jetfix.pb.h" -#include "lib/domain_model/Security.pb.h" -#include "lib/jetfix/codec/MessageDefinitions.h" -#include "lib/jetfix/common/OutFIXPtr.h" -#include "lib/md/SecurityOracle.h" -#include "lib/mercury/events/Update.h" +#include "lib/orderman/CancelOrderMsg.h" +#include "lib/orderman/CancelReplaceOrderMsg.h" #include "lib/orderman/CreateOrderMsg.h" -#include "lib/orderman/OrderMsg.h" -#include "lib/orderman/orderman_utils/ExchangeOpenOrders.h" -#include "lib/orderman/orderman_utils/ExchangePosition.h" -#include "lib/orderman/orderman_utils/orderman_common.h" -#include "lib/utils/bts_no_copy.h" +#include "lib/tracing/PerfLog.h" #include -#include -#include + + namespace bts::orderman { @@ -29,33 +20,28 @@ class NewAdminDataEvent; class ExecutionReport; class CancelReject; -struct OrderStateWithFills { - api::orderman::OrderState ordr_state; - std::vector fills = {}; -}; - -using OnOrderUpdate = std::function; +using OnExecutionReport = std::function; // fix execution gateway for a single FIX session class IFIXGenericAdapter { public: IFIXGenericAdapter() = default; - virtual ~IFIXGenericAdapter() {} + virtual ~IFIXGenericAdapter(); virtual void start() = 0; - // this callback should be added in a list, and on receving an update from the other side, it should be passed here - virtual void register_callback(const OnOrderUpdate &); - virtual void launch_new_order(const CreateOrderMsg &); - virtual void modify_order(const CancelReplaceOrderMsg &); - virtual void cancel_order(const CancelOrderMsg &); +// // this callback should be added in a list, and on receving an update from the other side, it should be passed here + virtual void register_callback(const OnExecutionReport &) =0 ; + virtual void launch_new_order(const CreateOrderMsg &) =0; + virtual void modify_order(const CancelReplaceOrderMsg &) = 0; + virtual void cancel_order(const CancelOrderMsg &) =0; // as new securities are added - virtual void update_securities( - const mercury::Update &) = 0; - virtual void update_admin_data( - const mercury::Update &) = 0; + // virtual void update_securities( + // const mercury::Update &) = 0; + // virtual void update_admin_data( + // const mercury::Update &) = 0; - private: - BTS_NO_COPY(IFIXGenericAdapter); +// private: +// BTS_NO_COPY(IFIXGenericAdapter); }; } // namespace bts::orderman diff --git a/src/lib/orderman/gateways/OnixFIXAdapter.cpp b/src/lib/orderman/gateways/OnixFIXAdapter.cpp new file mode 100644 index 000000000..a5cdfda40 --- /dev/null +++ b/src/lib/orderman/gateways/OnixFIXAdapter.cpp @@ -0,0 +1,377 @@ +#include "OnixFIXAdapter.h" +#include "OnixS/FIXEngine/FIX/Engine.h" +#include "lib/config/protos/marex_fix_cfg.pb.h" +#include "lib/domain_model/Security.pb.h" +#include "lib/logging/logging.h" +#include "lib/orderman/CancelOrderMsg.h" +#include "lib/orderman/CancelReplaceOrderMsg.h" +#include "lib/orderman/CreateOrderMsg.h" +#include "lib/orderman/gateways/OnixFIXListener.h" +#include "lib/orderman/gateways/TrafixOrderStateHandler.h" +#include "lib/orderman/orderman_utils/orderman_common.h" +#include "lib/utils/bts_assert.h" +#include "lib/utils/helper_macros.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bts::orderman { +void handleException( + const std::string &context, const std::string &error) { + bts_assert_msgf(false, "{} Exception: {}", context, error); +} +OnixFIXAdapter::OnixFIXAdapter(md::SecurityOracle sec_oracle) + : state_handler(std::make_shared(sec_oracle)), + listener(state_handler) { + try { + auto onix_cfg = config::get(); + LOG(OrderMan, Warn, "Initializing OnixFIXAdapter"); + OnixS::FIX::Engine:: + init(); // OnixS::FIX::Engine::init() needs to run before initialising session + LOG(OrderMan, Diag, "Initializing OnixFIXAdapter engine initailes trying login"); + session = std::make_unique( + onix_cfg.sender_sub_id(), onix_cfg.target_sub_id(), + OnixS::FIX::ProtocolVersion::FIX_42, &listener); + } catch (const OnixS::FIX::Exception &ex) { + const std::string err = ex.what(); + LOG(OrderMan, Error, "Initializing OnixFIXAdapter ...{}",err); + Engine::shutdown(); + handleException("OnixFIXAdapter::Construcotr init", err); + } +} +OnixFIXAdapter::~OnixFIXAdapter() { + try { + if (session) + session->logout(); + Engine::instance()->unregisterListener(); + Engine::shutdown(); + } catch (const OnixS::FIX::Exception &ex) { + const std::string err = ex.what(); + handleException("OnixFIXAdapter::Destructor ", err); + } +} +void OnixFIXAdapter::start() { + auto cfg = config::get(); + const std::string counterpartyHost = cfg.counterparty_host(); + const int heartBtInt = cfg.heart_bt_int(); + const int counterpartyPort = cfg.counterparty_port(); + const int retryInterval = cfg.retry_interval(); // Add this field in MarexFixCfg + const int maxRetries = cfg.max_retries(); // Optional: Set a limit on retries + + int attempt = 0; + + while (true) { + try { + LOG(OrderMan, Warn, "OnixFIXAdapter::start Attempting login (Attempt: {})", attempt + 1); + session->logonAsInitiator(counterpartyHost, counterpartyPort, heartBtInt); + LOG(OrderMan, Warn, "OnixFIXAdapter::start Login successful"); + break; // Exit loop on success + } catch (const OnixS::FIX::Exception &ex) { + const std::string err = ex.what(); + LOG(OrderMan, Error, "OnixFIXAdapter::start Login failed: {}. Retrying in {} seconds.", err, retryInterval); + + if (maxRetries > 0 && ++attempt >= maxRetries) { + Engine::shutdown(); + LOG(OrderMan, Error, "OnixFIXAdapter::start Exceeded max retries ({}). Aborting.", maxRetries); + handleException("OnixFIXAdapter::start", err); + break; // Exit if retries are exhausted + } + std::this_thread::sleep_for(std::chrono::seconds(std::max(60,retryInterval))); + } + } +} +template +uint translateEnum(T enum_value, const std::map &translationMap) { + auto it = translationMap.find(enum_value); + bts_assert(it != translationMap.end()); // Using assert instead of bts_assert + return it->second; +} + +uint translateTimeInForce(api::orderman::TimeInForce time_in_force) { + //0 = Day 1 = GTC 2 = At the Open 3 = IOC 4 = FOK 5 = GTX + static const std::map tifMap = { + {api::orderman::TimeInForce::TIME_IN_FORCE_DAY, 0}, + {api::orderman::TimeInForce::TIME_IN_FORCE_GTC, 1}, + {api::orderman::TimeInForce::TIME_IN_FORCE_IOC, 3}}; + + return translateEnum(time_in_force, tifMap); +} + +uint translateOrderType(api::orderman::NativeOrderType order_type) { + //1 = Market 2 = Limit 3 = Stop 4 = Stop Limit 5 = Market on close A = On Close B = Limit on Close + static const std::map orderTypeMap = { + {api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_MARKET, 1}, + {api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_LIMIT, 2}, + {api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_STOP, 3}, + {api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_STOP_LIMIT, 4}}; + + return translateEnum(order_type, orderTypeMap); +} +uint translateSide(api::Side side) { + //1 = Buy 2 = Sell 5 = Sell short 6 = Sell short Exempt + static const std::map sideMap = { + {api::Side::SIDE_BUY, 1}, {api::Side::SIDE_SELL, 2}}; + + return translateEnum(side, sideMap); +} + +std::string pad_string( + const std::string &str, size_t length, char padding = ' ') { + if (str.size() > length) { + return str.substr(0, length); + } + return str + std::string(length - str.size(), padding); +} + +// Function to create the OSI Symbol +static std::string create_osi_symbol( + const std::string &symbol, + const uint32_t &year, + const uint32_t &month, + const uint32_t &day, + const char &call_put_indicator, + const double &strike_price) { + if (symbol.size() > 6) { + throw std::invalid_argument("Symbol length cannot exceed 6 characters."); + } + + std::ostringstream osi_stream; + + // Symbol: 6 bytes, padded with spaces + osi_stream << pad_string(symbol, 6); + + // Year: last 2 digits of the year + osi_stream << std::setw(2) << std::setfill('0') << (year % 100); + + // Month: 2 digits + osi_stream << std::setw(2) << std::setfill('0') << month; + + // Day: 2 digits + osi_stream << std::setw(2) << std::setfill('0') << day; + + // Call/Put Indicator: 1 byte + osi_stream << call_put_indicator; + + // Strike Price: 5 bytes for dollars, 3 bytes for decimal part + int strike_dollars = static_cast(strike_price); + int strike_decimals = + static_cast((strike_price - strike_dollars) * 1000); + osi_stream << std::setw(5) << std::setfill('0') << strike_dollars; + osi_stream << std::setw(3) << std::setfill('0') << strike_decimals; + + return osi_stream.str(); +} + +void parse_option(Message &order, const api::Security &sec) { + auto osi = create_osi_symbol( + sec.symbol(), sec.option().expiration_year(), + sec.option().expiration_month(), sec.option().expiration_day(), + sec.option().call_put() == bts::api::PutOrCall::PUTORCALL_CALL ? 'c' + : 'p', + sec.option().strike()); + std::ostringstream month_stream; + month_stream << std::setw(2) << std::setfill('0') + << sec.option().expiration_month(); + std::string month = month_stream.str(); + LOG(OrderMan, Warn, "OnixFIXAdapter::parse_option parsing osi:{}", osi); + order + .set( + OnixS::FIX::FIX42::Tags::PutOrCall, + sec.option().call_put() == bts::api::PutOrCall::PUTORCALL_CALL ? "1" + : "0") + .set( + OnixS::FIX::FIX42::Tags::StrikePrice, + std::to_string(sec.option().strike())) + .set( + OnixS::FIX::FIX42::Tags::MaturityMonthYear, + std::to_string(sec.option().expiration_year()) + month) + .set( + OnixS::FIX::FIX42::Tags::MaturityDay, + std::to_string(sec.option().expiration_day())) + .set(OnixS::FIX::FIX42::Tags::SecurityType, "OPT") + .set(OnixS::FIX::FIX42::Tags::Symbol, sec.symbol()); +} + +void add_cfg_fields(Message &order, const config::MarexFixCfg &cfg) { + order + .set( + OnixS::FIX::FIX42::Tags::HandlInst, + cfg.handl_inst()) // 1: DMA 2: ALGO 3:CARE + .set( + OnixS::FIX::FIX42::Tags::Rule80A, + cfg.Rule80A_Name(cfg.rule80a())) //A = AGENCY P = PRINCIPAL + .set(OnixS::FIX::FIX42::Tags::SenderSubID, cfg.sender_sub_id()) + .set(OnixS::FIX::FIX42::Tags::TargetSubID, cfg.target_sub_id()) + .set(OnixS::FIX::FIX42::Tags::ClientID, cfg.client_id()) + .set(OnixS::FIX::FIX42::Tags::CustomerOrFirm, cfg.customer_or_firm()) + .set(OnixS::FIX::FIX42::Tags::OpenClose, cfg.open_close()); + order.set(FIX42::Tags::Currency,cfg.currency()); + order.set(FIX42::Tags::SecurityExchange,cfg.security_exchange()); + order.set(FIX42::Tags::ExDestination,cfg.ex_destination()); +} +void populate_common_order_fields(Message &order, const CreateOrderMsg &msg) { + order.set(OnixS::FIX::FIX42::Tags::Account, msg.routing_record->account()) + .set( + OnixS::FIX::FIX42::Tags::TimeInForce, + translateTimeInForce(msg.time_in_force)) + .set( + OnixS::FIX::FIX42::Tags::OrdType, translateOrderType(msg.order_type)) + .set(OnixS::FIX::FIX42::Tags::Side, translateSide(msg.side)) + .set( + OnixS::FIX::FIX42::Tags::Price, + static_cast(msg.price.value())) + .set(OnixS::FIX::FIX42::Tags::OrderQty, msg.quantity.value()); + if (msg.security->has_option()) { + parse_option(order, *msg.security); + } else { + order.set(OnixS::FIX::FIX42::Tags::Symbol, msg.security->symbol()) + .set(OnixS::FIX::FIX42::Tags::SecurityType, "FUT") //need to fix this + .set(OnixS::FIX::FIX42::Tags::Currency, msg.security->currency()); + } +} +void OnixFIXAdapter::modify_order( + const CancelReplaceOrderMsg &msg) { //confirm will have everything + try { + + //use this fileOnixS.FixEngineCpp-Ubuntu2204-gcc1210-c++17-x64-4.10.1/OnixS.FixEngineCpp-Ubuntu2204-gcc1210-c++17-x64-4.10.1/doc/html/group__double-decimal-conversion.html + + LOG(OrderMan, Warn, "OnixFIXAdapter::modify_order modifying order{}", + msg); + auto order_cfg = config::get(); + auto id = + state_handler->get_marex_key(OrderKey(msg.create_fields.key.value())); + + if (id == "") { + LOG(OrderMan, Error, + "OnixFIXAdapter::modify_order could not find clorid"); + return; + } + auto new_id = state_handler->new_id(id); + + Message order( + OnixS::FIX::FIX42::Values::MsgType::Order_Cancel_Replace_Request, + OnixS::FIX::ProtocolVersion::FIX_42); + order.set(OnixS::FIX::FIX42::Tags::OrigClOrdID, id) + .set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id) // ID for order + .set( + OnixS::FIX::FIX42::Tags::TransactTime, + OnixS::FIX::Timestamp::utc(), + OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec); + + add_cfg_fields(order, order_cfg); + populate_common_order_fields(order, msg.create_fields); + order.validate(); + LOG(OrderMan, Warn, "order Validated"); + order.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART"); + std::string order_string = order.toString('|'); + state_handler->modifyOrder(msg, new_id); + session->send(&order); + LOG(OrderMan, Warn, "Order update sent :{}", order_string); + } catch (const OnixS::FIX::Exception &ex) { + const std::string err = ex.what(); + handleException("Modify order", err); + } +} + +void OnixFIXAdapter::launch_new_order(const CreateOrderMsg &msg) { + LOG(OrderMan, Warn, "launch order {}", msg); + + try { + LOG(OrderMan, Warn, "OnixFIXAdapter::launch_new_order creating order"); + auto order_cfg = config::get(); + + auto new_id = + state_handler->new_id(":" + std::to_string(msg.key.value())); + LOG(OrderMan, Warn, "OnixFIXAdapter::launch_new_order new id:'{}'", + new_id); + std::string symbol = msg.security->symbol(); + Message order( + OnixS::FIX::FIX42::Values::MsgType::Order_Single, + OnixS::FIX::ProtocolVersion::FIX_42); + order + .set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id) // ID for order + .set( + OnixS::FIX::FIX42::Tags::TransactTime, + OnixS::FIX::Timestamp::utc(), + OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec); + add_cfg_fields(order, order_cfg); + populate_common_order_fields(order, msg); + + std::string order_string = order.toString('|'); + std::string order_json = order.toJson(); + order.validate(); + LOG(OrderMan, Warn, "order Validated"); + order.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART"); + state_handler->launchOrder(msg, new_id); + // listener.fire_dummy_report(msg); + session->send(&order); + LOG(OrderMan, Warn, "Order sent :{} json:{}", order_string, order_json); + + } + // Shut down all reactors if they exist. + catch (const OnixS::FIX::Exception &ex) { + const std::string err = ex.what(); + handleException("Launch order", err); + } +} +void OnixFIXAdapter::register_callback(const OnExecutionReport &cb) { + LOG(OrderMan, Warn, "OnixFIXAdapter::register_callback callback registered"); + listener.setCallback(cb); +} +void OnixFIXAdapter::cancel_order(const CancelOrderMsg &msg) { + LOG(OrderMan, Warn, "cancel order {}", msg); + try { + auto id = state_handler->get_marex_key(OrderKey(msg.key.value())); + auto order_cfg = config::get(); + + if (id == "") { + LOG(OrderMan, Error, + "OnixFIXAdapter::modify_order could not find clorid"); + return; + } + auto new_id = state_handler->new_id(id); + Message order_cancel( + OnixS::FIX::FIX42::Values::MsgType::Order_Cancel_Request, + OnixS::FIX::ProtocolVersion::FIX_42); + order_cancel.set( + OnixS::FIX::FIX42::Tags::HandlInst, order_cfg.handl_inst()); + order_cancel.set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id); + order_cancel.set(OnixS::FIX::FIX42::Tags::OrigClOrdID, id) + .set(OnixS::FIX::FIX42::Tags::Side, translateSide(msg.side)) + .set( + OnixS::FIX::FIX42::Tags::TransactTime, + OnixS::FIX::Timestamp::utc(), + OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec); + if (msg.security->has_option()) { + parse_option(order_cancel, *msg.security); + } else { + order_cancel + .set(OnixS::FIX::FIX42::Tags::Symbol, msg.security->symbol()) + .set(OnixS::FIX::FIX42::Tags::Currency, msg.security->currency()); + } + std::string order_str = order_cancel.toString('|'); + LOG(OrderMan, Warn, "order_cancel Created: {}", order_str); + // order_cancel.validate(); fails becasuse of HandlInst + LOG(OrderMan, Warn, "order Validated"); + order_cancel.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART"); + state_handler->cancelOrder(msg, new_id); + session->send(&order_cancel); + LOG(OrderMan, Warn, "order cancel sent"); + + } + // Shut down all reactors if they exist. + catch (const OnixS::FIX::Exception &ex) { + const std::string err = ex.what(); + handleException("Cacnel order", err); + } +} +} // namespace bts::orderman \ No newline at end of file diff --git a/src/lib/orderman/gateways/OnixFIXAdapter.h b/src/lib/orderman/gateways/OnixFIXAdapter.h new file mode 100644 index 000000000..06e1a234f --- /dev/null +++ b/src/lib/orderman/gateways/OnixFIXAdapter.h @@ -0,0 +1,27 @@ +#pragma once + +#include "IFIXGenericAdapter.h" +#include "lib/orderman/gateways/OnixFIXListener.h" +#include "lib/orderman/gateways/TrafixOrderStateHandler.h" +#include + +namespace bts::orderman { + +class OnixFIXAdapter : public IFIXGenericAdapter { + static constexpr int FIX_TAG_SMART_ROUTING = 847; + public: + OnixFIXAdapter(md::SecurityOracle sec_oracle); + ~OnixFIXAdapter() override; + void register_callback(const OnExecutionReport &report) override; + void launch_new_order(const CreateOrderMsg &msg) override; + void cancel_order(const CancelOrderMsg &msg) override; + void modify_order(const CancelReplaceOrderMsg &) override; + void start() override; + + private: + std::shared_ptr state_handler; + OnixFixListener listener; + std::unique_ptr session; +}; + +} // namespace bts::orderman \ No newline at end of file diff --git a/src/lib/orderman/gateways/OnixFIXListener.cpp b/src/lib/orderman/gateways/OnixFIXListener.cpp new file mode 100644 index 000000000..ea47c956a --- /dev/null +++ b/src/lib/orderman/gateways/OnixFIXListener.cpp @@ -0,0 +1,264 @@ +#include "OnixFIXListener.h" + +#include "lib/api/services/OrderService.pb.h" +#include "lib/domain_model/Security.pb.h" +#include "lib/domain_model/common.h" +#include "lib/logging/logging.h" +#include "lib/orderman/gateways/IExecutionGateway.h" +#include "lib/orderman/gateways/IFIXGenericAdapter.h" +#include "lib/orderman/orderman_utils/ExecutionReport.h" +#include "lib/orderman/orderman_utils/orderman_common.h" +#include "lib/utils/Uuid.h" +#include "lib/utils/bts_assert.h" +#include +#include +#include + +namespace bts::orderman { +ExecutionReport OnixFixListener::fire_dummy_report(CreateOrderMsg msg) { + ExecutionReport *er_p = exe_report_pool.construct(); + ExecutionReport &er = *er_p; + std::string account_name = "BTSIM"; + std::array an_array = {}; + size_t i = 0; + for (const char c : account_name) { + an_array[i++] = c; + } + er.account_name = AccountName(an_array); + to_array(msg.routing_record->account(), &er.account_name.value()); + to_array( + msg.routing_record->operator_id(), &er.operator_id.value()); + er.order_key = msg.key; + er.msg_seq_num = 0; + er.security = msg.security; + er.leg_security = nullptr; + er.side = msg.side; + er.time_in_force = msg.time_in_force; + er.order_type = msg.order_type; + er.exec_type = api::orderman::EXEC_TYPE_NEW; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_NEW; + er.manual_order_indicator = api::orderman::MANUAL_ORDER_INDICATOR_NO; + auto s = std::to_string(msg.key.value()); + std::array idAsArray{}; + std::copy_n(s.begin(), std::min(s.size()-1, idAsArray.size() - 1), idAsArray.begin()); + er.exchange_order_id = ExchangeOrderID{idAsArray}; + er.price = msg.price; + er.quantity = msg.quantity; + er.filled_quantity = Quantity(0); + er.last_shares = Quantity(0); + er.msg = "Message Placed"; + er.exchange = msg.security->exchange(); + // Set transaction time + ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now()); + er.transact_time = now; + cb(er,{}); + return er; +} + +namespace { + ExecutionReport createDefaultExecutionReport() { + ExecutionReport er; + er.exchange = api::VENUE_CBOE; + std::string account_name = "BTSIM"; + std::string ops_id = "op_id_bts"; + + std::array account_array = {}; + std::copy_n(account_name.begin(), std::min(account_name.size(), account_array.size()), account_array.begin()); + er.account_name = AccountName(account_array); + + to_array(ops_id, &er.operator_id.value()); + return er; + } + + void populateExecTypeAndOrderStatus(ExecutionReport &er, int exec_type_value) { + switch (exec_type_value) { + case 0: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_NEW; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_NEW; + break; + case 1: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_PARTIAL_FILL; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_PARTIALLY_FILLED; + break; + case 2: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_FILL; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_FILLED; + break; + case 4: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_CANCELED; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_CANCELED; + break; + case 5: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_REPLACED; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_REPLACED; + break; + case 8: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_REJECTED; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED; + break; + default: + er.exec_type = api::orderman::ExecType::EXEC_TYPE_UNDEFINED; + er.order_status = api::orderman::NATIVE_ORDER_STATUS_UNDEFINED; + break; + } + } + + void populateCreateOrderFields(ExecutionReport &er, const CreateOrderMsg &create_order_msg) { + er.security = create_order_msg.security; + er.order_type = create_order_msg.order_type; + er.time_in_force = create_order_msg.time_in_force; + er.manual_order_indicator = create_order_msg.manual_order_indicator; + + to_array(create_order_msg.routing_record->account(), &er.account_name.value()); + to_array(create_order_msg.routing_record->operator_id(), &er.operator_id.value()); + er.exchange = create_order_msg.security->exchange(); + } + + std::array stringToFixedArray(const std::string &s) { + std::array idAsArray{}; + std::copy_n(s.begin(), std::min(s.size(), idAsArray.size() - 1), idAsArray.begin()); + return idAsArray; + } +} +ExecutionReport OnixFixListener::get_exe_report(Message msg) { + ExecutionReport *er_p = exe_report_pool.construct(); + ExecutionReport &er = *er_p; + + auto msg_str = msg.toString(); + LOG(OrderMan,Info, "OnixFixListener::get_exe_report {}", msg_str); + + // Extract order key and create order message + er.order_key = state_handler->getOrderKey(msg.get(FIX42::Tags::ClOrdID).toString()); //OrderKey(extractUInt(msg.get(FIX42::Tags::ClOrdID).toString())); + auto m = state_handler->getCreateOrderMsg(er.order_key); + + if(m.has_value()){ + populateCreateOrderFields(er, m.value()); + }else{ //handle this later + LOG(OrderMan,Error, "OnixFixListener::get_exe_report {} could not find create order message", msg_str); + er = createDefaultExecutionReport(); + } + // Populate fields from the message + + int tmp; + msg.get(FIX42::Tags::MsgSeqNum).toNumber(tmp); + er.msg_seq_num = tmp; + + er.leg_security = nullptr; + + er.side = msg.get(FIX42::Tags::Side).toString()=="1"?api::Side::SIDE_BUY:api::Side::SIDE_SELL; + + int exec_type_value = 9; // Default to undefined + msg.get(FIX42::Tags::ExecType).toNumber(exec_type_value); + populateExecTypeAndOrderStatus(er, exec_type_value); + + auto price_str = msg.get(FIX42::Tags::Price); + er.price = Price(price_str.valid() ? std::stod(price_str.toString()) : 0); + + auto order_qty_str = msg.get(FIX42::Tags::OrderQty); + er.quantity = Quantity(order_qty_str.valid() ? std::stoi(order_qty_str.toString()) : 0); + + auto leaves_qty_str = msg.get(FIX42::Tags::LeavesQty); + er.filled_quantity = leaves_qty_str.valid() ? er.quantity - Quantity(std::stoi(leaves_qty_str.toString())) : er.quantity; + + auto last_shares_str = msg.get(FIX42::Tags::LastShares); + er.last_shares = Quantity(last_shares_str.valid() ? std::stoi(last_shares_str.toString()) : 0); + + auto avg_px_str = msg.get(FIX42::Tags::AvgPx); + er.avg_price = Price(avg_px_str.valid() ? std::stod(avg_px_str.toString()) : 0); + + auto text_str = msg.get(FIX42::Tags::Text).toString(); + er.msg = text_str.substr(0, std::min(text_str.length(), size_t(15))); + + auto exec_id_str = msg.get(FIX42::Tags::ExecID); + std::string fill_id = exec_id_str.valid() ? exec_id_str.toString() : bts::utils::Uuid().to_text_string(); + to_array(fill_id, &er.exchange_fill_id.value()); + + auto last_px_str = msg.get(FIX42::Tags::LastPx); + er.last_price = Price(last_px_str.valid() ? std::stod(last_px_str.toString()) : 0); + + + er.exchange_order_id = ExchangeOrderID{stringToFixedArray(msg.get(FIX42::Tags::OrderID).toString())}; + + + // er.transact_time = ExchangeTimeStamp64(datetime::unix_nano_now()); + //exchgane gives this "TransactTime": "20241111-16:36:08.243", + // er.transact_time = + // Set transaction time + ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now()); + er.transact_time = now; + + return er; +} + +void OnixFixListener::onInboundApplicationMsg(Message &msg, Session * /*sn*/) { + std::string msg_str = msg.toString(); + std::string msg_json = msg.toJson(); + + FieldValueRef t = msg.type(); + char message_type; + if(!t.toChar(message_type)){ + LOG(OrderMan, Warn, + "no message type message: {} As json {}", + msg_str, msg_json); + return; + } + LOG(OrderMan, Warn, + "Incoming application-level message: {} As json {}, type: {}", + msg_str, msg_json,message_type); + + switch (message_type) { + case '8': + { + //exection report + auto er = get_exe_report(msg); + if(er.security==nullptr){ + LOG(OrderMan, Error,"DID not have sec"); + return; + } + LOG(OrderMan, Error,"msg {}",er.msg); + cb(er,{}); + } + } + + // LOG(OrderMan, Warn, + // "onInboundApplicationMsg::inbound firing er: {}", + // er->to_string()); + + // cb(*er, {}); +} + + +void OnixFixListener::onInboundSessionMsg(Message & msg, Session * /*sn*/) +{ + std::string msg_str = msg.toString(); + std::string msg_json = msg.toJson(); + LOG( OrderMan, Warn, + "Incoming session-level message:\n {} As json {}", + msg_str, + msg_json); +} + +void OnixFixListener::onStateChange(SessionState::Enum newState, SessionState::Enum prevState, + Session * /*sn*/) +{ + std::string prevst = SessionState::toString(prevState); + std::string newst = SessionState::toString(newState); + LOG(OrderMan, Warn, + "Session's state is changed, prevState = {} , newState= {}", + prevst, + newst); +} + +void OnixFixListener::onError(ErrorReason::Enum /*reason*/, const std::string & description, Session * /*sn*/) +{ + LOG( OrderMan, Error, "Session-level error: {}", description); +} + +void OnixFixListener::onWarning(WarningReason::Enum /*reason*/, const std::string & description, + Session * /*sn*/) +{ + LOG( OrderMan, Error, "Session-level warning: {}", description); +} + + +} // namespace bts::orderman \ No newline at end of file diff --git a/src/lib/orderman/gateways/OnixFIXListener.h b/src/lib/orderman/gateways/OnixFIXListener.h new file mode 100644 index 000000000..cc0f8755e --- /dev/null +++ b/src/lib/orderman/gateways/OnixFIXListener.h @@ -0,0 +1,52 @@ +#pragma once + +#include "lib/orderman/CreateOrderMsg.h" +#include "lib/orderman/gateways/IExecutionGateway.h" +#include "lib/orderman/gateways/TrafixOrderStateHandler.h" +#include "lib/orderman/orderman_utils/ExecutionReport.h" +#include +#include +#include + +namespace bts::orderman { +class TrafixOrderStateHandler; +using namespace OnixS::FIX; +class OnixFixListener : public OnixS::FIX::ISessionListener { + public: + OnixFixListener( std::shared_ptr state_handler) : state_handler(state_handler),cb(dummy) { + LOG(OrderMan, Info, "OnixFixListener::cb set callback registered"); + } + std::shared_ptr state_handler; + unsigned long exchange_oid = 0; + /// Is called when the application-level message is received from the counterparty. + void onInboundApplicationMsg(Message &msg, Session *sn) + ONIXS_FIXENGINE_FINAL; + // Set the real callback function + void setCallback(const OnExeReport &callback) { + LOG(OrderMan, Info, "OnixFixListener::setCallback callback registered"); + cb = callback; + } + /// Is called when the session-level message is received from the counterparty. + void onInboundSessionMsg(Message &msg, Session *sn) ONIXS_FIXENGINE_FINAL; + void onStateChange( + SessionState::Enum newState, + SessionState::Enum prevState, + Session *sn) ONIXS_FIXENGINE_FINAL; + OnExeReport cb; + ExecutionReport fire_dummy_report(CreateOrderMsg msg); + ExecutionReport get_exe_report(Message msg); + static void dummy(const ExecutionReport &, const trace::CorID &) { + LOG(OrderMan, Error, "OnixFixListener::Dummy fired"); + } + void onError( + ErrorReason::Enum reason, + const std::string &description, + Session *sn) ONIXS_FIXENGINE_FINAL; + void onWarning( + WarningReason::Enum reason, + const std::string &description, + Session *sn) ONIXS_FIXENGINE_FINAL; + boost::object_pool exe_report_pool; +}; + +} // namespace bts::orderman \ No newline at end of file diff --git a/src/lib/orderman/gateways/TrafixOrderStateHandler.cpp b/src/lib/orderman/gateways/TrafixOrderStateHandler.cpp new file mode 100644 index 000000000..0bb702e38 --- /dev/null +++ b/src/lib/orderman/gateways/TrafixOrderStateHandler.cpp @@ -0,0 +1,88 @@ +#include "TrafixOrderStateHandler.h" +#include "lib/logging/logging.h" +#include "lib/orderman/CreateOrderMsg.h" +#include "lib/orderman/order_factory/IOrderFactory.h" +#include "lib/orderman/orderman_utils/orderman_common.h" +#include +#include +#include +#include + +namespace bts::orderman { + +TrafixOrderStateHandler::TrafixOrderStateHandler(md::SecurityOracle sec_oracle) + : pimpl_(std::make_unique(sec_oracle)) {} + +TrafixOrderStateHandler::~TrafixOrderStateHandler() = default; + +TrafixOrderStateHandler::OrderInfo* TrafixOrderStateHandler::get_order_info(const OrderKey&key) const { + auto it = pimpl_->orders.find(key); + if (it != pimpl_->orders.end()) { + return &it->second; + } + return nullptr; +} + + +OrderKey TrafixOrderStateHandler::launchOrder(const CreateOrderMsg& msg, const std::string& clor_id) { + std::lock_guard lock(pimpl_->orders_mutex); + pimpl_->orders.emplace(msg.key, OrderInfo(msg, clor_id)); + return msg.key; +} +OrderKey TrafixOrderStateHandler::modifyOrder(const CancelReplaceOrderMsg& msg, const std::string& clor_id) { + std::lock_guard lock(pimpl_->orders_mutex); + LOG(md,Info,"Modifying Order: {}, id:{}",msg,clor_id); + auto v = get_order_info(msg.create_fields.key); + if(v==nullptr) return null_order_key(); + LOG(md,Info,"Modifying found order info Order: {}, id:{}",msg,clor_id); + v->update(msg, clor_id); + return msg.create_fields.key; +} +OrderKey TrafixOrderStateHandler::cancelOrder(const CancelOrderMsg& msg, const std::string& clor_id) { + std::lock_guard lock(pimpl_->orders_mutex); + LOG(md,Info,"canceling Order: {}, id:{}",msg,clor_id); + auto v = get_order_info(msg.key); + if(v==nullptr) return null_order_key(); + std::string cancel_id = clor_id; + v->cancel(msg, cancel_id); + return msg.key; +} +//static +std::string TrafixOrderStateHandler::new_id(const std::string& current_id) { + LOG(OrderMan, Warn, "new_id {}", current_id); + int update_number = getUpdateNumber(current_id)+1; + LOG(OrderMan, Warn, "new_id {}, update number {}", current_id,update_number); + + return std::to_string(update_number) + ":" + std::to_string(getOrderKey(current_id).value()); +} +OrderKey TrafixOrderStateHandler::getOrderKey(const std::string& id) { + std::size_t pos = id.find(':'); + if (pos == std::string::npos) return null_order_key(); + return OrderKey(std::stoi(id.substr(pos + 1))); +} +int TrafixOrderStateHandler::getUpdateNumber(const std::string& id) { + std::size_t pos = id.find(':'); + if(pos==std::string::npos || pos ==0){ + return 0; + } + std::string prefix = id.substr(0, pos); + bts_assert_msgf(std::all_of(prefix.begin(), prefix.end(), ::isdigit), "Failed to get update number id:{}",id); + return std::stoi(prefix); +} + +std::optional TrafixOrderStateHandler::getCreateOrderMsg(const OrderKey& key) const { + std::lock_guard lock(pimpl_->orders_mutex); + auto v = get_order_info(key); + if(v!=nullptr) return v->order_msg; + LOG(md, Error, "Order with key {} not found.", key); + return std::nullopt; +} +std::string TrafixOrderStateHandler::get_marex_key(const OrderKey &key) { + std::lock_guard lock(pimpl_->orders_mutex); + + auto v = get_order_info(key); + if(v!=nullptr) return v->current_id; + return ""; +} + +} // namespace b diff --git a/src/lib/orderman/gateways/TrafixOrderStateHandler.h b/src/lib/orderman/gateways/TrafixOrderStateHandler.h new file mode 100644 index 000000000..ea058065e --- /dev/null +++ b/src/lib/orderman/gateways/TrafixOrderStateHandler.h @@ -0,0 +1,61 @@ +#pragma once +#include "IFIXGenericAdapter.h" +#include "lib/orderman/CancelOrderMsg.h" +#include "lib/orderman/CreateOrderMsg.h" +#include "lib/orderman/gateways/OnixFIXListener.h" +#include "lib/orderman/order_factory/IOrderFactory.h" +#include "lib/orderman/orderman_utils/orderman_common.h" +#include "lib/utils/bts_assert.h" +#include +#include +#include +#include + +namespace bts::orderman { + +class TrafixOrderStateHandler { +public: + explicit TrafixOrderStateHandler(md::SecurityOracle sec_oracle); + ~TrafixOrderStateHandler(); + + OrderKey launchOrder(const CreateOrderMsg& msg, const std::string& clor_id); + std::optional getCreateOrderMsg(const OrderKey& key) const; + std::string get_marex_key(const OrderKey &key) ; + OrderKey modifyOrder(const CancelReplaceOrderMsg& msg, const std::string& clor_id); + OrderKey cancelOrder(const CancelOrderMsg& msg, const std::string& clor_id); + static std::string new_id(const std::string& current_id); + static OrderKey getOrderKey(const std::string& id); + static int getUpdateNumber(const std::string& id); + +private: + struct OrderInfo { + explicit OrderInfo(const CreateOrderMsg& msg, const std::string& id) + : current_id(id), order_msg(msg) {} + void update(const CancelReplaceOrderMsg& msg, const std::string& new_id) { + LOG(md,Info,"OrderInfo:update Order: {}, id:{}",msg,new_id); + + upd_msg = msg; + current_id = new_id; + } + + void cancel(const CancelOrderMsg& msg, const std::string& new_id) { + cancel_msg = msg; + current_id = new_id; + } + std::string current_id; + CreateOrderMsg order_msg; + CancelReplaceOrderMsg upd_msg; + CancelOrderMsg cancel_msg; + }; + OrderInfo* get_order_info(const OrderKey&key) const; + struct PImpl { + explicit PImpl(md::SecurityOracle sec_oracle) : sec_oracle(sec_oracle) {} + mutable std::mutex orders_mutex;//need to use this every time orders are accesed + //can be improved later on + std::unordered_map orders; + md::SecurityOracle sec_oracle; + }; + + std::unique_ptr pimpl_; +}; +} \ No newline at end of file diff --git a/src/lib/orderman/order_factory/OrderFactory.cpp b/src/lib/orderman/order_factory/OrderFactory.cpp index 370299c5b..0812a2d5f 100644 --- a/src/lib/orderman/order_factory/OrderFactory.cpp +++ b/src/lib/orderman/order_factory/OrderFactory.cpp @@ -129,7 +129,7 @@ NativeOrderBase *OrderFactory::make_order(const api::Venue exchange) { return new ICENativeOrder(); else if (exchange == api::VENUE_GLBX || exchange == api::VENUE_SIM) return new GLBXNativeOrder(); - return new EMNativeOrder(); + return new EMNativeOrder(exchange); } NativeOrderBase *OrderFactory::make_native_order( diff --git a/src/lib/orderman/orders/EMNativeOrder.cpp b/src/lib/orderman/orders/EMNativeOrder.cpp index 9413b58f0..af01a0d29 100644 --- a/src/lib/orderman/orders/EMNativeOrder.cpp +++ b/src/lib/orderman/orders/EMNativeOrder.cpp @@ -5,6 +5,7 @@ #include "lib/config/protos/toggles.pb.h" #include "lib/domain/OrderStatus.h" #include "lib/domain_model/Security.h" +#include "lib/domain_model/Security.pb.h" #include "lib/domain_model/common.h" #include "lib/jetfix/codec/FIXFragment.h" #include "lib/jetfix/codec/FieldDefinitions.h" @@ -52,10 +53,10 @@ struct InflightUpdateState { }; struct EMNativeOrder::PImpl { - PImpl() + PImpl(const api::Venue exchange) : instance_id(EMNativeOrder::count++), services(nullptr), session_services(), sec(nullptr), previous_working_position(0), - definition(), state(), exchange_order_id() {} + definition(), state(), exchange_order_id(), exchange(exchange) {} const int instance_id; bool inited = false; OrderServices *services; @@ -67,7 +68,7 @@ struct EMNativeOrder::PImpl { OrderDefinition definition; OrderState state; std::optional exchange_order_id; - + const api::Venue exchange; OrderServices::ExeReportSubID exe_report_sid; OrderServices::CancelRejectSubID cancel_reject_sid; // If a cancel is sent before the order placement is ack'ed this is set to @@ -82,7 +83,7 @@ struct EMNativeOrder::PImpl { RoutingRecord routing_record; }; -EMNativeOrder::EMNativeOrder() : me(new PImpl()) { +EMNativeOrder::EMNativeOrder(const api::Venue exchange) : me(new PImpl(exchange)) { LOG(OrderMan, Info, "EMNativeOrder::EMNativeOrder instance_id: {} t_id: {} t_name: {}", me->instance_id, os::get_thread_id(), os::get_application_name()); @@ -342,6 +343,10 @@ OrderOpResult EMNativeOrder::on_cancel() { case ORDER_STATUS_WORKING: { // TODO, CancelOrderMsg is not completely filled CancelOrderMsg cxl_msg = {}; + cxl_msg.side = me->definition.side(); + cxl_msg.security = me->sec; + // cxl_msg.exchange_order_id=me->exchange_order_id;//handle this + cxl_msg.routing_record = &me->routing_record; cxl_msg.key = key(); LOG(OrderMan, Info, "EMNativeOrder queue_order_msg for cxl"); me->services->queue_order_msg(*me->session_services, cxl_msg); @@ -501,11 +506,22 @@ OrderOpResult EMNativeOrder::_on_activate( me->state.set_filled_quantity(0); me->state.set_last_price(0); me->state.set_avg_price(0); - me->session_services = &me->services->get_session_services( get_native_state(me->state).routing_record().session()); // TODO: just to get going - me->session_services = &me->services->get_session_services("deribit_1"); + std::string session_name; + switch (me->exchange) { + case api::VENUE_CBOE: + session_name = "cboe_1"; + break; + case api::VENUE_TAIFEX: + session_name="taifex_1"; + break; + default: + session_name ="deribit_1"; + + } + me->session_services = &me->services->get_session_services(session_name); // Checks for the following: // 1. Gateway connection diff --git a/src/lib/orderman/orders/EMNativeOrder.h b/src/lib/orderman/orders/EMNativeOrder.h index 4564fab95..55fb3b1ae 100644 --- a/src/lib/orderman/orders/EMNativeOrder.h +++ b/src/lib/orderman/orders/EMNativeOrder.h @@ -1,6 +1,7 @@ #pragma once #include "lib/api/services/OrderService.pb.h" +#include "lib/domain_model/Security.pb.h" #include "lib/domain_model/common.h" #include "lib/orderman/CreateOrderMsg.h" #include "lib/orderman/orders/NativeOrderBase.h" @@ -19,7 +20,7 @@ class SessionServices; // Wrapper around Emumba's order class EMNativeOrder : public NativeOrderBase { public: - EMNativeOrder(); + EMNativeOrder(const api::Venue exchange = api::VENUE_DERIBIT); ~EMNativeOrder() override; api::orderman::OrderDefinition const &definition() const override; void on_init_new( diff --git a/src/lib/orderman/orders/GLBXNativeOrder.cpp b/src/lib/orderman/orders/GLBXNativeOrder.cpp index da0537a20..e350c1190 100644 --- a/src/lib/orderman/orders/GLBXNativeOrder.cpp +++ b/src/lib/orderman/orders/GLBXNativeOrder.cpp @@ -478,7 +478,7 @@ OrderOpResult GLBXNativeOrder::on_activate( now.to_uint64(), (now - corid.app_arrival).nanoseconds()); // in sim mode, we dont need this, me->encoder->send_create(*me->services, *me->session_services); - if (me->in_sim_mode) { + if (me->in_sim_mode) { //CME EXECUTION NEEDS TO BE UPDATED CreateOrderMsg c_msg = {}; _create_create_msg(c_msg); me->services->queue_order_msg(*me->session_services, c_msg); @@ -579,7 +579,7 @@ OrderOpResult GLBXNativeOrder::on_update(api::orderman::Update const &update) { auto p = Price(update.price()); auto q = Quantity(update.quantity()); auto send_update_in_sim_mode = [&]() { - if (me->in_sim_mode) { + if (me->in_sim_mode) { //CME EXECUTION NEEDS TO BE UPDATED CancelReplaceOrderMsg cxl_repl_msg = {}; _create_create_msg(cxl_repl_msg.create_fields); Price new_p = Price(update.price()); @@ -799,7 +799,7 @@ OrderOpResult GLBXNativeOrder::on_cancel() { me->encoder->update_cancel_encoding(*me->services); me->encoder->send_cancel(*me->services, *me->session_services); // TODO: check for sim mode here - if (me->in_sim_mode) { + if (me->in_sim_mode) { //CME EXECUTION NEEDS TO BE UPDATED CancelOrderMsg cxl_msg = {}; cxl_msg.key = key(); LOG(OrderMan, Info, "GLBXNativeOrder queue_order_msg for cxl"); @@ -1381,7 +1381,7 @@ void GLBXNativeOrder::_create_create_msg(CreateOrderMsg &c) { bts::orderman::get_quantity(me->state, StateQuantityType::OrderQuantity); c.security = me->sec; c.side = me->definition.side(); - c.order_type = get_glbx_native_ord_type(me->definition); + c.order_type = get_glbx_native_ord_type(me->definition); //CME EXECUTION NEEDS TO BE UPDATED c.time_in_force = me->definition.time_in_force(); c.order_type = api::orderman::NATIVE_ORDER_TYPE_LIMIT; c.manual_order_indicator = me->definition.manual_order_indicator(); diff --git a/src/lib/orderman/orders/glbx/iLink2OrderEncoder.cpp b/src/lib/orderman/orders/glbx/iLink2OrderEncoder.cpp index 7bdbd303d..6e4aa74cb 100644 --- a/src/lib/orderman/orders/glbx/iLink2OrderEncoder.cpp +++ b/src/lib/orderman/orders/glbx/iLink2OrderEncoder.cpp @@ -277,6 +277,7 @@ void iLink2OrderEncoder::update_exchange_order_id(ExchangeOrderIDVar const &v) { auto oid = to_trim_str(std::get(v)); me->update_encoding.add(oid); me->cancel_encoding.add(oid); + //CME EXECUTION NEEDS TO BE UPDATED, can be commented out } void iLink2OrderEncoder::update_create_order_encoding( diff --git a/src/lib/os/spark_data_directory.cpp b/src/lib/os/spark_data_directory.cpp new file mode 100644 index 000000000..edd1ced61 --- /dev/null +++ b/src/lib/os/spark_data_directory.cpp @@ -0,0 +1,46 @@ +#include "spark_data_directory.h" +#include "lib/config/config.h" +#include "lib/config/protos/system_config.pb.h" +#include "lib/os/os.h" + +#include +#include +#include + +namespace bts::spark { + +SparkDataDirectory::SparkDataDirectory() + : SparkDataDirectory(config::get().data_path()) {} + +SparkDataDirectory::SparkDataDirectory(const std::string &root) : root_(root) {} + +boost::filesystem::path SparkDataDirectory::root() const { return root_; } + +boost::filesystem::path SparkDataDirectory::log_dir() const { + return root_ / fmt::format("logging.{}/", os::get_application_name()); +} + +boost::filesystem::path SparkDataDirectory::log_file() const { + return log_dir() / "log"; +} + +boost::filesystem::path SparkDataDirectory::log_cache_file() const { + return log_dir() / "log_cache"; +} + +boost::filesystem::path SparkDataDirectory::database_dir( + const std::string &name) const { + return root_ / "databases" / (name + "/"); +} + +boost::filesystem::path SparkDataDirectory::database_file( + const std::string &name) const { + return root_ / "databases" / name / (name + ".sqlite"); +} + +boost::filesystem::path SparkDataDirectory::database_file( + const std::string &dir_name, const std::string &db_name) const { + return root_ / "databases" / dir_name / (db_name + ".sqlite"); +} + +} // namespace bts::spark diff --git a/src/lib/os/spark_data_directory.h b/src/lib/os/spark_data_directory.h new file mode 100644 index 000000000..e33248f85 --- /dev/null +++ b/src/lib/os/spark_data_directory.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +namespace bts::spark { + +class SparkDataDirectory { + public: + SparkDataDirectory(); + + explicit SparkDataDirectory(const std::string &root); + + boost::filesystem::path root() const; + boost::filesystem::path log_dir() const; + boost::filesystem::path log_file() const; + boost::filesystem::path log_cache_file() const; + + boost::filesystem::path database_dir(const std::string &name) const; + boost::filesystem::path database_file(const std::string &name) const; + boost::filesystem::path database_file( + const std::string &dir_name, const std::string &db_name) const; + + private: + boost::filesystem::path root_; +}; + +} // namespace bts::spark diff --git a/src/lib/persistence/IDGenerator.cpp b/src/lib/persistence/IDGenerator.cpp index 569181eef..7cf38724e 100644 --- a/src/lib/persistence/IDGenerator.cpp +++ b/src/lib/persistence/IDGenerator.cpp @@ -9,6 +9,8 @@ namespace bts::persistence { +using bts::api::domain::SecurityType; +using bts::api::domain::SecurityType_Name; using bts::api::domain::Venue; using bts::api::domain::Venue_Name; using std::string; @@ -52,24 +54,27 @@ uint64_t get_next_id( } std::optional IDGenerator::get_id( - const Venue &venue, const string &name, bool add_if_new) { - return get_id(Venue_Name(venue), name, add_if_new); + const Venue &venue, + SecurityType type, + const string &name, + bool add_if_new) { + std::string ns = fmt::format( + "{}.{}", Venue_Name(venue).substr(6), + SecurityType_Name(type).substr(14)); + return get_id(ns, name, add_if_new); } std::optional IDGenerator::get_id( const std::string &ns, const std::string &key, bool add_if_new) { std::lock_guard myLock(_instance_mx); - auto [exists, val] = me->kv_store->retrieve_one(ns, key); + auto [exists, rowid, val] = me->kv_store->retrieve_one(ns, key); if (exists) { - uint64_t rv = str_to_id(val); - LOG(Gen, Diag, "IDGenerator::get_id ns: {} key:{} id: {}", ns, key, rv); - return std::optional(rv); - + LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rowid); + return std::optional(rowid); } else { if (add_if_new) { auto all = me->kv_store->retrieve_all(); - uint64_t new_id = get_next_id(all); - me->kv_store->insert(ns, key, std::to_string(new_id)); + auto new_id = me->kv_store->insert(ns, key, ""); LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, new_id); return std::optional(new_id); @@ -80,15 +85,11 @@ std::optional IDGenerator::get_id( } } -std::uint64_t IDGenerator::get_or_add(const Venue &venue, const string &name) { - return get_or_add(Venue_Name(venue), name); -} - std::uint64_t IDGenerator::get_or_add( - const std::string &ns, const std::string &key) { - auto id = get_id(ns, key, true); - bts_assert(id.has_value()); - return *id; + api::domain::Venue venue, + api::domain::SecurityType type, + const std::string &name) { + return *get_id(venue, type, name, true); } IDGenerator::~IDGenerator() {} diff --git a/src/lib/persistence/IDGenerator.h b/src/lib/persistence/IDGenerator.h index 364b3c369..908997aee 100644 --- a/src/lib/persistence/IDGenerator.h +++ b/src/lib/persistence/IDGenerator.h @@ -1,6 +1,8 @@ #pragma once +#include "lib/api/domain/SecurityType.pb.h" #include "lib/api/domain/Venue.pb.h" + #include #include #include @@ -21,6 +23,7 @@ class IDGenerator { std::optional get_id( const api::domain::Venue &venue, + api::domain::SecurityType type, const std::string &name, bool add_if_new = false); @@ -28,9 +31,9 @@ class IDGenerator { const std::string &ns, const std::string &key, bool add_if_new); std::uint64_t get_or_add( - const api::domain::Venue &venue, const std::string &name); - - std::uint64_t get_or_add(const std::string &ns, const std::string &key); + api::domain::Venue venue, + api::domain::SecurityType type, + const std::string &name); private: static std::shared_ptr instance_ptr_; @@ -38,6 +41,10 @@ class IDGenerator { struct PImpl; std::unique_ptr me; + + // These methods are private to force callers to use the [public] overloads with venue, type and + // name params, to ensure consistent formatting. + std::uint64_t get_or_add(const std::string &ns, const std::string &key); }; } // namespace bts::persistence diff --git a/src/lib/persistence/IStore.h b/src/lib/persistence/IStore.h index 1f33ddf28..29e38eb7a 100644 --- a/src/lib/persistence/IStore.h +++ b/src/lib/persistence/IStore.h @@ -1,8 +1,13 @@ #pragma once +#include "lib/persistence/store_row.h" + +#include #include #include #include +#include +#include namespace bts { namespace persistence { @@ -11,17 +16,45 @@ class IStore { public: virtual std::map> retrieve_all() = 0; - virtual std::pair - retrieve_one(std::string const &ns, std::string const &k) = 0; + + /** + * Finds the row with the specified namespace and key. + * + * @return std::tuple The first item indicates + * whether or not a row was found. For successful queries, the second and third + * items are the rowid and value, respectively. + */ + virtual std::tuple retrieve_one( + std::string const &ns, std::string const &k) = 0; virtual void remove(std::string const &ns, std::set const &keys) = 0; virtual void remove_all(std::string const &ns) = 0; - virtual void insert( + virtual std::int64_t insert( std::string const &ns, std::string const &k, std::string const &v) = 0; + + void insert_many(const std::vector &rows); + void insert_many_resolve_ids(std::vector &rows); + virtual void update( std::string const &ns, std::string const &k, std::string const &v) = 0; virtual ~IStore(); }; +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline void IStore::insert_many(const std::vector &rows) { + for (const auto &row : rows) { + insert(row.ns, row.key, row.value); + } +} + +inline void IStore::insert_many_resolve_ids(std::vector &rows) { + for (auto &row : rows) { + row.id = insert(row.ns, row.key, row.value); + } +} + } // namespace persistence } // namespace bts diff --git a/src/lib/persistence/SqliteStore.cpp b/src/lib/persistence/SqliteStore.cpp index fb8eac840..50bf7cbf6 100644 --- a/src/lib/persistence/SqliteStore.cpp +++ b/src/lib/persistence/SqliteStore.cpp @@ -18,82 +18,112 @@ namespace bts { namespace persistence { +using bts::config::SystemConfig; + +static boost::filesystem::path get_db_path(const std::string &str) { + for (char c : str) { + switch (c) { + case '.': + case '/': + case '\\': + // Looks like 'str' is already a path + return {str}; + } + } + return boost::filesystem::path{config::get().data_path()} / + "databases" / str / (str + ".sqlite"); +} + struct SqliteStore::PImpl { - PImpl(std::string const &store_root, std::string const &db_name) - : db(db_name), db_path(persistence::storage_location(db, store_root)), - conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize), - remove(nullptr, sqlite3_finalize), - remove_all(nullptr, sqlite3_finalize), - update(nullptr, sqlite3_finalize) { + std::string const db; + boost::filesystem::path db_path; + sqlite3_ptr conn{nullptr, sqlite3_close}; + sqlite3_stmt_ptr insert{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr insert_many{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr insert_with_id{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr remove{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr remove_all{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr update{nullptr, sqlite3_finalize}; + + PImpl(const boost::filesystem::path &db_path) + : db(db_path.stem().string()), db_path(db_path) { bts_assert(db.size()); } - std::string const db; - boost::filesystem::path db_path; - sqlite3_ptr conn; - sqlite3_stmt_ptr insert; - sqlite3_stmt_ptr remove; - sqlite3_stmt_ptr remove_all; - sqlite3_stmt_ptr update; - BTS_NO_COPY(PImpl); + PImpl(std::string const &store_root, std::string const &db_name) + : PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {} + + explicit PImpl(sqlite3_ptr conn) : db(), db_path(), conn(std::move(conn)) {} + + PImpl(const PImpl &) = delete; + PImpl &operator=(const PImpl &) = delete; }; SqliteStore::SqliteStore(std::string const &db) - : SqliteStore(config::get().data_path(), db) {} + : me(std::make_unique(get_db_path(db))) { + LOG(persist, Info, "SqliteStore ctr. db: {}; db_path: {}", db, + me->db_path.string()); + init_db(); +} -SqliteStore::SqliteStore(std::string const &root_path, std::string const &db) - : me(new PImpl(root_path, db)) { +SqliteStore::SqliteStore(sqlite3_ptr db) + : me(std::make_unique(std::move(db))) { + LOG(persist, Info, "SqliteStore ctr. db: "); init_db(); } void SqliteStore::init_db() { - auto path = me->db_path; - auto parent = path.parent_path(); + const auto path = me->db_path; - if (!boost::filesystem::is_directory(parent)) - bts_assert(boost::filesystem::create_directories(parent)); - - if (!boost::filesystem::exists(path)) { - LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", - me->db, path.string()); - - me->conn = sqlite::open_db(path.c_str()); - bts_assert_msg(me->conn, path.c_str()); - - const char *create_schema_sql = - "create table store ( ns text, key text, value text)"; - sqlite::exec_nofail(me->conn, create_schema_sql); - - const char *key_index_sql = - "create unique index key_index on store(ns,key)"; - sqlite::exec_nofail(me->conn, key_index_sql); - - const char *ns_index_sql = "create index ns_index on store(ns)"; - sqlite::exec_nofail(me->conn, ns_index_sql); - } else { - me->conn = sqlite::open_db(path.c_str()); + if (!path.empty()) { + if (!boost::filesystem::is_directory(path.parent_path())) { + bts_assert(boost::filesystem::create_directories(path.parent_path())); + } + if (!boost::filesystem::exists(path)) { + LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", + me->db, path.string()); + } } + auto &conn = me->conn; + if (conn == nullptr) { + conn = sqlite::open_db(path.c_str()); + bts_assert(conn != nullptr); + } + sqlite::exec_nofail( + conn, + "CREATE TABLE IF NOT EXISTS store (ns TEXT, key TEXT, value TEXT)"); + sqlite::exec_nofail( + conn, "CREATE UNIQUE INDEX IF NOT EXISTS key_index on store(ns, key)"); + sqlite::exec_nofail( + conn, "CREATE INDEX IF NOT EXISTS ns_index on store(ns)"); + conn = sqlite::open_db(path.c_str()); + // Asynchronous for speed - sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;"); - sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL"); + sqlite::exec_nofail(conn, "PRAGMA synchronous = 0;"); + sqlite::exec_nofail(conn, "PRAGMA journal_mode=WAL"); // prepare insert statement const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)"; - me->insert = sqlite::prepare_query(me->conn, insert_sql); + me->insert = sqlite::prepare_query(conn, insert_sql); + + // prepare insert_with_id statement + const char *insert_with_id_sql = + "insert into store (rowid, ns, key, value) values (?,?,?,?)"; + me->insert_with_id = sqlite::prepare_query(conn, insert_with_id_sql); // prepare update statement const char *update_sql = "update store set value = ? where ns = ? and key = ?"; - me->update = sqlite::prepare_query(me->conn, update_sql); + me->update = sqlite::prepare_query(conn, update_sql); // prepare remove statement const char *remove_all_sql = "delete from store where ns = ?"; - me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql); + me->remove_all = sqlite::prepare_query(conn, remove_all_sql); // prepare remove statement const char *remove_sql = "delete from store where ns = ? and key = ?"; - me->remove = sqlite::prepare_query(me->conn, remove_sql); + me->remove = sqlite::prepare_query(conn, remove_sql); } SqliteStore::~SqliteStore() {} @@ -122,25 +152,27 @@ SqliteStore::Data SqliteStore::retrieve_all() { return result; } -std::pair -SqliteStore::retrieve_one(std::string const &ns, std::string const &k) { +std::tuple SqliteStore::retrieve_one( + std::string const &ns, std::string const &k) { auto query_str = fmt::format( - "select ns,key,value from store where ns='{}' and key='{}'", ns, k); + "select rowid,ns,key,value from store where ns='{}' and key='{}'", ns, + k); sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str()); - std::pair result{false, ""}; - + auto result = std::make_tuple(false, 0, ""); auto pq = query.get(); auto sqlite_result = sqlite3_step(pq); if (sqlite_result == SQLITE_ROW) { + std::int64_t rowid = sqlite3_column_int64(pq, 0); std::string ns( - (char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0)); - std::string key( (char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1)); + std::string key( + (char *)sqlite3_column_text(pq, 2), sqlite3_column_bytes(pq, 2)); std::string val( - (char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2)); - result.first = true; - result.second = std::move(val); + (char *)sqlite3_column_blob(pq, 3), sqlite3_column_bytes(pq, 3)); + std::get<0>(result) = true; + std::get<1>(result) = rowid; + std::get<2>(result) = std::move(val); } return result; } @@ -149,91 +181,185 @@ void SqliteStore::remove( std::string const &ns, std::set const &keys) { bts_assert(ns.size()); - for (auto &k : keys) { - SQLITE_CHK( - sqlite3_bind_text( - me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->remove.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not delete row in store. Will retry once. error_code: {} " - "ns:{}, k: {}", - r, ns, k); - sleep_for(2s); - r = sqlite3_step(me->remove.get()); + try { + for (auto &k : keys) { + SQLITE_CHK( + sqlite3_bind_text( + me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->remove.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to delete row. Bailing out. error_code: " - "{}", - r); - bts_assert(r == SQLITE_DONE); + "Could not delete row in store. Will retry once. error_code: " + "{} " + "ns:{}, k: {}", + r, ns, k); + sleep_for(2s); + r = sqlite3_step(me->remove.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to delete row. Bailing out. " + "error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); } - SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::remove threw. what: {}", + std::string{ex.what()}); } } void SqliteStore::remove_all(std::string const &ns) { bts_assert(ns.size()); - SQLITE_CHK( - sqlite3_bind_text( - me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->remove_all.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not delete all rows in store. Will retry once. error_code: {} " - "ns:{}", - r, ns); - sleep_for(2s); - r = sqlite3_step(me->remove_all.get()); + try { + SQLITE_CHK( + sqlite3_bind_text( + me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->remove_all.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to delete all rows. Bailing out. " - "error_code: " - "{}", - r); - bts_assert(r == SQLITE_DONE); + "Could not delete all rows in store. Will retry once. error_code: " + "{} " + "ns:{}", + r, ns); + sleep_for(2s); + r = sqlite3_step(me->remove_all.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to delete all rows. Bailing out. " + "error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::remove_all threw. what: {}", + std::string{ex.what()}); } - SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); } -void SqliteStore::insert( +std::int64_t SqliteStore::insert( std::string const &ns, std::string const &k, std::string const &v) { - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->insert.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not insert row in store. Will retry once. error_code: {} ns: " - "{}, k: {}, v: {}", - r, ns, k, v); - sleep_for(2s); - r = sqlite3_step(me->insert.get()); + try { + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->insert.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to insert row. Bailing out. error_code: {}", - r); - bts_assert(r == SQLITE_DONE); + "Could not insert row in store. Will retry once. error_code: {} " + "ns: " + "{}, k: {}, v: {}", + r, ns, k, v); + sleep_for(2s); + r = sqlite3_step(me->insert.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to insert row. Bailing out. error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + const auto rowid = sqlite3_last_insert_rowid(me->conn.get()); + SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); + return rowid; + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::insert threw. what: {}", + std::string{ex.what()}); } - SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); + return 0; +} + +void SqliteStore::insert_many(const std::vector &rows) { + LOG(md, Info, "SqliteStore::insert_many size: {}", rows.size()); + if (rows.empty()) { + return; + } + try { + sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION"); + sqlite3 *conn_rawp = me->conn.get(); + sqlite3_stmt *insert_rawp = me->insert.get(); + sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get(); + for (const auto &row : rows) { + int col_idx = 1; + sqlite3_stmt *stmt; + if (row.id > 0) { + stmt = insert_with_id_rawp; + sqlite3_bind_int(stmt, col_idx++, row.id); + } else { + stmt = insert_rawp; + } + sqlite3_bind_text( + stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC); + sqlite3_bind_text( + stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC); + int result = sqlite3_step(stmt); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + sqlite3_last_insert_rowid(conn_rawp); + } + sqlite3_reset(stmt); + } + sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION"); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::insert_many threw. what: {}", + std::string{ex.what()}); + } +} + +void SqliteStore::insert_many_resolve_ids(std::vector &rows) { + LOG(md, Info, "SqliteStore::insert_many_resolve_ids size: {}", rows.size()); + if (rows.empty()) { + return; + } + sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION"); + sqlite3 *conn_rawp = me->conn.get(); + sqlite3_stmt *insert_rawp = me->insert.get(); + sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get(); + for (auto &row : rows) { + int col_idx = 1; + sqlite3_stmt *stmt; + if (row.id > 0) { + stmt = insert_with_id_rawp; + sqlite3_bind_int(stmt, col_idx++, row.id); + } else { + stmt = insert_rawp; + } + sqlite3_bind_text( + stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC); + sqlite3_bind_text( + stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC); + int result = sqlite3_step(stmt); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + if (row.id > 0) { + bts_assert(sqlite3_last_insert_rowid(conn_rawp) == row.id); + } else { + row.id = sqlite3_last_insert_rowid(conn_rawp); + } + } + sqlite3_reset(stmt); + } + sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION"); } void SqliteStore::update( diff --git a/src/lib/persistence/SqliteStore.h b/src/lib/persistence/SqliteStore.h index dbc7e0836..6a833df80 100644 --- a/src/lib/persistence/SqliteStore.h +++ b/src/lib/persistence/SqliteStore.h @@ -1,29 +1,39 @@ #pragma once #include "IStore.h" +#include "lib/persistence/store_row.h" +#include "lib/utils/Sqlite_utils.h" + +#include #include +#include namespace bts { namespace persistence { class SqliteStore : public IStore { public: - SqliteStore(std::string const &storage_root, std::string const &db); SqliteStore(std::string const &db); + explicit SqliteStore(sqlite3_ptr db); ~SqliteStore(); + static SqliteStore from_path(const std::string &full_path); + typedef std::map> Data; Data retrieve_all() override; - std::pair - retrieve_one(std::string const &ns, std::string const &k) override; - void - remove(std::string const &ns, std::set const &keys) override; + std::tuple retrieve_one( + std::string const &ns, std::string const &k) override; + + void remove( + std::string const &ns, std::set const &keys) override; void remove_all(std::string const &ns) override; - void insert( + std::int64_t insert( std::string const &ns, std::string const &k, std::string const &v) override; + void insert_many(const std::vector &rows); + void insert_many_resolve_ids(std::vector &rows); void update( std::string const &ns, std::string const &k, @@ -33,6 +43,8 @@ class SqliteStore : public IStore { static std::string storage_location(std::string const &db); private: + SqliteStore(); + struct PImpl; std::unique_ptr me; void init_db(); diff --git a/src/lib/persistence/store_row.h b/src/lib/persistence/store_row.h new file mode 100644 index 000000000..1c7b7f079 --- /dev/null +++ b/src/lib/persistence/store_row.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace bts::persistence { + +struct StoreRow { + std::string ns; + std::string key; + std::string value; + /** + * Represents the unique 'rowid' in the sqlite db. A value of 0 indicates that + * the rowid isn't known, and should either be generated by the db (for inserts), + * or resolved by finding the row with matching ns, key, value (for retrieves). + */ + std::int64_t id{0}; +}; + +} // namespace bts::persistence diff --git a/src/lib/persistence/tests/PersistenceServiceTests.cpp b/src/lib/persistence/tests/PersistenceServiceTests.cpp index 6a8d646e1..b9ed80a5a 100644 --- a/src/lib/persistence/tests/PersistenceServiceTests.cpp +++ b/src/lib/persistence/tests/PersistenceServiceTests.cpp @@ -16,7 +16,8 @@ namespace persistence { TEST_CASE("PersistenceService Sqlite storage", "[persistence]") { utils::TempDir db_dir; - SqliteStore st(db_dir.name(), "test_db"); + SqliteStore st( + (boost::filesystem::path{db_dir.name()} / "test_db.sqlite").string()); SECTION("initial_db_is_empty") { REQUIRE(st.retrieve_all().empty()); } diff --git a/src/lib/utils/checked_arithmetic.h b/src/lib/utils/checked_arithmetic.h new file mode 100644 index 000000000..91e60a6b9 --- /dev/null +++ b/src/lib/utils/checked_arithmetic.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +namespace bts::system::math { + +static std::uint32_t checked_ltou(std::int64_t l); +static std::uint64_t checked_ltoul(std::int64_t l); + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::uint32_t checked_ltou(std::int64_t l) { + if (l < 0) { + throw std::overflow_error{ + "Failed to convert int64 to uint32 (value < 0)"}; + } + if (l > + static_cast(std::numeric_limits::max())) { + throw std::overflow_error{ + "Failed to convert int64 to uint32. value > max)"}; + } + return static_cast(l); +} + +inline std::uint64_t checked_ltoul(std::int64_t l) { + if (l < 0) { + throw std::overflow_error{ + "Failed to convert int64 to uint64 (value < 0)"}; + } + return static_cast(l); +} + +} // namespace bts::system::math diff --git a/src/server/sparkd/analytics/AnalyticsServiceServer.cpp b/src/server/sparkd/analytics/AnalyticsServiceServer.cpp index 7cd0cc84a..83ff900fd 100644 --- a/src/server/sparkd/analytics/AnalyticsServiceServer.cpp +++ b/src/server/sparkd/analytics/AnalyticsServiceServer.cpp @@ -52,7 +52,7 @@ void add_or_update_to_table( const std::string &ns, const std::string &key, const std::string &val) { - auto [exists, existing_val] = store->retrieve_one(ns, key); + auto [exists, rowid, existing_val] = store->retrieve_one(ns, key); LOG(Gen, Info, "add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns, key, val, exists, existing_val); @@ -386,7 +386,8 @@ void AnalyticsServiceServer::save_report( const api::Response<::bts::api::ResultMessage> &resp) { Report report = report_arg; std::string key = std::to_string(report.id()); - auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key); + auto [present, rowid, val] = + me->reports_store->retrieve_one(me->reports_ns, key); if (!present) { int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1; report.set_id(new_id); @@ -408,7 +409,8 @@ void AnalyticsServiceServer::delete_report( const bts::api::analytics::IntWrapper &id, const api::Response<::bts::api::ResultMessage> &resp) { std::string key = std::to_string(id.data()); - auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key); + auto [present, rowid, val] = + me->reports_store->retrieve_one(me->reports_ns, key); api::ResultMessage res; res.set_success(false); if (!present) { diff --git a/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp b/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp index 055e718b4..5acc34f92 100644 --- a/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp +++ b/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp @@ -581,10 +581,11 @@ void negate_id(api::Security *sec) { } void ExchangeSimApplication::persist_book(OrderBook const &b) { - auto existing = me->store->retrieve_one("books", b.sec_desc()); + auto [exists, rowid, oldval] = + me->store->retrieve_one("books", b.sec_desc()); auto val = utils::pb::to_string(b); - if (existing.first) + if (exists) me->store->update("books", b.sec_desc(), val); else me->store->insert("books", b.sec_desc(), val); diff --git a/src/server/sparkd/md/MarketDataGateway.cpp b/src/server/sparkd/md/MarketDataGateway.cpp index 48c6bea01..2303c7ba8 100644 --- a/src/server/sparkd/md/MarketDataGateway.cpp +++ b/src/server/sparkd/md/MarketDataGateway.cpp @@ -246,21 +246,23 @@ void MarketDataGateway::unsubscribe(MarketDataListenerID id) { ////////////////////////////////////////////////////////////////////////////// void MarketDataGateway::start_glbx(const bts::api::md::MarketDataConfig &cfg){ for (const auto &glbx : cfg.glbx()) { - if(glbx.use_databento()){ + if(!glbx.use_databento()){ + _start_onix_glbx(glbx); + return; + } auto p = me_->venue_listeners.emplace_back( std::make_shared(bts::api::VENUE_GLBX)); + // Capture `p->listener` and `me_->sec_id_store` by value + auto listener = p->listener; + auto sec_id_store = me_->sec_id_store; std::string tid_name = "glbx"; - bts::threading::make_thread(tid_name, [&]() { + bts::threading::make_thread(tid_name, [listener, sec_id_store, glbx](){ api::md::glbx::GlbxMarketDatahandler::run( - p->listener, - me_->sec_id_store, + listener, + sec_id_store, glbx); }).detach(); - } - else{ - _start_onix_glbx(glbx); - } - } + } } static void run_onixs() { LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name", @@ -295,7 +297,10 @@ void MarketDataGateway::_start_onix_glbx( "MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. " "cfg:\n{}", os::get_thread_id(), os::get_thread_name(), cfg.DebugString()); - bts::md::run_onixs(); + bts::threading::make_thread("Onix CME", [&]() { + bts::md::run_onixs(); + }).detach(); + } @@ -374,35 +379,38 @@ void MarketDataGateway::_start_replays( void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) { // taifex - int counter = 0; for (const auto &taifex : cfg.taifex()) { - counter++; auto p = me_->venue_listeners.emplace_back( std::make_shared(bts::api::VENUE_TAIFEX)); - std::string tid_name = "taifex" +std::to_string(counter); - bts::threading::make_thread(tid_name, [&]() { + // Capture `p->listener` and `me_->sec_id_store` by value + auto listener = p->listener; + auto sec_id_store = me_->sec_id_store; + std::string tid_name = "taifex"; + bts::threading::make_thread(tid_name, [listener, sec_id_store, taifex]() { bts::md::taifex::TaifexDataGateway::run( - taifex, p->listener, me_->sec_id_store); + taifex, listener, sec_id_store); }).detach(); } } -void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg){ - // cboe +void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg) { for (const api::md::CboeMarketDataConfig &cboe : cfg.cboe()) { - auto p = me_->venue_listeners.emplace_back( - std::make_shared(bts::api::VENUE_CBOE)); - std::string tid_name = "CBOE"; - bts::threading::make_thread(tid_name, [&]() { - api::md::cboe::CboeMarketDatahandler::run( - p->listener, - me_->sec_id_store, - cboe); - }).detach(); + auto p = me_->venue_listeners.emplace_back( + std::make_shared(bts::api::VENUE_CBOE)); + std::string tid_name = "CBOE"; + + // Capture `p->listener` and `me_->sec_id_store` by value + auto listener = p->listener; + auto sec_id_store = me_->sec_id_store; + + bts::threading::make_thread(tid_name, [listener, sec_id_store, cboe]() { + api::md::cboe::CboeMarketDatahandler::run( + listener, + sec_id_store, + cboe); + }).detach(); - UNUSED(cboe); } } - } // namespace bts::md diff --git a/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.cpp b/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.cpp index 3561d8247..d69533a2f 100644 --- a/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.cpp +++ b/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.cpp @@ -10,20 +10,21 @@ namespace bts::md::binance { +using bts::api::domain::SecurityType; using bts::domain::SecurityID; using bts::domain::SecurityMarketDataStatistics; -using std::pair; using std::string; +using std::tuple; using std::vector; #define VENUE_NAME (Venue_Name(Venue::BINANCE)) struct BinancePriceStatisticsAdaptor::PImpl { - vector> securities; + vector> securities; IMarketDataListener *listener; SecurityID get_sec_id(const string &sec_desc) const { - for (const auto &[desc, id] : securities) { + for (const auto &[type, desc, id] : securities) { if (desc == sec_desc) { return id; } @@ -36,7 +37,8 @@ struct BinancePriceStatisticsAdaptor::PImpl { }; BinancePriceStatisticsAdaptor::BinancePriceStatisticsAdaptor( - vector> securities, IMarketDataListener *listener) + vector> securities, + IMarketDataListener *listener) : me_(std::make_unique()) { bts_assert(listener != nullptr); me_->securities = std::move(securities); diff --git a/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.h b/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.h index 3b3694fc6..280deaf34 100644 --- a/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.h +++ b/src/server/sparkd/md/binance/BinancePriceStatisticsAdaptor.h @@ -1,9 +1,11 @@ #pragma once +#include "lib/api/domain/SecurityType.pb.h" #include "lib/domain/SecurityID.h" #include "server/sparkd/md/ccapi/ICCApiMessageHandler.h" #include +#include #include #include @@ -20,7 +22,10 @@ namespace bts::md::binance { class BinancePriceStatisticsAdaptor : public ICCApiMessageHandler { public: BinancePriceStatisticsAdaptor( - std::vector> securities, + std::vector> securities, IMarketDataListener *listener); ~BinancePriceStatisticsAdaptor(); diff --git a/src/server/sparkd/md/binance/BinanceSecurityRequestor.cpp b/src/server/sparkd/md/binance/BinanceSecurityRequestor.cpp index 38c3c9a85..d9483fb86 100644 --- a/src/server/sparkd/md/binance/BinanceSecurityRequestor.cpp +++ b/src/server/sparkd/md/binance/BinanceSecurityRequestor.cpp @@ -49,6 +49,7 @@ using std::vector; #define _WARN(...) _LOG(Warn, __VA_ARGS__) #define _ERROR(...) _LOG(Error, __VA_ARGS__) +#define VENUE Venue::VENUE_BINANCE #define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE #define BINANCE_MSG_TYPE_NAME(x) \ @@ -156,7 +157,8 @@ struct BinanceSecurityRequestor::PImpl { for (auto &sec : added_secs) { // Since the ids haven't been set yet, we should expect them to be 0 bts_assert(sec.id().value() == 0); - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + SecurityID sec_id{ + sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())}; sec.security_common.id = sec_id; } } diff --git a/src/server/sparkd/md/binance/BinanceSubscription.cpp b/src/server/sparkd/md/binance/BinanceSubscription.cpp index 25a01ec32..bead58867 100644 --- a/src/server/sparkd/md/binance/BinanceSubscription.cpp +++ b/src/server/sparkd/md/binance/BinanceSubscription.cpp @@ -34,6 +34,7 @@ namespace bts::md::binance { +using bts::api::domain::SecurityType; using bts::api::domain::Venue; using bts::api::domain::Venue_Name; using bts::api::md::BinanceMarketDataConfig; @@ -50,6 +51,7 @@ using std::unique_ptr; using std::unordered_map; using std::vector; +#define VENUE Venue::VENUE_BINANCE #define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE @@ -123,14 +125,15 @@ static CCApiCorrelationID mk_stats_corr_id(const string &desc) { } static ccapi::Request mk_stats_req( - const vector &sec_descs, uint32_t batch_id) { + const vector> &sec_descs, + uint32_t batch_id) { const auto corr_id = mk_stats_corr_id(FMT("BATCH_{}", batch_id)); ccapi::Request req{ ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, CCAPI_VENUE_NAME}; req.setCorrelationId(corr_id.to_string()); auto quoted_sec_descs = - sec_descs | - ranges::views::transform([](const auto &s) { return "\"" + s + "\""; }); + sec_descs | ranges::views::transform( + [](const auto &s) { return "\"" + s.second + "\""; }); const string symbols_query_arg = FMT_JOIN(quoted_sec_descs, ","); req.appendParam( {{CCAPI_HTTP_PATH, BINANCE_PRICE_STATS_PATH}, @@ -209,16 +212,18 @@ struct BinanceSubscription::PImpl { sec_requestor.request(); } - DepthHandler &mk_depth_handler(const string &sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + DepthHandler &mk_depth_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = depth_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert(is_new); return it->second; } - TickerHandler &mk_ticker_handler(const string &sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + TickerHandler &mk_ticker_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = ticker_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -227,23 +232,25 @@ struct BinanceSubscription::PImpl { } StatsHandler &mk_stats_handler( - const vector &sec_descs, uint32_t hnd_id) { - vector> sec_desc_id_pairs; - sec_desc_id_pairs.reserve(sec_descs.size()); - for (const auto &sec_desc : sec_descs) { - sec_desc_id_pairs.emplace_back( - sec_desc, - SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}); + const vector> &sec_descs, + uint32_t hnd_id) { + vector> sec_desc_id_tuples; + sec_desc_id_tuples.reserve(sec_descs.size()); + for (auto [type, name] : sec_descs) { + SecurityID id{sec_id_store->get_or_add(VENUE, type, name)}; + sec_desc_id_tuples.emplace_back(type, std::move(name), id); } auto [it, is_new] = stats_handlers.try_emplace( - hnd_id, std::make_unique(sec_desc_id_pairs, listener)); + hnd_id, + std::make_unique(sec_desc_id_tuples, listener)); bts_assert_msgf( is_new, "StatsHandler already created with id '{}'", hnd_id); return *it->second; } void add_stats_handler( - const vector &sec_descs, CCApiSession::BatchRequest &reqs) { + const vector> &sec_descs, + CCApiSession::BatchRequest &reqs) { if (sec_descs.empty()) { return; } @@ -268,14 +275,14 @@ struct BinanceSubscription::PImpl { CCApiSession::BatchRequest stats_reqs; const int N_SECS_PER_STATS_REQ = 16; - vector curr_stats_sec_descs; - for (const auto &sec_desc : securities) { + vector> curr_stats_sec_descs; + for (const auto &[type, desc] : securities) { depth_subs.push_back( - {mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)}); + {mk_depth_sub(desc, depth_opts), &mk_depth_handler(type, desc)}); ticker_subs.push_back( - {mk_ticker_sub(sec_desc, ticker_opts), - &mk_ticker_handler(sec_desc)}); - curr_stats_sec_descs.push_back(sec_desc); + {mk_ticker_sub(desc, ticker_opts), + &mk_ticker_handler(type, desc)}); + curr_stats_sec_descs.push_back(std::make_pair(type, desc)); if (curr_stats_sec_descs.size() == N_SECS_PER_STATS_REQ) { add_stats_handler(curr_stats_sec_descs, stats_reqs); curr_stats_sec_descs.clear(); @@ -299,7 +306,7 @@ struct BinanceSubscription::PImpl { */ void on_securities(vector secs) { // 'sec_descs' is used to make depth subscriptions - vector sec_descs; + vector> sec_descs; sec_descs.reserve(secs.size()); // 'sec_ptrs' is passed to IMarketDataListeners vector sec_ptrs; @@ -312,7 +319,8 @@ struct BinanceSubscription::PImpl { id, std::make_unique(std::move(s))); bts_assert_msgf( is_new, "Received duplicate Binance security '{}'", desc); - sec_descs.push_back(std::move(desc)); + sec_descs.push_back( + std::make_pair(it->second->type(), std::move(desc))); // add ptr to entry in 'securities' (not the loop var, we moved that) sec_ptrs.push_back(it->second.get()); } diff --git a/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp b/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp index 301e25b1c..954106e1c 100644 --- a/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp +++ b/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp @@ -10,20 +10,21 @@ namespace bts::md::binance_us { +using bts::api::domain::SecurityType; using bts::domain::SecurityID; using bts::domain::SecurityMarketDataStatistics; -using std::pair; using std::string; +using std::tuple; using std::vector; #define VENUE_NAME (Venue_Name(Venue::BINANCE)) struct BinanceUsPriceStatisticsAdaptor::PImpl { - vector> securities; + vector> securities; IMarketDataListener *listener; SecurityID get_sec_id(const string &sec_desc) const { - for (const auto &[desc, id] : securities) { + for (const auto &[type, desc, id] : securities) { if (desc == sec_desc) { return id; } @@ -36,7 +37,8 @@ struct BinanceUsPriceStatisticsAdaptor::PImpl { }; BinanceUsPriceStatisticsAdaptor::BinanceUsPriceStatisticsAdaptor( - vector> securities, IMarketDataListener *listener) + vector> securities, + IMarketDataListener *listener) : me_(std::make_unique()) { bts_assert(listener != nullptr); me_->securities = std::move(securities); diff --git a/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.h b/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.h index 2caadce6b..a0af77602 100644 --- a/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.h +++ b/src/server/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.h @@ -1,9 +1,11 @@ #pragma once +#include "lib/api/domain/SecurityType.pb.h" #include "lib/domain/SecurityID.h" #include "server/sparkd/md/ccapi/ICCApiMessageHandler.h" #include +#include #include #include @@ -25,7 +27,10 @@ namespace bts::md::binance_us { class BinanceUsPriceStatisticsAdaptor : public ICCApiMessageHandler { public: BinanceUsPriceStatisticsAdaptor( - std::vector> securities, + std::vector> securities, IMarketDataListener *listener); ~BinanceUsPriceStatisticsAdaptor(); diff --git a/src/server/sparkd/md/binance_us/BinanceUsSecurityRequestor.cpp b/src/server/sparkd/md/binance_us/BinanceUsSecurityRequestor.cpp index 3288e7327..620b861a8 100644 --- a/src/server/sparkd/md/binance_us/BinanceUsSecurityRequestor.cpp +++ b/src/server/sparkd/md/binance_us/BinanceUsSecurityRequestor.cpp @@ -156,7 +156,8 @@ struct BinanceUsSecurityRequestor::PImpl { for (auto &sec : added_secs) { // Since the ids haven't been set yet, we should expect them to be 0 bts_assert(sec.id().value() == 0); - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + SecurityID sec_id{ + sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())}; sec.security_common.id = sec_id; } } diff --git a/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp b/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp index 6d841cdd4..a582f1eb9 100644 --- a/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp +++ b/src/server/sparkd/md/binance_us/BinanceUsSubscription.cpp @@ -36,6 +36,7 @@ namespace bts::md::binance_us { +using bts::api::domain::SecurityType; using bts::api::domain::Venue; using bts::api::domain::Venue_Name; using bts::api::md::BinanceUsMarketDataConfig; @@ -47,11 +48,13 @@ using bts::io::QueryParams; using bts::persistence::IDGenerator; using std::pair; using std::string; +using std::tuple; using std::uint32_t; using std::unique_ptr; using std::unordered_map; using std::vector; +#define VENUE Venue::VENUE_BINANCE_US #define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_US)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_US @@ -126,14 +129,14 @@ static CCApiCorrelationID mk_stats_corr_id(const string &desc) { } static ccapi::Request mk_stats_req( - const vector &sec_descs, uint32_t batch_id) { + const vector> &sec_descs, uint32_t batch_id) { const auto corr_id = mk_stats_corr_id(FMT("BATCH_{}", batch_id)); ccapi::Request req{ ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, CCAPI_VENUE_NAME}; req.setCorrelationId(corr_id.to_string()); auto quoted_sec_descs = - sec_descs | - ranges::views::transform([](const auto &s) { return "\"" + s + "\""; }); + sec_descs | ranges::views::transform( + [](const auto &s) { return "\"" + s.second + "\""; }); const string symbols_query_arg = FMT_JOIN(quoted_sec_descs, ","); req.appendParam( {{CCAPI_HTTP_PATH, BINANCE_US_PRICE_STATS_PATH}, @@ -212,16 +215,18 @@ struct BinanceUsSubscription::PImpl { sec_requestor.request(); } - DepthHandler &mk_depth_handler(const string &sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + DepthHandler &mk_depth_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = depth_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert(is_new); return it->second; } - TickerHandler &mk_ticker_handler(const string &sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + TickerHandler &mk_ticker_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = ticker_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -230,23 +235,25 @@ struct BinanceUsSubscription::PImpl { } StatsHandler &mk_stats_handler( - const vector &sec_descs, uint32_t hnd_id) { - vector> sec_desc_id_pairs; - sec_desc_id_pairs.reserve(sec_descs.size()); - for (const auto &sec_desc : sec_descs) { - sec_desc_id_pairs.emplace_back( - sec_desc, - SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}); + const vector> &sec_descs, + uint32_t hnd_id) { + vector> sec_desc_id_tuples; + sec_desc_id_tuples.reserve(sec_descs.size()); + for (auto [type, name] : sec_descs) { + SecurityID id{sec_id_store->get_or_add(VENUE, type, name)}; + sec_desc_id_tuples.emplace_back(type, std::move(name), id); } auto [it, is_new] = stats_handlers.try_emplace( - hnd_id, std::make_unique(sec_desc_id_pairs, listener)); + hnd_id, + std::make_unique(sec_desc_id_tuples, listener)); bts_assert_msgf( is_new, "StatsHandler already created with id '{}'", hnd_id); return *it->second; } void add_stats_handler( - const vector &sec_descs, CCApiSession::BatchRequest &reqs) { + const vector> &sec_descs, + CCApiSession::BatchRequest &reqs) { if (sec_descs.empty()) { return; } @@ -271,14 +278,14 @@ struct BinanceUsSubscription::PImpl { CCApiSession::BatchRequest stats_reqs; const int N_SECS_PER_STATS_REQ = 16; - vector curr_stats_sec_descs; - for (const auto &sec_desc : securities) { + vector> curr_stats_sec_descs; + for (const auto &[type, name] : securities) { depth_subs.push_back( - {mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)}); + {mk_depth_sub(name, depth_opts), &mk_depth_handler(type, name)}); ticker_subs.push_back( - {mk_ticker_sub(sec_desc, ticker_opts), - &mk_ticker_handler(sec_desc)}); - curr_stats_sec_descs.push_back(sec_desc); + {mk_ticker_sub(name, ticker_opts), + &mk_ticker_handler(type, name)}); + curr_stats_sec_descs.push_back(std::make_pair(type, name)); if (curr_stats_sec_descs.size() == N_SECS_PER_STATS_REQ) { add_stats_handler(curr_stats_sec_descs, stats_reqs); curr_stats_sec_descs.clear(); @@ -302,7 +309,7 @@ struct BinanceUsSubscription::PImpl { */ void on_securities(vector secs) { // 'sec_descs' is used to make depth subscriptions - vector sec_descs; + vector> sec_descs; sec_descs.reserve(secs.size()); // 'sec_ptrs' is passed to IMarketDataListeners vector sec_ptrs; @@ -315,7 +322,7 @@ struct BinanceUsSubscription::PImpl { id, std::make_unique(std::move(s))); bts_assert_msgf( is_new, "Received duplicate Binance US security '{}'", desc); - sec_descs.push_back(std::move(desc)); + sec_descs.push_back(std::make_pair(s.type(), std::move(desc))); // add ptr to entry in 'securities' (not the loop var, we moved that) sec_ptrs.push_back(it->second.get()); } diff --git a/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSecurityRequestor.cpp b/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSecurityRequestor.cpp index f65506e89..8f78c2f11 100644 --- a/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSecurityRequestor.cpp +++ b/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSecurityRequestor.cpp @@ -48,6 +48,7 @@ using std::vector; #define _WARN(...) _LOG(Warn, __VA_ARGS__) #define _ERROR(...) _LOG(Error, __VA_ARGS__) +#define VENUE Venue::VENUE_BINANCE_USDS_FUTURES #define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_USDS_FUTURES)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_USDS_FUTURES #define VENUE_PRICE_STATS_PATH BINANCE_USDS_FUTURES_PRICE_STATS_PATH @@ -157,7 +158,8 @@ struct BinanceUsdsFuturesSecurityRequestor::PImpl { // the ids, if it does (for whatever reason), better to assert/crash // than quietly let a potential bug slip through the cracks. bts_assert(sec.id().value() == 0); - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + SecurityID sec_id{ + sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())}; sec.security_common.id = sec_id; } } diff --git a/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp b/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp index d1fb9f47f..9dbbb247f 100644 --- a/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp +++ b/src/server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp @@ -30,6 +30,7 @@ namespace bts::md::binance_usds_futures { +using bts::api::domain::SecurityType; using bts::api::domain::Venue; using bts::api::domain::Venue_Name; using bts::api::md::BinanceUsdsFuturesMarketDataConfig; @@ -39,12 +40,14 @@ using bts::domain::SecurityID; using bts::io::ProtoPrinter; using bts::io::QueryParams; using bts::persistence::IDGenerator; +using std::pair; using std::string; using std::uint32_t; using std::unique_ptr; using std::unordered_map; using std::vector; +#define VENUE Venue::VENUE_BINANCE_USDS_FUTURES #define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_USDS_FUTURES)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_USDS_FUTURES #define VENUE_PRICE_STATS_PATH BINANCE_USDS_FUTURES_PRICE_STATS_PATH @@ -192,8 +195,8 @@ struct BinanceUsdsFuturesSubscription::PImpl { sec_requestor.request(); } - SecurityID get_sec_id(const string &sec_desc) { - return SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + SecurityID get_sec_id(SecurityType sec_type, const string &sec_desc) { + return SecurityID{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; } /** @@ -211,8 +214,9 @@ struct BinanceUsdsFuturesSubscription::PImpl { return sec_desc; } - DepthHandler &add_depth_handler(const string &sec_desc) { - const auto sec_id = get_sec_id(sec_desc); + DepthHandler &add_depth_handler( + SecurityType sec_type, const string &sec_desc) { + const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = depth_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -220,8 +224,9 @@ struct BinanceUsdsFuturesSubscription::PImpl { return it->second; } - TickerHandler &add_ticker_handler(const string &sec_desc) { - const auto sec_id = get_sec_id(sec_desc); + TickerHandler &add_ticker_handler( + SecurityType sec_type, const string &sec_desc) { + const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = ticker_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -229,8 +234,9 @@ struct BinanceUsdsFuturesSubscription::PImpl { return it->second; } - StatsHandler &add_stats_handler(const string &sec_desc) { - const auto sec_id = get_sec_id(sec_desc); + StatsHandler &add_stats_handler( + SecurityType sec_type, const string &sec_desc) { + const auto sec_id = get_sec_id(sec_type, sec_desc); auto [it, is_new] = stats_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -252,16 +258,16 @@ struct BinanceUsdsFuturesSubscription::PImpl { CCApiSession::BatchSubscription ticker_subs; CCApiSession::BatchRequest stats_reqs; - for (const auto &sec_desc : securities) { - const auto sec_desc_for_reqs = get_sec_desc_for_reqs(sec_desc); + for (auto [type, name] : securities) { + const auto sec_desc_for_reqs = get_sec_desc_for_reqs(name); depth_subs.push_back( {mk_depth_sub(sec_desc_for_reqs, depth_opts), - &add_depth_handler(sec_desc)}); + &add_depth_handler(type, name)}); ticker_subs.push_back( {mk_ticker_sub(sec_desc_for_reqs, ticker_opts), - &add_ticker_handler(sec_desc)}); + &add_ticker_handler(type, name)}); stats_reqs.push_back( - {mk_stats_req(sec_desc), &add_stats_handler(sec_desc)}); + {mk_stats_req(name), &add_stats_handler(type, name)}); } session.subscribe( @@ -278,7 +284,7 @@ struct BinanceUsdsFuturesSubscription::PImpl { */ void on_securities(vector secs) { // 'sec_descs' is used to make depth subscriptions - vector sec_descs; + vector> sec_descs; sec_descs.reserve(secs.size()); // 'sec_ptrs' is passed to IMarketDataListeners vector sec_ptrs; @@ -291,7 +297,7 @@ struct BinanceUsdsFuturesSubscription::PImpl { id, std::make_unique(std::move(s))); bts_assert_msgf( is_new, "Received duplicate Binance USDS Future '{}'", desc); - sec_descs.push_back(std::move(desc)); + sec_descs.push_back(std::make_pair(s.type(), std::move(desc))); // add ptr to entry in 'securities' (not the loop var, we moved that) sec_ptrs.push_back(it->second.get()); } diff --git a/src/server/sparkd/md/cboe/CboeMarketDataHandler.cpp b/src/server/sparkd/md/cboe/CboeMarketDataHandler.cpp index ee818d5a0..47071cb1f 100644 --- a/src/server/sparkd/md/cboe/CboeMarketDataHandler.cpp +++ b/src/server/sparkd/md/cboe/CboeMarketDataHandler.cpp @@ -30,7 +30,8 @@ void CboeMarketDatahandler::run( base_path + config.databento_settings().bbo_file_name(), config.databento_settings().key(), config.databento_settings().dataset(), - config.databento_settings().filter_securities_one_year()); + config.databento_settings().filter_securities_one_year(), + config.databento_settings().load_reference_data()); } } // namespace bts::md::cboe \ No newline at end of file diff --git a/src/server/sparkd/md/coinbase/CoinbaseSecurityRequestor.cpp b/src/server/sparkd/md/coinbase/CoinbaseSecurityRequestor.cpp index 4742a7fd3..d7e8b37c2 100644 --- a/src/server/sparkd/md/coinbase/CoinbaseSecurityRequestor.cpp +++ b/src/server/sparkd/md/coinbase/CoinbaseSecurityRequestor.cpp @@ -147,7 +147,8 @@ struct CoinbaseSecurityRequestor::PImpl { for (auto &sec : added_secs) { // Since the ids haven't been set yet, we should expect them to be 0 bts_assert(sec.id().value() == 0); - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + SecurityID sec_id{ + sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())}; sec.security_common.id = sec_id; } } diff --git a/src/server/sparkd/md/coinbase/CoinbaseSubscription.cpp b/src/server/sparkd/md/coinbase/CoinbaseSubscription.cpp index d7b311210..5fc29eb42 100644 --- a/src/server/sparkd/md/coinbase/CoinbaseSubscription.cpp +++ b/src/server/sparkd/md/coinbase/CoinbaseSubscription.cpp @@ -25,6 +25,9 @@ #include namespace bts::md::coinbase { + +using bts::api::domain::SecurityType; +using bts::api::domain::Venue; using bts::api::md::CoinbaseMarketDataConfig; using bts::domain::Security; using bts::domain::SecurityCriteria; @@ -32,10 +35,13 @@ using bts::domain::SecurityID; using bts::io::ProtoPrinter; using bts::io::QueryParams; using bts::persistence::IDGenerator; +using std::pair; using std::string; using std::unique_ptr; using std::unordered_map; using std::vector; + +#define VENUE Venue::VENUE_COINBASE #define VENUE_NAME (Venue_Name(Venue::VENUE_COINBASE)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_COINBASE @@ -183,24 +189,29 @@ struct CoinbaseSubscription::PImpl { sec_requestor.callback().bind<&PImpl::on_securities>(*this); sec_requestor.request(); } - StatsHandler &mk_stats_handler(const string &sec_desc) { - auto sec_id = SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + + StatsHandler &mk_stats_handler( + SecurityType sec_type, const string &sec_desc) { + auto sec_id = + SecurityID{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = stats_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( is_new, "StatsHandler already created with id '{}'", sec_id); return it->second; } - DepthHandler &mk_depth_handler(const string &sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + DepthHandler &mk_depth_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = depth_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert(is_new); return it->second; } - TickerHandler &mk_ticker_handler(const string &sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + TickerHandler &mk_ticker_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = ticker_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -222,14 +233,15 @@ struct CoinbaseSubscription::PImpl { CCApiSession::BatchRequest stats_reqs; vector curr_stats_sec_descs; - for (const auto &sec_desc : securities) { + for (const auto &[sec_type, sec_desc] : securities) { depth_subs.push_back( - {mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)}); + {mk_depth_sub(sec_desc, depth_opts), + &mk_depth_handler(sec_type, sec_desc)}); ticker_subs.push_back( {mk_ticker_sub(sec_desc, ticker_opts), - &mk_ticker_handler(sec_desc)}); + &mk_ticker_handler(sec_type, sec_desc)}); stats_reqs.push_back( - {mk_stats_req(sec_desc), &mk_stats_handler(sec_desc)}); + {mk_stats_req(sec_desc), &mk_stats_handler(sec_type, sec_desc)}); } session.subscribe( @@ -247,7 +259,7 @@ struct CoinbaseSubscription::PImpl { void on_securities(std::vector secs) { LOG(md, Info, "on sec {}", secs.size()); // 'sec_descs' is used to make depth subscriptions - vector sec_descs; + vector> sec_descs; sec_descs.reserve(secs.size()); // 'sec_ptrs' is passed to IMarketDataListeners vector sec_ptrs; @@ -260,7 +272,7 @@ struct CoinbaseSubscription::PImpl { id, std::make_unique(std::move(s))); bts_assert_msgf( is_new, "Received duplicate Coinbase security '{}'", desc); - sec_descs.push_back(std::move(desc)); + sec_descs.push_back(std::make_pair(s.type(), std::move(desc))); // add ptr to entry in 'securities' (not the loop var, we moved that) sec_ptrs.push_back(it->second.get()); } diff --git a/src/server/sparkd/md/databento/DatabentoDataHandler.cpp b/src/server/sparkd/md/databento/DatabentoDataHandler.cpp index a546c4a03..665855fa7 100644 --- a/src/server/sparkd/md/databento/DatabentoDataHandler.cpp +++ b/src/server/sparkd/md/databento/DatabentoDataHandler.cpp @@ -25,18 +25,36 @@ #include #include #include +#include namespace bts::md::DatabentoHandler { -std::string remove_spaces(const std::string_view& str) { - std::string result; - result.reserve(str.size()); // Reserve space for better performance - for (char c : str) { - if (!isspace(c)) { - result += c; - } - } - return result; +#define SEC_TYPE(name) bts::api::domain::SECURITY_TYPE_##name + +using bts::domain::SecurityID; +using bts::api::domain::CallPut; +using bts::api::domain::SecurityType; +using bts::api::domain::Venue; + +std::string remove_spaces(const std::string_view &str) { + std::string result; + result.reserve(str.size()); // Reserve space for better performance + for (char c : str) { + if (!isspace(c)) { + result += c; + } + } + return result; +} + +static SecurityType parse_sec_type(std::string_view str) { + if (str.find("OPT") != std::string::npos) { + return SEC_TYPE(OPTION); + } + if (str.find("FUT") != std::string::npos) { + return SEC_TYPE(FUTURE); + } + return SEC_TYPE(UNSPECIFIED); } void DatabentoDatahandler::run( @@ -49,8 +67,10 @@ void DatabentoDatahandler::run( const std::string &key, const std::string &dataset, const bool filter_securities, - const std::chrono::duration sleep_duration) { + const bool load_reference_data, + const std::chrono::duration sleep_duration ) { + (void)filter_securities; auto sleep_duration_updated = sleep_duration; if (!use_file) { @@ -77,22 +97,9 @@ void DatabentoDatahandler::run( bts_assert_msg(false, "Exchange not supported on databento"); return databento::KeepGoing::Stop; } - if(filter_securities){ - const auto exp_nanos = msg.expiration; - std::string exp_string = databento::ToIso8601(exp_nanos); - const auto expiration_time = bts::core::TimeStamp::from_iso_string(exp_string); - if(expiration_time.seconds() < expiration_time.now().seconds() || expiration_time.seconds() > (expiration_time.now().seconds() + 60*60*24*365)){ - return databento::KeepGoing::Continue; - } - } - - - bts::domain::SecurityID sec_id{sec_id_store->get_or_add( - venue, std::to_string(msg.hd.instrument_id))}; bts::domain::Security sec; sec.security_common = domain::SecurityCommon(); sec.security_common.venue = venue; - sec.security_common.id = sec_id; sec.security_common.exchange_sec_id = msg.hd.instrument_id; sec.security_common.desc = msg.RawSymbol(); sec.security_common.symbol = msg.Asset(); @@ -104,38 +111,55 @@ void DatabentoDatahandler::run( sec.security_common.currency = {msg.currency.data(), msg.currency.size()}; // sec.security_common.cpp = msg.hd. - const std::string_view sec_type = {msg.security_type.data(), msg.security_type.size()}; + const SecurityType sec_type = + parse_sec_type({msg.security_type.data(), msg.security_type.size()}); const auto exp_nanos = msg.expiration; std::string exp_string = databento::ToIso8601(exp_nanos); const auto expiration_time = bts::core::TimeStamp::from_iso_string(exp_string); - if (sec_type.find("OPT") != std::string::npos) { - auto call_put = api::domain::CallPut::PUT; - if(msg.instrument_class == 'C' ){ - call_put=api::domain::CallPut::CALL; - } - else{ - bts_assert(msg.instrument_class == 'P'); - } - sec.security.emplace( - expiration_time, bts::domain::Price(msg.strike_price, -9),call_put); - } else if (sec_type.find("FUT") != std::string::npos) { - sec.security.emplace(expiration_time); - } else { - bts_assert_msg(false, "Only futures and options supported on databento"); - return databento::KeepGoing::Stop; - } + switch (sec_type) { + case SEC_TYPE(FUTURE): + sec.security.emplace(expiration_time); + break; + case SEC_TYPE(OPTION): { + auto call_put = CallPut::PUT; + if (msg.instrument_class == 'C') { + call_put = CallPut::CALL; + } else { + bts_assert(msg.instrument_class == 'P'); + } + sec.security.emplace( + expiration_time, bts::domain::Price(msg.strike_price, -9), + call_put); + break; + } + default: + bts_assert_msg( + false, "Only futures and options supported on databento"); + return databento::KeepGoing::Stop; + } + *sec.security_common.id = sec_id_store->get_or_add( + venue, sec_type, std::to_string(msg.hd.instrument_id)); listener->on_securities({&sec}); return databento::KeepGoing::Continue; }; auto process_bbo = [&](const databento::Record &record) { const auto &msg = record.Get(); - auto sec_id_opt = sec_id_store->get_id( - venue, std::to_string(msg.hd.instrument_id), false); - if (!sec_id_opt.has_value()) - return databento::KeepGoing::Continue; + // The security type to is needed to get the id. Since the sec type isn't known, + // check for an id using both types (OPTION and FUTURE). This is safe to do since + // options and futures never have common names (instrument ids). + const auto name = std::to_string(msg.hd.instrument_id); + auto sec_id_opt = + sec_id_store->get_id(venue, SEC_TYPE(OPTION), name, false); + if (!sec_id_opt.has_value()) { + sec_id_opt = + sec_id_store->get_id(venue, SEC_TYPE(FUTURE), name, false); + if (!sec_id_opt.has_value()) { + return databento::KeepGoing::Continue; + } + } sleep_for(sleep_duration_updated); bts::domain::SecurityID sec_id = bts::domain::SecurityID(sec_id_opt.value()); @@ -181,19 +205,16 @@ void DatabentoDatahandler::run( {} }; //hardcoding for now will update later + std::vector symbols; switch (venue) { - case api::domain::VENUE_CBOE: - client.Subscribe({"SPX.OPT","SPXW.OPT"}, databento::Schema::Definition, databento::SType::Parent,databento::UnixNanos{}); - client.Subscribe({"SPX.OPT","SPXW.OPT"}, databento::Schema::Mbp1, databento::SType::Parent); - break; - case api::domain::VENUE_GLBX: - client.Subscribe({"ES.FUT"}, databento::Schema::Definition, databento::SType::Parent); - client.Subscribe({"ES.FUT"}, databento::Schema::Mbp1, databento::SType::Parent); - break; - default: - bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW"); - + case api::domain::VENUE_CBOE: symbols= {"SPX.OPT","SPXW.OPT"}; break; + case api::domain::VENUE_GLBX: symbols= {"ES.FUT"}; break; + default:bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW"); } + if(load_reference_data) + client.Subscribe(symbols, databento::Schema::Definition, databento::SType::Parent,databento::UnixNanos{}); + else client.Subscribe(symbols, databento::Schema::Definition, databento::SType::Parent); + client.Subscribe(symbols, databento::Schema::Mbp1, databento::SType::Parent); client.Start(); while (true) { try { diff --git a/src/server/sparkd/md/databento/DatabentoDataHandler.h b/src/server/sparkd/md/databento/DatabentoDataHandler.h index 45abb5dc7..00626b4c9 100644 --- a/src/server/sparkd/md/databento/DatabentoDataHandler.h +++ b/src/server/sparkd/md/databento/DatabentoDataHandler.h @@ -15,6 +15,7 @@ class DatabentoDatahandler { const std::string &dataset="", const std::string& key = "db-Q4eCbTQfrAHaG6sAcuFUYB9YqkF6j", const bool filter_securities=false, + const bool load_reference_data = true, const std::chrono::duration sleep_duration = 0.04s); }; } // namespace bts::md::databento \ No newline at end of file diff --git a/src/server/sparkd/md/deribit/DeribitInstrumentStore.cpp b/src/server/sparkd/md/deribit/DeribitInstrumentStore.cpp index f557c4044..dab42a2fe 100644 --- a/src/server/sparkd/md/deribit/DeribitInstrumentStore.cpp +++ b/src/server/sparkd/md/deribit/DeribitInstrumentStore.cpp @@ -19,6 +19,7 @@ namespace bts::md::deribit { using bts::api::domain::CallPut; +using bts::api::domain::SecurityType; using bts::domain::FutureSecurity; using bts::domain::OptionSecurity; using bts::domain::Price; @@ -106,8 +107,8 @@ struct DeribitInstrumentStore::PImpl { return &it->second; } - Security *find(const string &name) { - auto id = sec_id_store_->get_id(VENUE, name); + Security *find(SecurityType type, const string &name) { + auto id = sec_id_store_->get_id(VENUE, type, name); if (!id) { return nullptr; } @@ -147,7 +148,7 @@ struct DeribitInstrumentStore::PImpl { continue; } const SecurityID id{ - sec_id_store_->get_or_add(VENUE, di.instrument_name)}; + sec_id_store_->get_or_add(VENUE, sec.type(), di.instrument_name)}; sec.security_common.id = id; auto [it, is_new] = securities_.try_emplace(id, std::move(sec)); if (!is_new) { @@ -188,8 +189,8 @@ struct DeribitInstrumentStore::PImpl { // DeribitCombo object, which is required to resolve the leg definitions. unordered_map unresolved_combos_; - const Security *find_security_(const string &name) { - const auto id = sec_id_store_->get_id(VENUE, name); + const Security *find_security_(SecurityType type, const string &name) { + const auto id = sec_id_store_->get_id(VENUE, type, name); if (!id) { return nullptr; } @@ -202,7 +203,8 @@ struct DeribitInstrumentStore::PImpl { bool resolve_combo_(const DeribitCombo &combo, SpreadSecurity &spd) { for (const auto &leg : combo.legs) { - const auto *leg_sec = find_security_(leg.instrument_name); + const auto *leg_sec = + find_security_(SEC_TYPE(SPREAD), leg.instrument_name); if (leg_sec == nullptr) { spd.legs.clear(); return false; @@ -259,7 +261,8 @@ struct DeribitInstrumentStore::PImpl { continue; } } - SecurityID id{sec_id_store_->get_or_add(VENUE, sec.desc())}; + SecurityID id{ + sec_id_store_->get_or_add(VENUE, sec.type(), sec.desc())}; sec.security_common.id = id; auto [sec_it, is_new] = securities_.try_emplace(id, std::move(sec)); if (!is_new) { @@ -292,8 +295,8 @@ const Security *DeribitInstrumentStore::find(SecurityID id) const { return me_->find(id); } -Security *DeribitInstrumentStore::find(const string &id) { - return me_->find(id); +Security *DeribitInstrumentStore::find(SecurityType type, const string &name) { + return me_->find(type, name); } void DeribitInstrumentStore::on_instruments( diff --git a/src/server/sparkd/md/deribit/DeribitInstrumentStore.h b/src/server/sparkd/md/deribit/DeribitInstrumentStore.h index 3db3c8e26..34295c9d4 100644 --- a/src/server/sparkd/md/deribit/DeribitInstrumentStore.h +++ b/src/server/sparkd/md/deribit/DeribitInstrumentStore.h @@ -45,11 +45,12 @@ class DeribitInstrumentStore { const domain::Security *find(domain::SecurityID id) const; /** - * @brief Finds the security with the specified \p name + * @brief Finds the security with the specified \p name and \p type * @details No 'const' overload bc the fn that converts name to numeric id * isn't const */ - domain::Security *find(const std::string &name); + domain::Security *find( + api::domain::SecurityType type, const std::string &name); void on_instruments( const std::vector &instruments, diff --git a/src/server/sparkd/md/deribit/DeribitSubscription.cpp b/src/server/sparkd/md/deribit/DeribitSubscription.cpp index 5269d9681..9243f6408 100644 --- a/src/server/sparkd/md/deribit/DeribitSubscription.cpp +++ b/src/server/sparkd/md/deribit/DeribitSubscription.cpp @@ -41,6 +41,7 @@ namespace bts::md::deribit { +using bts::api::domain::SecurityType; using bts::api::md::DeribitMarketDataConfig; using bts::domain::Security; using bts::domain::SecurityCriteria; @@ -58,6 +59,7 @@ using std::unordered_map; using std::unordered_set; using std::vector; +#define VENUE bts::api::domain::VENUE_DERIBIT #define VENUE_NAME \ (bts::api::domain::Venue_Name(bts::api::domain::VENUE_DERIBIT)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_DERIBIT @@ -166,8 +168,9 @@ struct DeribitSubscription::PImpl { subscribe_instrument_state(); } - SecurityData &add_security_data(const string sec_desc) { - const SecurityID sid{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + SecurityData &add_security_data( + SecurityType sec_type, const string &sec_desc) { + const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = security_data_map.try_emplace( sec_desc, std::make_unique(sid, listener)); _ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc); @@ -199,27 +202,31 @@ struct DeribitSubscription::PImpl { void subscribe_md(SecurityNames security_names) { // Populate list of 'channels', and prepare their corresponding handlers vector channels; - for (const string &sec : security_names) { - auto &sec_data = add_security_data(sec); + for (auto [type, name] : security_names) { + auto &sec_data = add_security_data(type, name); // Depth if (subscribe_depth) { const auto &ch = channels.emplace_back( - FMT("book.{}.none.{}.100ms", sec, cfg.num_depth_levels())); + FMT("book.{}.none.{}.100ms", name, cfg.num_depth_levels())); channel_demuxer.add_handler(ch) - .bind<&DeribitDepthAdaptor::on_depths>(sec_data.depth_hnd); + .template bind<&DeribitDepthAdaptor::on_depths>( + sec_data.depth_hnd); } // Ticker if (subscribe_ticker) { const auto &ch = - channels.emplace_back(FMT("incremental_ticker.{}", sec)); + channels.emplace_back(FMT("incremental_ticker.{}", name)); channel_demuxer.add_handler(ch) - .bind<&DeribitTickerAdaptor::on_tickers>(sec_data.ticker_hnd); + .template bind<&DeribitTickerAdaptor::on_tickers>( + sec_data.ticker_hnd); } // Trades if (subscribe_trades) { - const auto &ch = channels.emplace_back(FMT("trades.{}.100ms", sec)); + const auto &ch = + channels.emplace_back(FMT("trades.{}.100ms", name)); channel_demuxer.add_handler(ch) - .bind<&DeribitTradeAdaptor::on_trades>(sec_data.trade_hnd); + .template bind<&DeribitTradeAdaptor::on_trades>( + sec_data.trade_hnd); } } // If 'channels' is empty, it's a no-op @@ -253,7 +260,9 @@ struct DeribitSubscription::PImpl { bts_assert(!CONTAINS_KEY(security_data_map, sec->desc())); } listener->on_securities(secs); - auto sec_names = TRANSFORM(secs, [](const auto *s) { return s->desc(); }); + auto sec_names = TRANSFORM(secs, [](const auto *s) { + return std::make_pair(s->type(), s->desc()); + }); batch_subscribe_md(sec_names); } diff --git a/src/server/sparkd/md/emumba/EmumbaDeribitData.h b/src/server/sparkd/md/emumba/EmumbaDeribitData.h index 81295af80..c35d44075 100644 --- a/src/server/sparkd/md/emumba/EmumbaDeribitData.h +++ b/src/server/sparkd/md/emumba/EmumbaDeribitData.h @@ -18,10 +18,11 @@ struct EmumbaDeribitData { EmumbaDeribitData() : sec_index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(sec_index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(sec_index.get_key(k.value())); + }, + &utils::Dispatcher::current()), md_producer(recorder), listener(sec_index, md_producer) {} }; } // namespace bts::md diff --git a/src/server/sparkd/md/ftx/FTXSecurityRequestor.cpp b/src/server/sparkd/md/ftx/FTXSecurityRequestor.cpp index 4ddd4dfd7..cb721864f 100644 --- a/src/server/sparkd/md/ftx/FTXSecurityRequestor.cpp +++ b/src/server/sparkd/md/ftx/FTXSecurityRequestor.cpp @@ -35,6 +35,7 @@ using std::string; using std::unordered_set; using std::vector; +#define VENUE Venue::VENUE_FTX #define VENUE_NAME (Venue_Name(Venue::VENUE_FTX)) #define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x @@ -172,7 +173,8 @@ struct FTXSecurityRequestor::PImpl { for (auto &sec : added_secs) { // Since the ids haven't been set yet, we should expect them to be 0 bts_assert(sec.id().value() == 0); - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + SecurityID sec_id{ + sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())}; sec.security_common.id = sec_id; } } diff --git a/src/server/sparkd/md/ftx/FTXSubscription.cpp b/src/server/sparkd/md/ftx/FTXSubscription.cpp index 1ac286e26..3f6fd41f0 100644 --- a/src/server/sparkd/md/ftx/FTXSubscription.cpp +++ b/src/server/sparkd/md/ftx/FTXSubscription.cpp @@ -28,6 +28,7 @@ namespace bts::md::ftx { +using bts::api::domain::SecurityType; using bts::api::domain::Venue; using bts::api::domain::Venue_Name; using bts::api::md::FTXMarketDataConfig; @@ -36,11 +37,13 @@ using bts::domain::SecurityCriteria; using bts::domain::SecurityID; using bts::io::ProtoPrinter; using bts::persistence::IDGenerator; +using std::pair; using std::string; using std::unique_ptr; using std::unordered_map; using std::vector; +#define VENUE Venue::VENUE_FTX #define VENUE_NAME (Venue_Name(Venue::VENUE_FTX)) #define FTX_MSG_TYPE_NAME(x) \ @@ -130,16 +133,18 @@ struct FTXSubscription::PImpl { sec_requestor.request(); } - DepthHandler &create_depth_handler(string sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + DepthHandler &create_depth_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = depth_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert(is_new); return it->second; } - TickerHandler &create_ticker_handler(string sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + TickerHandler &create_ticker_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = ticker_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -147,8 +152,9 @@ struct FTXSubscription::PImpl { return it->second; } - StatsHandler &create_stats_handler(string sec_desc) { - SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + StatsHandler &create_stats_handler( + SecurityType sec_type, const string &sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = stats_handlers.try_emplace(sec_desc, sec_id, listener); bts_assert_msgf( @@ -171,15 +177,16 @@ struct FTXSubscription::PImpl { CCApiSession::BatchSubscription ticker_subs; CCApiSession::BatchRequest stats_reqs; - for (const auto &sec_desc : securities) { + for (const auto &[sec_type, sec_desc] : securities) { depth_subs.push_back( {create_depth_sub(sec_desc, depth_opts), - &create_depth_handler(sec_desc)}); + &create_depth_handler(sec_type, sec_desc)}); ticker_subs.push_back( {create_ticker_sub(sec_desc, ticker_opts), - &create_ticker_handler(sec_desc)}); + &create_ticker_handler(sec_type, sec_desc)}); stats_reqs.push_back( - {create_stats_req(sec_desc), &create_stats_handler(sec_desc)}); + {create_stats_req(sec_desc), + &create_stats_handler(sec_type, sec_desc)}); } session.subscribe( @@ -196,7 +203,7 @@ struct FTXSubscription::PImpl { */ void on_securities(vector secs) { // 'sec_descs' is used to make depth subscriptions - vector sec_descs; + vector> sec_descs; sec_descs.reserve(secs.size()); // 'sec_ptrs' is passed to IMarketDataListeners vector sec_ptrs; @@ -208,7 +215,7 @@ struct FTXSubscription::PImpl { auto [it, is_new] = securities.try_emplace( id, std::make_unique(std::move(s))); bts_assert_msgf(is_new, "Received duplicate FTX security '{}'", desc); - sec_descs.push_back(std::move(desc)); + sec_descs.push_back(std::make_pair(s.type(), std::move(desc))); // add ptr to entry in 'securities' (not the loop var, we moved that) sec_ptrs.push_back(it->second.get()); } diff --git a/src/server/sparkd/md/okx/OkxInstrumentStore.cpp b/src/server/sparkd/md/okx/OkxInstrumentStore.cpp index 38d55b160..0d5aea201 100644 --- a/src/server/sparkd/md/okx/OkxInstrumentStore.cpp +++ b/src/server/sparkd/md/okx/OkxInstrumentStore.cpp @@ -14,6 +14,7 @@ namespace bts::md::okx { using bts::api::domain::CallPut; +using bts::api::domain::SecurityType; using bts::domain::FutureSecurity; using bts::domain::OptionSecurity; using bts::domain::Price; @@ -94,8 +95,8 @@ struct OkxInstrumentStore::PImpl { return &it->second; } - Security *find(const string &name) { - auto id = sec_id_store_->get_id(VENUE, name); + Security *find(SecurityType type, const string &name) { + auto id = sec_id_store_->get_id(VENUE, type, name); if (!id) { return nullptr; } @@ -124,7 +125,7 @@ struct OkxInstrumentStore::PImpl { if (!filter_(sec)) { continue; } - SecurityID id{sec_id_store_->get_or_add(VENUE, oi.instId)}; + SecurityID id{sec_id_store_->get_or_add(VENUE, sec.type(), oi.instId)}; sec.security_common.id = id; auto [it, is_new] = securities_.try_emplace(id, std::move(sec)); if (!is_new) { @@ -152,7 +153,9 @@ const Security *OkxInstrumentStore::find(SecurityID id) const { return me_->find(id); } -Security *OkxInstrumentStore::find(const string &id) { return me_->find(id); } +Security *OkxInstrumentStore::find(SecurityType type, const string &name) { + return me_->find(type, name); +} void OkxInstrumentStore::on_instruments( const vector &instruments, diff --git a/src/server/sparkd/md/okx/OkxInstrumentStore.h b/src/server/sparkd/md/okx/OkxInstrumentStore.h index f3c8a3e0e..00a19be78 100644 --- a/src/server/sparkd/md/okx/OkxInstrumentStore.h +++ b/src/server/sparkd/md/okx/OkxInstrumentStore.h @@ -50,7 +50,8 @@ class OkxInstrumentStore { * @details No 'const' overload bc the fn that converts name to numeric id * isn't const */ - domain::Security *find(const std::string &name); + domain::Security *find( + api::domain::SecurityType type, const std::string &name); void on_instruments( const std::vector &instruments, diff --git a/src/server/sparkd/md/okx/OkxSubscription.cpp b/src/server/sparkd/md/okx/OkxSubscription.cpp index c88fd6db4..f2fc1306b 100644 --- a/src/server/sparkd/md/okx/OkxSubscription.cpp +++ b/src/server/sparkd/md/okx/OkxSubscription.cpp @@ -38,6 +38,7 @@ namespace bts::md::okx { +using bts::api::domain::SecurityType; using bts::api::md::OkxMarketDataConfig; using bts::domain::Security; using bts::domain::SecurityCriteria; @@ -60,6 +61,7 @@ using std::vector; #define _WARN(...) _LOG(Warn, __VA_ARGS__) #define _ERROR(...) _LOG(Error, __VA_ARGS__) +#define VENUE bts::api::domain::VENUE_OKX #define VENUE_NAME (bts::api::domain::Venue_Name(bts::api::domain::VENUE_OKX)) #define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_OKX #define SEC_TYPE(t) bts::api::domain::SECURITY_TYPE_##t @@ -164,8 +166,9 @@ struct OkxSubscription::PImpl { [this](const auto &secs) { on_securities(secs); }); } - SecurityData &add_security_data(const string sec_desc) { - const SecurityID sid{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + SecurityData &add_security_data( + SecurityType sec_type, const string &sec_desc) { + const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)}; auto [it, is_new] = security_data_map.try_emplace( sec_desc, std::make_unique(sid, listener)); _ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc); @@ -182,13 +185,13 @@ struct OkxSubscription::PImpl { void subscribe_md(SecurityNames security_names) { // Populate list of 'channels', and prepare their corresponding handlers vector channels; - for (const string &sec : security_names) { - auto &sec_data = add_security_data(sec); + for (const auto &[type, name] : security_names) { + auto &sec_data = add_security_data(type, name); // Depth if (subscribe_depth) { auto &ch = channels.emplace_back(); ch.channel = "books5"; - ch.instId = sec; + ch.instId = name; channel_demuxer.add_handler(ch).bind<&OkxDepthAdaptor::on_depths>( sec_data.depth_hnd); } @@ -196,7 +199,7 @@ struct OkxSubscription::PImpl { if (subscribe_ticker) { auto &ch = channels.emplace_back(); ch.channel = "tickers"; - ch.instId = sec; + ch.instId = name; channel_demuxer.add_handler(ch).bind<&OkxTickerAdaptor::on_tickers>( sec_data.ticker_hnd); } @@ -204,7 +207,7 @@ struct OkxSubscription::PImpl { if (subscribe_trades) { auto &ch = channels.emplace_back(); ch.channel = "trades"; - ch.instId = sec; + ch.instId = name; channel_demuxer.add_handler(ch).bind<&OkxTradeAdaptor::on_trades>( sec_data.trade_hnd); } @@ -240,7 +243,9 @@ struct OkxSubscription::PImpl { bts_assert(!CONTAINS_KEY(security_data_map, sec->desc())); } listener->on_securities(secs); - auto sec_names = TRANSFORM(secs, [](const auto *s) { return s->desc(); }); + auto sec_names = TRANSFORM(secs, [](const auto *s) { + return std::make_pair(s->type(), s->desc()); + }); batch_subscribe_md(sec_names); } }; diff --git a/src/server/sparkd/md/platforms/emumba/EmMDListener.cpp b/src/server/sparkd/md/platforms/emumba/EmMDListener.cpp index 35e6a42b2..1cdad5c88 100644 --- a/src/server/sparkd/md/platforms/emumba/EmMDListener.cpp +++ b/src/server/sparkd/md/platforms/emumba/EmMDListener.cpp @@ -59,6 +59,10 @@ using namespace bts::orderman; namespace bts::md { +#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x +#define TRADING_STATUS(x) bts::co2::TRADINGSTATUS_##x +#define VENUE bts::api::domain::VENUE_DERIBIT + std::atomic EmMDListener::_order_mux = nullptr; using MDArray = std::array; using EMOrderKey = uint64_t; @@ -1144,16 +1148,32 @@ DescSearchResult EmMDListener::_find_em_sec( } } +static api::domain::SecurityType convert_sec_type( + emumba::market_data::security_type emumba_sec_type) { + switch (emumba_sec_type) { + case emumba::market_data::security_type::index: + return SEC_TYPE(SPOT); + case emumba::market_data::security_type::future: + return SEC_TYPE(FUTURE); + case emumba::market_data::security_type::spread: + return SEC_TYPE(SPREAD); + case emumba::market_data::security_type::option: + return SEC_TYPE(OPTION); + default: + bts_assert_msg(false, "Unknown sec type"); + return SEC_TYPE(UNSPECIFIED); + } +} + security_key EmMDListener::_add_sec_entry( const emumba::market_data::security &sec) { const std::string &desc = extract_security_desc(sec); // id_gen either adds a key and returns a new id, or finds one and returns // the previously added id bts_assert(me->sec_emdesc_map.find(desc) == me->sec_emdesc_map.end()); - auto new_key_opt = me->id_gen.get_id("exchange_key", desc, true); - bts_assert(new_key_opt.has_value()); + const security_key new_key = static_cast( + me->id_gen.get_or_add(VENUE, convert_sec_type(sec.sec_type()), desc)); // TODO (64 -> 32 conversion) - security_key new_key = new_key_opt.value(); me->sec_emdesc_map.insert({desc, {new_key, sec}}); bts_assert(me->sec_btsid_map.find(new_key) == me->sec_btsid_map.end()); me->sec_btsid_map.insert({new_key, desc}); diff --git a/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp b/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp index 748a89251..70e8e0eee 100644 --- a/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp +++ b/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp @@ -43,10 +43,11 @@ struct EmumbaMain::PImpl { PImpl() : sec_index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(sec_index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(sec_index.get_key(k.value())); + }, + &utils::Dispatcher::current()), producer(recorder), listener(sec_index, producer) { const auto config_path = Path{config::get().config_path()} / diff --git a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp index 4bcd0c9d4..0dc491f5d 100644 --- a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp +++ b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp @@ -1,70 +1,66 @@ #include "ICEMain.h" #include "lib/config/config.h" -#include "lib/md/SecurityDB.h" -#include "lib/md/coconut2.h" -#include "lib/md/ice/ICEUtils.h" -#include "lib/md/protos/co2.pb.h" -#include "lib/md/vendor/OnixsUtils.h" -#include "lib/os/os.h" -#include "lib/utils/Dispatcher.h" -#include "lib/utils/bts_chrono.h" -#include "lib/utils/helper_macros.h" -#include "lib/utils/proto_helper.h" - +#include "lib/config/protos/deployment_type.pb.h" #include "lib/config/protos/market_data.pb.h" #include "lib/config/protos/system_config.pb.h" -#include "lib/utils/thread_manager.h" +#include "lib/md/ice/ICEUtils.h" +#include "lib/md/vendor/OnixsUtils.h" +#include #include #include -#include - -void recursive_copy( - const boost::filesystem::path &src, const boost::filesystem::path &dst) { - if (boost::filesystem::exists(dst)) - throw std::runtime_error(dst.generic_string() + " exists"); - if (boost::filesystem::is_directory(src)) { - boost::filesystem::create_directories(dst); - for (boost::filesystem::directory_entry &item : - boost::filesystem::directory_iterator(src)) - recursive_copy(item.path(), dst / item.path().filename()); - } else if (boost::filesystem::is_regular_file(src)) - boost::filesystem::copy(src, dst); - else - throw std::runtime_error(dst.generic_string() + " not dir or file"); -} using namespace std; using namespace bts::co2; using std::size_t; -namespace bts { -namespace md { -namespace ice { +namespace bts::md::ice { using bts::config::SystemConfig; using Path = boost::filesystem::path; -ICEMain::ICEMain() - : _index(), _recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(_index.get_key(k.value())); - }}), - _producer(_recorder), _handlers(), _listeners() {} +struct ICEMain::PImpl { + ICEOptions options; + std::string config_file; + ICESecurityIndex index; + SecurityIDRecorder recorder; + MDProducer producer; + std::vector>> + handlers; + std::vector> listeners; -ICEMain::~ICEMain() {} + PImpl(ICEOptions options) + : options(options), + config_file((Path{config::get().data_path()} / + "onixs_ice" / this->options.ice_config_file()) + .string()), + index(), recorder( + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(index.get_key(k.value())); + }, + &utils::Dispatcher::current()), + producer(recorder), handlers(), listeners() {} -const IceChannelConfig channel_settings_for(const ICEChannel &channel) { - const auto &config = config::get(); - for (const auto &ch : config.channel()) { - if (ch.name() == channel.name) - return ch; + IceChannelConfig channel_settings_for(const ICEChannel &channel) { + for (const auto &ch : options.channel()) { + if (ch.name() == channel.name) + return ch; + } + return {}; } - return {}; +}; + +ICEMain::ICEMain() : ICEMain(config::get()) {} + +ICEMain::ICEMain(ICEOptions options) + : me_(std::make_unique(std::move(options))) { + me_->options = std::move(options); } +ICEMain::~ICEMain() = default; + onixs_ice::MarketSubscriptions market_subcription_for_channel( const ICEChannel &chan) { onixs_ice::MarketSubscriptions subs; @@ -90,23 +86,30 @@ onixs_ice::MarketSubscriptions market_subcription_for_channel( return subs; } -void ICEMain::start(const std::vector &symbols) { - auto ice_conf_location = - Path{config::get().data_path()} / "onixs_ice/"; - if (!boost::filesystem::is_directory(ice_conf_location)) { - auto conf_path = Path{os::sparkd_wd()} / "data_files" / "onixs_ice/"; - recursive_copy(Path{conf_path}, ice_conf_location); +static void validate_config_file(const std::string &config_file) { + if (!boost::filesystem::exists(config_file)) { + const std::string config_url = + config::get().type() == "dev" + ? "https://community.theice.com/docs/DOC-19989" + : "https://drive.google.com/" + "open?id=13_eyF2pKtn3F7chgsa_75ftUdf7emMbp"; + bts_assert_msgf( + false, "Ice config file not found '{}'. Please download from {}", + config_file, config_url); } +} +void ICEMain::start(const std::vector &symbols) { await_ice_config(); + validate_config_file(me_->config_file); auto license = onixs_create_temporary_license(); const auto &channels = parse_ice_channels(symbols); for (const auto &channel : channels) { unique_ptr phandler = prepare_handler(channel, license.path.string()); - _listeners.push_back(prepare_listener(*phandler, channel)); - _handlers.push_back(std::make_pair(channel, std::move(phandler))); + me_->listeners.push_back(prepare_listener(*phandler, channel)); + me_->handlers.push_back(std::make_pair(channel, std::move(phandler))); std::stringstream symstr; for (const auto &sym : channel.symbols) @@ -115,7 +118,7 @@ void ICEMain::start(const std::vector &symbols) { symstr.str()); } - for (auto &pair : _handlers) { + for (auto &pair : me_->handlers) { LOG(md, Info, "ICE: Init Handler {}", pair.first.to_string()); sleep_for(16s); onixs_ice::MarketSubscriptions subs = @@ -132,7 +135,7 @@ void ICEMain::start(const std::vector &symbols) { } void ICEMain::send_heartbeats() { - for (const auto &listener : _listeners) + for (const auto &listener : me_->listeners) listener->heartbeat(); } @@ -140,20 +143,18 @@ void ICEMain::send_heartbeats() { // Initialize HandlerSettings and allocate the channel Handler unique_ptr ICEMain::prepare_handler( const ICEChannel &channel, const std::string &license) { - const auto &options = channel_settings_for(channel); + const auto &options = me_->channel_settings_for(channel); LOG(md, Info, "Loaded channel options for channel {}:\n{}", channel.to_string(), options.DebugString()); onixs_ice::HandlerSettings settings; settings.licenseDirectory = license; - // - // Logging settings.logLevel = (onixs_ice::LogLevels::Enum)options.onixs_log_mode(); - settings.logDirectory = ice_log_path(); - - // - // Config file - settings.connectivityConfiguration = ice_config_path(); + settings.logDirectory = + (boost::filesystem::path{config::get().data_path()} / + "logging" / "onixs_ice/") + .string(); + settings.connectivityConfiguration = me_->config_file; // // Network settings @@ -231,12 +232,12 @@ unique_ptr ICEMain::prepare_listener( // heartbeats vector producer_channels; for (auto &symbol : channel.symbols) { - producer_channels.push_back(_producer.get_channel( + producer_channels.push_back(me_->producer.get_channel( symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } unique_ptr listener(new ICEListener( - _index, _producer, /*_settlement_recorder,*/ producer_channels)); + me_->index, me_->producer, /*_settlement_recorder,*/ producer_channels)); for (auto &symbol : channel.symbols) listener->add_filter_symbol(symbol.symbol()); @@ -254,6 +255,4 @@ unique_ptr ICEMain::prepare_listener( return listener; } -} // namespace ice -} // namespace md -} // namespace bts +} // namespace bts::md::ice diff --git a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h index 1ea217a39..8c37f9476 100644 --- a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h +++ b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h @@ -1,21 +1,12 @@ #pragma once #include "ICEListener.h" -#include "ICESecurityIndex.h" -#include "lib/md/coconut2.h" -#include "lib/utils/bts_no_copy.h" -#include "onixs.h" #include "server/sparkd/md/platforms/IMDPlatform.h" -#include #include #include -namespace bts { -namespace md { -namespace ice { - -// void run_onixs_ice_server(UserSymbol* sym, size_t symcount); +namespace bts::md::ice { struct ICEChannel; @@ -28,27 +19,22 @@ struct ICEChannel; class ICEMain : public IMDPlatform { public: ICEMain(); + explicit ICEMain(ICEOptions options); + ICEMain(const ICEMain &) = delete; + ICEMain &operator=(const ICEMain &) = delete; ~ICEMain(); void start(const std::vector &symbols); void send_heartbeats() override; - std::unique_ptr - prepare_handler(const ICEChannel &channel, const std::string &license); - std::unique_ptr - prepare_listener(onixs_ice::Handler &handler, const ICEChannel &channel); + std::unique_ptr prepare_handler( + const ICEChannel &channel, const std::string &license); + std::unique_ptr prepare_listener( + onixs_ice::Handler &handler, const ICEChannel &channel); private: - BTS_NO_COPY(ICEMain); - - ICESecurityIndex _index; - SecurityIDRecorder _recorder; - MDProducer _producer; - std::vector>> - _handlers; - std::vector> _listeners; + struct PImpl; + std::unique_ptr me_; }; -} // namespace ice -} // namespace md -} // namespace bts +} // namespace bts::md::ice diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 9ad5c53ba..0a54040ad 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -1,24 +1,18 @@ #include "MDP3Main.h" - +#include "lib/api/md/GlbxMarketDataConfig.pb.h" #include "lib/config/config.h" +#include "lib/config/protos/deployment_type.pb.h" #include "lib/logging/logging.h" #include "lib/md/MBOProcessor.h" #include "lib/md/SecurityDB.h" #include "lib/md/cme/MDP3Utils.h" -#include "lib/md/coconut2.h" -#include "lib/md/protos/co2.pb.h" +#include "lib/md/cme/mdp3_data_directory.h" #include "lib/md/vendor/OnixsUtils.h" -#include "lib/os/os.h" -#include "lib/utils/Dispatcher.h" -#include "lib/utils/proto_helper.h" - -#include "lib/config/protos/market_data.pb.h" -#include "lib/config/protos/system_config.pb.h" +#include "lib/os/spark_data_directory.h" #include "lib/utils/thread_manager.h" #include #include -#include #include #include @@ -29,16 +23,33 @@ using std::size_t; namespace bts { namespace md { -MDP3Main::MDP3Main() - : _index(), _recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(_index.get_key(k.value())); - }}), - _producer(_recorder), _engines(), _handlers(), _listeners(), _recovery() { -} +#define TRADING_STATUS(x) bts::domain::protobufs::TRADINGSTATUS_##x -MDP3Main::~MDP3Main() {} +using Path = boost::filesystem::path; + +struct MDP3Main::PImpl { + Mdp3DataDirectory data_dir; + CMESecurityIndex index; + SecurityIDRecorder recorder; + MDProducer producer; + std::map> engines; + std::map> handlers; + std::vector> listeners; + std::vector> recovery; + PImpl() + : index(), recorder( + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(index.get_key(k.value())); + }, + &utils::Dispatcher::current()), + producer(recorder), engines(), handlers(), listeners(), recovery() {} +}; + +MDP3Main::MDP3Main() + : me_(std::make_unique()) {} + +MDP3Main::~MDP3Main() = default; const CmeChannelConfig channel_settings_for(const string &id) { const auto &config = config::get(); @@ -173,11 +184,11 @@ void MDP3Main::start( auto license = onixs_create_temporary_license(); initialize(license.path.string()); const auto &channels = parse_channels(symbols); - _engines = create_feed_engines(channels); + me_->engines = create_feed_engines(channels); for (const auto &channel : channels) { unique_ptr phandler = prepare_handler(channel.id, mbo); - _listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd)); - _handlers.insert(std::make_pair(channel.id, std::move(phandler))); + me_->listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd)); + me_->handlers.insert(std::make_pair(channel.id, std::move(phandler))); std::stringstream symstr; for (const auto &sym : channel.symbols) { symstr << sym.symbol() << "-" << SecurityType_Name(sym.type()) << " "; @@ -186,7 +197,7 @@ void MDP3Main::start( symstr.str()); } - for (auto &pair : _handlers) { + for (auto &pair : me_->handlers) { onixs::SessionSettings settings = session_settings_for_channel(pair.first, mbo); pair.second->start(settings); @@ -196,7 +207,7 @@ void MDP3Main::start( } void MDP3Main::send_heartbeats() { - for (const auto &listener : _listeners) { + for (const auto &listener : me_->listeners) { listener->heartbeat(); } } @@ -219,7 +230,8 @@ unique_ptr MDP3Main::prepare_handler( onixs::HandlerSettings settings; settings.channel(std::stoul(channel_id)); - settings.feeds().connectivityConfigurationFile(config_path()); + settings.feeds().connectivityConfigurationFile( + (me_->data_dir.config_file()).string()); // Market data options settings.bookManagement().directBooks().maintain(true); @@ -306,7 +318,8 @@ unique_ptr MDP3Main::prepare_handler( logger = new onixs::NullLogger(); } else { onixs::FileLoggerSettings loggerSettings; - loggerSettings.filename(log_path() + "/log_" + channel_id); + loggerSettings.filename( + (me_->data_dir.log_dir() / ("log_" + channel_id)).string()); loggerSettings.severityLevel( (onixs::LogSeverity::Enum)options.onixs_log_mode()); logger = new onixs::FileLogger(loggerSettings); @@ -338,19 +351,19 @@ unique_ptr MDP3Main::prepare_handler( try { auto handler = unique_ptr(new onixs::Handler(settings)); - auto iter = _engines.find(options.feed_engine()); - if (iter == _engines.end()) { + auto iter = me_->engines.find(options.feed_engine()); + if (iter == me_->engines.end()) { LOG(md, Fatal, "Missing feed engine definition '{}' for channel id '{}'. Add a " "feed_engine entry with name:'{}' to the CMEOptions config key.", options.feed_engine(), channel_id, options.feed_engine()); bts_assert_msg( - iter != _engines.end(), "Missing feed engine definition."); + iter != me_->engines.end(), "Missing feed engine definition."); } handler->bindFeedEngine(*(iter->second)); if (recovery) { handler->bindTcpRecovery(*recovery); - _recovery.push_back(std::move(recovery)); + me_->recovery.push_back(std::move(recovery)); } if (logger) { handler->bindLogger(*logger); @@ -375,12 +388,13 @@ unique_ptr MDP3Main::prepare_listener( // heartbeats vector producer_channels; for (auto &symbol : channel.symbols) { - producer_channels.push_back(_producer.get_channel( + producer_channels.push_back(me_->producer.get_channel( symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } unique_ptr listener(new MDP3Listener( - _index, _producer, mbo, producer_channels, trade_summary_dispatcher)); + me_->index, me_->producer, mbo, producer_channels, + trade_summary_dispatcher)); for (auto &symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index c67dc93d4..fc3043ac3 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -29,7 +29,9 @@ struct CMEChannel; */ class MDP3Main : public IMDPlatform { public: - MDP3Main(); + explicit MDP3Main(); + MDP3Main(const MDP3Main &) = delete; + MDP3Main &operator=(const MDP3Main &) = delete; ~MDP3Main(); void start( @@ -39,8 +41,8 @@ class MDP3Main : public IMDPlatform { void send_heartbeats() override; void initialize(const std::string &license_store); - std::unique_ptr - prepare_handler(const std::string &channel, MBOWriter *); + std::unique_ptr prepare_handler( + const std::string &channel, MBOWriter *); std::unique_ptr prepare_listener( onixs::Handler &handler, const CMEChannel &channel, @@ -48,16 +50,8 @@ class MDP3Main : public IMDPlatform { TradeSummarySignalDispatcher *); private: - BTS_NO_COPY(MDP3Main); - - CMESecurityIndex _index; - SecurityIDRecorder _recorder; - MDProducer _producer; - // co2::SettlementRecorder _settlement_recorder; - std::map> _engines; - std::map> _handlers; - std::vector> _listeners; - std::vector> _recovery; + struct PImpl; + std::unique_ptr me_; }; } // namespace md diff --git a/src/server/sparkd/md/taifex/TaifexDataProcessors.h b/src/server/sparkd/md/taifex/TaifexDataProcessors.h index c0b80dd22..106b8aafa 100644 --- a/src/server/sparkd/md/taifex/TaifexDataProcessors.h +++ b/src/server/sparkd/md/taifex/TaifexDataProcessors.h @@ -201,8 +201,9 @@ class TaifexDataProcessors { const auto id = TaifexUtils::remove_spaces(msg.prod_id); const auto symbol = id.substr(0, 3); + const auto sec_type = convert_sec_type(msg.parsedProductInfo.productType); bts::domain::SecurityID sec_id{ - sec_id_store->get_or_add(api::domain::VENUE_TAIFEX, id)}; + sec_id_store->get_or_add(api::domain::VENUE_TAIFEX, sec_type, id)}; auto check = security_details->add_i010_mapping(id, sec_id, msg.decimal_locator); if (!check) { @@ -286,6 +287,20 @@ class TaifexDataProcessors { security_details->add_i011_mapping( id, 1.0, msg.getCurrencyString(), msg.contract_size); } + + private: + static api::domain::SecurityType convert_sec_type( + TaifexMessages::I010Data::ProductInfo::ProductType type) { + switch (type) { + case TaifexMessages::I010Data::ProductInfo::ProductType::FUTURES: + return api::domain::SecurityType::SECURITY_TYPE_FUTURE; + case TaifexMessages::I010Data::ProductInfo::ProductType::OPTIONS: + return api::domain::SecurityType::SECURITY_TYPE_OPTION; + default: + bts_assert_msg( + false, "Unexpected security type (must be FUTURES or OPTIONS)"); + } + } }; -} //namespace bts::md::taifex \ No newline at end of file +} //namespace bts::md::taifex diff --git a/src/server/sparkd/position/PositionServiceServer.cpp b/src/server/sparkd/position/PositionServiceServer.cpp index c3e511409..26c95f74f 100644 --- a/src/server/sparkd/position/PositionServiceServer.cpp +++ b/src/server/sparkd/position/PositionServiceServer.cpp @@ -293,9 +293,9 @@ void PositionServiceServer::update_manual_fill( auto *this_resp = resp.add_response(); auto id = std::to_string(mf.record_id()); - auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id); - auto found = p.first; - if (!found) { + auto [exists, rowid, val] = + me->manual_fills_store->retrieve_one(manual_fills_ns, id); + if (!exists) { this_resp->set_ok(false); this_resp->set_msg("Could not find record"); } else { @@ -326,9 +326,9 @@ void PositionServiceServer::remove_manual_fill( auto *this_resp = resp.add_response(); auto id = std::to_string(mf.record_id()); - auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id); - auto found = p.first; - if (!found) { + auto [exists, rowid, val] = + me->manual_fills_store->retrieve_one(manual_fills_ns, id); + if (!exists) { this_resp->set_ok(false); this_resp->set_msg("Could not find record"); } else { @@ -367,10 +367,9 @@ void PositionServiceServer::update_security_parameters( req.op_params_size()); for (const auto &p : req.op_params()) { std::string key_str = std::to_string(p.security_key()); - auto db_p = + auto [exists, rowid, val] = me->security_params_store->retrieve_one(security_params_ns, key_str); - auto found = db_p.first; - if (!found) { + if (!exists) { me->security_params_store->insert( security_params_ns, key_str, utils::pb::to_string(p)); } else { diff --git a/src/server/sparkd/user/AdminServiceDB.cpp b/src/server/sparkd/user/AdminServiceDB.cpp index 755ff079d..c89407e0e 100644 --- a/src/server/sparkd/user/AdminServiceDB.cpp +++ b/src/server/sparkd/user/AdminServiceDB.cpp @@ -21,7 +21,8 @@ AdminServiceDB::AdminServiceDB() : me(new PImpl()) {} AdminServiceDB::~AdminServiceDB() {} std::optional AdminServiceDB::retrieve() { - auto [has_table, data] = me->store.retrieve_one(table_name, unique_key); + auto [has_table, rowid, data] = + me->store.retrieve_one(table_name, unique_key); if (!has_table) return {}; @@ -33,7 +34,8 @@ std::optional AdminServiceDB::retrieve() { } bool AdminServiceDB::update(api::admin::AdminData const &state) { - auto [has_table, data] = me->store.retrieve_one(table_name, unique_key); + auto [has_table, rowid, data] = + me->store.retrieve_one(table_name, unique_key); UNUSED(data); std::string ser_state;