From ccc71c5a657dd519aa39e9d2bd1e4d2a0e16ea2c Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 14 Oct 2024 03:33:36 +0000 Subject: [PATCH] Put cme config url in GlbxMarketDataConfig proto Instead of using magic strings --- .../sparkd/cmds/md/cmds/Channels.cpp | 29 +++++++- .../sparkd/md/glbx/OnixsGlbxSubscription.cpp | 2 +- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 21 ++++-- .../sparkd/md/platforms/mdp3_onixs/MDP3Main.h | 2 +- .../spark/api/md/GlbxMarketDataConfig.proto | 2 + .../profiles/md_dev/market_data_config.conf | 5 ++ .../profiles/md_prod/market_data_config.conf | 5 ++ src/lib_spark/spark/md/cme/MDP3Utils.cpp | 69 ++++++++++--------- src/lib_spark/spark/md/cme/MDP3Utils.h | 15 +++- .../spark/orderman/OrderServiceServer.cpp | 7 +- .../spark/orderman/OrderServiceServer.h | 21 ++++-- .../spark/orderman/actors/MDSource.h | 11 ++- 12 files changed, 133 insertions(+), 56 deletions(-) diff --git a/src/app_sparkd/sparkd/cmds/md/cmds/Channels.cpp b/src/app_sparkd/sparkd/cmds/md/cmds/Channels.cpp index d2269402d..45d1f65ed 100644 --- a/src/app_sparkd/sparkd/cmds/md/cmds/Channels.cpp +++ b/src/app_sparkd/sparkd/cmds/md/cmds/Channels.cpp @@ -1,11 +1,38 @@ #include "Channels.h" +#include "domain/protobufs/venue.pb.h" +#include "domain/security_criteria.h" +#include "spark/api/md/MarketDataConfig.pb.h" +#include "spark/config/config.h" +#include "spark/md/SymbolLoader.h" #include "spark/md/cme/MDP3Utils.h" +#include "spark/md/cme/mdp3_data_directory.h" namespace bts::cmds::md::cmds { +using bts::domain::SecurityCriteria; +using bts::md::await_cme_config; +using bts::md::Mdp3DataDirectory; +using bts::md::parse_channels; +using bts::md::SymbolLoader; + Channels::Channels() : CLI::App("List configured cme channels", "channels") { callback([] { - auto channels = bts::md::get_configured_cme_channels(); + Mdp3DataDirectory data_dir; + auto md_cfg = bts::config::get(); + if (md_cfg.glbx().empty()) { + std::cerr << "'MarketDataConfig.glbx' not defined (check spark.conf)." + << std::endl; + } + const auto cfg_file = data_dir.config_file().string(); + auto glbx_cfg = md_cfg.glbx()[0]; + const auto templates_file = data_dir.templates_file().string(); + await_cme_config( + glbx_cfg.cme_config_url(), cfg_file, glbx_cfg.cme_templates_url(), + templates_file); + const auto symbols = SymbolLoader::load( + SecurityCriteria::create({domain::protobufs::VENUE_GLBX})); + const auto channels = + parse_channels(cfg_file, {symbols.begin(), symbols.end()}); for (const auto &ch : channels) { std::cout << ch.id << " - "; std::set symbols; diff --git a/src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp b/src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp index b61bb7166..cb3b2c16a 100644 --- a/src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp +++ b/src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp @@ -122,7 +122,7 @@ void OnixsGlbxSubscription::start( auto cme_symbols = SymbolLoader::load( bts::domain::SecurityCriteria::create({domain::protobufs::VENUE_GLBX})); if (!cme_symbols.empty()) { - auto *main = new bts::md::MDP3Main(); + auto *main = new bts::md::MDP3Main(me_->config); main->start({cme_symbols.begin(), cme_symbols.end()}, mbo, tssd); platforms.emplace_back(main); } diff --git a/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 6bfe2b7ee..81d3903f2 100644 --- a/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -1,5 +1,6 @@ #include "MDP3Main.h" +#include "spark/api/md/GlbxMarketDataConfig.pb.h" #include "spark/config/config.h" #include "spark/config/protos/deployment_type.pb.h" #include "spark/config/protos/market_data.pb.h" @@ -28,6 +29,9 @@ namespace md { using Path = boost::filesystem::path; struct MDP3Main::PImpl { + PImpl(api::md::GlbxMarketDataConfig config) : config(std::move(config)) {} + + api::md::GlbxMarketDataConfig config; Mdp3DataDirectory data_dir; CMESecurityIndex index; SecurityIDRecorder recorder; @@ -46,8 +50,8 @@ struct MDP3Main::PImpl { producer(recorder), engines(), handlers(), listeners(), recovery() {} }; -MDP3Main::MDP3Main() - : me_(std::make_unique()) {} +MDP3Main::MDP3Main(api::md::GlbxMarketDataConfig config) + : me_(std::make_unique(std::move(config))) {} MDP3Main::~MDP3Main() = default; @@ -180,10 +184,19 @@ void MDP3Main::start( oss << us << " "; } LOG(md, Info, "MDP3Main::start symbols: {}", oss.str()); - await_cme_config(); + const auto cfg_file = me_->data_dir.config_file().string(); + const auto &cfg_url = me_->config.cme_config_url(); + const auto templates_file = me_->data_dir.templates_file().string(); + const auto &templates_url = me_->config.cme_templates_url(); + if (!await_cme_config(cfg_url, cfg_file, templates_url, templates_file)) { + LOG(md, Error, + "MDP3Main::start exiting early because cme config.xml and/or " + "templates.xml files are missing"); + return; + } auto license = onixs_create_temporary_license(); initialize(license.path.string()); - const auto &channels = parse_channels(symbols); + const auto &channels = parse_channels(cfg_file, symbols); me_->engines = create_feed_engines(channels); for (const auto &channel : channels) { unique_ptr phandler = prepare_handler(channel.id, mbo); diff --git a/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index 0b5b33c07..33dcec369 100644 --- a/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -29,7 +29,7 @@ struct CMEChannel; */ class MDP3Main : public IMDPlatform { public: - explicit MDP3Main(); + explicit MDP3Main(api::md::GlbxMarketDataConfig config); MDP3Main(const MDP3Main &) = delete; MDP3Main &operator=(const MDP3Main &) = delete; ~MDP3Main(); diff --git a/src/lib_spark/spark/api/md/GlbxMarketDataConfig.proto b/src/lib_spark/spark/api/md/GlbxMarketDataConfig.proto index f8e692341..46a77c28a 100644 --- a/src/lib_spark/spark/api/md/GlbxMarketDataConfig.proto +++ b/src/lib_spark/spark/api/md/GlbxMarketDataConfig.proto @@ -11,6 +11,8 @@ option csharp_namespace = "bts.api.md"; message GlbxMarketDataConfig { uint32 unique_check_value = 1; bts.domain.protobufs.SecurityCriteria security_criteria = 3; + string cme_config_url = 4; + string cme_templates_url = 5; bool use_databento = 10; DatabentoMarketDataConfig databento_settings =20; //uint32 num_depth_levels = 1; diff --git a/src/lib_spark/spark/config/profiles/md_dev/market_data_config.conf b/src/lib_spark/spark/config/profiles/md_dev/market_data_config.conf index d0bc313aa..96d1968c9 100644 --- a/src/lib_spark/spark/config/profiles/md_dev/market_data_config.conf +++ b/src/lib_spark/spark/config/profiles/md_dev/market_data_config.conf @@ -84,6 +84,11 @@ MarketDataConfig { protocol_pong_timeout_ms: 30000 } + glbx { + cme_config_url: "ftp://ftp.cmegroup.com/SBEFix/Cert/Configuration/config.xml" + cme_templates_url: "ftp://ftp.cmegroup.com/SBEFix/Cert/Templates/templates_FixBinary.xml" + } + recording { num_of_seconds_to_run: 300 venue: VENUE_DERIBIT diff --git a/src/lib_spark/spark/config/profiles/md_prod/market_data_config.conf b/src/lib_spark/spark/config/profiles/md_prod/market_data_config.conf index 5afd98e04..9db75dfb7 100644 --- a/src/lib_spark/spark/config/profiles/md_prod/market_data_config.conf +++ b/src/lib_spark/spark/config/profiles/md_prod/market_data_config.conf @@ -71,6 +71,11 @@ MarketDataConfig { protocol_pong_timeout_ms: 30000 } + glbx { + cme_config_url: "ftp://ftp.cmegroup.com/sbefix/production/configuration/config.xml" + cme_templates_url: "ftp://ftp.cmegroup.com/SBEFix/Production/Templates/templates_FixBinary.xml" + } + recording { num_of_seconds_to_run: 300 venue: VENUE_DERIBIT diff --git a/src/lib_spark/spark/md/cme/MDP3Utils.cpp b/src/lib_spark/spark/md/cme/MDP3Utils.cpp index 6c64636dd..74e7d917b 100644 --- a/src/lib_spark/spark/md/cme/MDP3Utils.cpp +++ b/src/lib_spark/spark/md/cme/MDP3Utils.cpp @@ -44,42 +44,47 @@ bool check_file_exists_with_instructions( // For exponential backoff on logging this return true if x is a power of 2 or 0 bool is_pow_two_or_0(uint32_t x) { return (x & (~x + 1)) == x; } -void await_cme_config() { +bool await_cme_config( + const std::string &config_url, + const std::string &config_file, + const std::string &templates_url, + const std::string &templates_file, + core::TimeSpan timeout) { + static const core::TimeSpan max_timeout = core::TimeSpan::from_seconds(120); + bts_assert_msgf( + timeout <= max_timeout, + "await_cme_config timeout ({}) is suspiciously long (max timeout is {})", + timeout, max_timeout); static std::mutex config_mutex; std::lock_guard guard(config_mutex); - std::string config_url; - std::string templates_url; - - if (config::get().type() != "dev") { - config_url = - "ftp://ftp.cmegroup.com/SBEFix/Production/Configuration/config.xml"; - templates_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Templates/" - "templates_FixBinary.xml"; - } else { - config_url = - "ftp://ftp.cmegroup.com/SBEFix/Cert/Configuration/config.xml"; - templates_url = "ftp://ftp.cmegroup.com/SBEFix/Cert/Templates/" - "templates_FixBinary.xml"; + boost::filesystem::path config_path{config_file}; + boost::filesystem::path templates_path{templates_file}; + if (config_path.has_parent_path()) { + boost::filesystem::create_directories(config_path.parent_path()); + } + if (templates_path.has_parent_path()) { + boost::filesystem::create_directories(templates_path.parent_path()); } - - Mdp3DataDirectory data_dir; - boost::filesystem::create_directories(data_dir.data_dir()); - boost::filesystem::create_directories(data_dir.log_dir()); // Check config is present and give instructions for downloading it if not // If the file is not present log an error and wait a second. Continue to log // the error ever power of 2 seconds until the file appears or the process is // killed. uint32_t count = 0; - while (!check_file_exists_with_instructions( - data_dir.config_file().string(), config_url, is_pow_two_or_0(count))) { + // There is no reason we should wait longer than 2 minutes for config + auto start_time = core::TimeStamp::now(); + while ((core::TimeStamp::now() - start_time) < timeout && + !check_file_exists_with_instructions( + config_file, config_url, is_pow_two_or_0(count))) { ++count; std::this_thread::sleep_for(1s); } count = 0; - while (!check_file_exists_with_instructions( - data_dir.templates_file().string(), templates_url, - is_pow_two_or_0(count))) { + // There is no reason we should wait longer than 2 minutes for config + start_time = core::TimeStamp::now(); + while ((core::TimeStamp::now() - start_time) < timeout && + !check_file_exists_with_instructions( + templates_file, templates_url, is_pow_two_or_0(count))) { ++count; std::this_thread::sleep_for(1s); } @@ -89,6 +94,8 @@ void await_cme_config() { if (count > 0) { std::this_thread::sleep_for(1s); } + return boost::filesystem::exists(config_file) && + boost::filesystem::exists(templates_file); } SecurityType infer_security_type(const string &label) { @@ -108,12 +115,6 @@ SecurityType infer_security_type(const string &label) { return SECURITYTYPE_OPTION; } -const std::vector get_configured_cme_channels() { - // Get all glbx symbols - const auto symbols = SymbolLoader::load( - SecurityCriteria::create({domain::protobufs::VENUE_GLBX})); - return parse_channels({symbols.begin(), symbols.end()}); -} template std::string to_str(const std::vector &vec) { std::stringstream ss; @@ -121,17 +122,19 @@ std::string to_str(const std::vector &vec) { ss << t << " "; return ss.str(); } -const vector parse_channels(const vector &symbols) { +const vector parse_channels( + const std::string &config_file, const vector &symbols) { LOG(md, Info, "parse_channels symbols: {}", FMT_JOIN(symbols, ", ")); - await_cme_config(); + if (symbols.empty()) { + return {}; + } vector result; bool skip_options = config::get().skip_options(); pugi::xml_document doc; Mdp3DataDirectory data_dir{}; bts_assert_msg( - doc.load_file(data_dir.config_file().c_str()), - "Error parsing config.xml as xml"); + doc.load_file(config_file.c_str()), "Error parsing config.xml as xml"); auto configuration = doc.root().child("configuration"); bts_assert(configuration); diff --git a/src/lib_spark/spark/md/cme/MDP3Utils.h b/src/lib_spark/spark/md/cme/MDP3Utils.h index c329cc82d..5e00907ce 100644 --- a/src/lib_spark/spark/md/cme/MDP3Utils.h +++ b/src/lib_spark/spark/md/cme/MDP3Utils.h @@ -7,10 +7,19 @@ namespace bts::md { -void await_cme_config(); -const std::vector get_configured_cme_channels(); +/** + * Check if cme config.xml and templates.xml files are present. If they're not, and @p timeout > 0, + * download them from cme. + */ +bool await_cme_config( + const std::string &config_url, + const std::string &config_dest, + const std::string &templates_url, + const std::string &templates_dest, + core::TimeSpan timeout = core::TimeSpan::from_seconds(5)); + const std::vector parse_channels( - const std::vector &symbols); + const std::string &config_file, const std::vector &symbols); //https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP%203.0%20-%20CME%20Globex%20Pricing inline void decode_ticks(int vtt, double mintick, FixedSecurity &dest) { diff --git a/src/lib_spark/spark/orderman/OrderServiceServer.cpp b/src/lib_spark/spark/orderman/OrderServiceServer.cpp index 68aac3eca..ef10503a7 100644 --- a/src/lib_spark/spark/orderman/OrderServiceServer.cpp +++ b/src/lib_spark/spark/orderman/OrderServiceServer.cpp @@ -12,6 +12,7 @@ #include "spark/domain_model/common.h" #include "spark/md/MDClient.h" #include "spark/md/MDConsumer.h" +#include "spark/md/UserSymbol.h" #include "spark/mercury/actors/Timer.h" #include "spark/mercury/executor/Executor.h" #include "spark/metrics/metrics.h" @@ -171,7 +172,8 @@ OrderServiceServer::OrderServiceServer( DynamicOrderRegistry const &d, metrics::MetricsContainer &stats, trace::PerfLog &log, - md::TradeSummarySignalDispatcher &tssd) + md::TradeSummarySignalDispatcher &tssd, + std::map symbol_to_channel) : api::orderman::OrderServiceBase(ctx), me(new PImpl( ctx, @@ -202,7 +204,8 @@ OrderServiceServer::OrderServiceServer( thread, "Could not find a configured thread for MDSource. See the " "OrdermanCfg key"); - thread->executor.spawn(me->md_consumer, me->stats, log, &tssd); + thread->executor.spawn( + me->md_consumer, me->stats, log, &tssd, std::move(symbol_to_channel)); // OrderManager thread = find_thread_for_actor("OrderManager", me->actor_threads); diff --git a/src/lib_spark/spark/orderman/OrderServiceServer.h b/src/lib_spark/spark/orderman/OrderServiceServer.h index 93618b286..abcef6793 100644 --- a/src/lib_spark/spark/orderman/OrderServiceServer.h +++ b/src/lib_spark/spark/orderman/OrderServiceServer.h @@ -3,9 +3,11 @@ #include "spark/api/services/OrderService.pb.h" #include "spark/api/services/OrderService.server.pb.h" #include "spark/md/MDClient.h" +#include "spark/md/UserSymbol.h" #include "spark/mercury/actors/IPollableActor.h" #include "spark/orderman/events/ExchangeOpenOrdersEvent.h" #include +#include namespace bts { namespace md { @@ -24,13 +26,24 @@ class ExchangePositionEvent; class OrderServiceServer : public api::orderman::OrderServiceBase { public: + /** + * Creates the OrderServiceServer. + * + * @param context Service context for grpc services and quickbus. + * @param md_client Market data client + * @param dyn_order_reg Dynamic order registry + * @param stats Metrics container + * @param perf_log Performance logger + * @param md_channels A map of symbol to channel id (for trace logs). + */ OrderServiceServer( bts::api::ServiceContext &context, - md::MDClient &, - DynamicOrderRegistry const &, - metrics::MetricsContainer &, + md::MDClient &md_client, + DynamicOrderRegistry const &dyn_order_reg, + metrics::MetricsContainer &stats, trace::PerfLog &, - md::TradeSummarySignalDispatcher &); + md::TradeSummarySignalDispatcher &, + std::map md_channels = {}); ~OrderServiceServer(); void create_order( diff --git a/src/lib_spark/spark/orderman/actors/MDSource.h b/src/lib_spark/spark/orderman/actors/MDSource.h index 2826f4f14..634e359fa 100644 --- a/src/lib_spark/spark/orderman/actors/MDSource.h +++ b/src/lib_spark/spark/orderman/actors/MDSource.h @@ -34,17 +34,14 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor { md::MDConsumer &consumer, metrics::MetricsContainer &stats, trace::PerfLog &log, - md::TradeSummarySignalDispatcher *trade_signal_dispatcher) + md::TradeSummarySignalDispatcher *trade_signal_dispatcher, + std::map symbol_to_channel = {}) : _handles(), _trade_signal_dispatcher(trade_signal_dispatcher), _pending_handles(), _status_subs(), _md_consumer(consumer), _book_update(), _ticker_update(), _status_update(), _perf_log(log), - _stats(stats), _loop_timer(), _counter(0) { - + _stats(stats), _loop_timer(), + _symbol_to_channel(std::move(symbol_to_channel)), _counter(0) { _trades_buffer.Reserve(1000); - auto cme_channels = md::get_configured_cme_channels(); - for (auto const &c : cme_channels) - for (auto const &s : c.symbols) - _symbol_to_channel[s] = c.id; } void init() override {