From 1faef232aa6c7ab772cbedfe1aaa15deaf27cb72 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 25 Sep 2024 04:01:13 +0000 Subject: [PATCH] fix: Resolve onixs log & data dirs relative to app data root When our logging code moved to lib_system, it lost access to the config system. As a result, the "log_path" function, which was used by ICEMain and MDP3Main, broke. I added a new class, SparkDataDirectory, to be the central/common authority on paths to local storage. --- .../platforms/ice_onixs/ICEListenerHelpers.h | 4 +- .../sparkd/md/platforms/ice_onixs/ICEMain.cpp | 22 +++------- .../sparkd/md/platforms/ice_onixs/ICEMain.h | 5 +-- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 8 +--- .../sparkd/md/platforms/mdp3_onixs/MDP3Main.h | 2 +- .../spark/config}/spark_data_directory.cpp | 6 +++ .../spark/config}/spark_data_directory.h | 1 + src/lib_spark/spark/md/cme/MDP3Utils.cpp | 31 +++++-------- src/lib_spark/spark/md/cme/MDP3Utils.h | 9 +--- .../spark/md/cme/mdp3_data_directory.h | 5 +-- src/lib_spark/spark/md/ice/ICEUtils.cpp | 44 +++++-------------- src/lib_spark/spark/md/ice/ICEUtils.h | 20 +++++---- src/lib_spark/spark/md/ice/ice_channel.cpp | 39 ++++++++++++++++ src/lib_spark/spark/md/ice/ice_channel.h | 23 ++++++++++ .../spark/md/ice/ice_data_directory.h | 38 ++++++++++++++++ .../spark/orderman/actors/MDSource.h | 2 +- 16 files changed, 154 insertions(+), 105 deletions(-) rename src/{lib_system/system => lib_spark/spark/config}/spark_data_directory.cpp (83%) rename src/{lib_system/system => lib_spark/spark/config}/spark_data_directory.h (96%) create mode 100644 src/lib_spark/spark/md/ice/ice_channel.cpp create mode 100644 src/lib_spark/spark/md/ice/ice_channel.h create mode 100644 src/lib_spark/spark/md/ice/ice_data_directory.h diff --git a/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEListenerHelpers.h b/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEListenerHelpers.h index 10cfaea13..985d222ff 100644 --- a/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEListenerHelpers.h +++ b/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEListenerHelpers.h @@ -1,7 +1,7 @@ #pragma once #include "spark/md/coconut2.h" -#include "spark/md/ice/ICEUtils.h" +#include "spark/md/ice/ice_channel.h" #include "spark/utils/bts_assert.h" #include "spark/utils/helper_macros.h" @@ -49,4 +49,4 @@ inline datetime::Date date_of_timestamp(const onixs_ice::DateTime &dt) { } // namespace ice } // namespace md -} // namespace bts \ No newline at end of file +} // namespace bts diff --git a/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp b/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp index 672492e33..fb1c5ca84 100644 --- a/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp +++ b/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp @@ -2,19 +2,11 @@ #include "spark/config/config.h" #include "spark/config/protos/deployment_type.pb.h" -#include "spark/md/SecurityDB.h" -#include "spark/md/coconut2.h" -#include "spark/md/ice/ICEUtils.h" -#include "spark/md/protos/co2.pb.h" -#include "spark/md/vendor/OnixsUtils.h" -#include "spark/utils/Dispatcher.h" -#include "spark/utils/bts_chrono.h" -#include "spark/utils/helper_macros.h" - #include "spark/config/protos/market_data.pb.h" -#include "spark/utils/thread_manager.h" -#include "system/os.h" -#include "system/protobufs/proto_helper.h" +#include "spark/config/spark_data_directory.h" +#include "spark/md/ice/ICEUtils.h" +#include "spark/md/ice/ice_channel.h" +#include "spark/md/vendor/OnixsUtils.h" #include "system/protobufs/config/system_config.pb.h" #include @@ -114,7 +106,7 @@ void ICEMain::start(const std::vector &symbols) { validate_config_file(me_->config_file); auto license = onixs_create_temporary_license(); - const auto &channels = parse_ice_channels(symbols); + const auto &channels = ICEChannel::parse_all(symbols); for (const auto &channel : channels) { unique_ptr phandler = prepare_handler(channel, license.path.string()); @@ -161,9 +153,7 @@ unique_ptr ICEMain::prepare_handler( settings.licenseDirectory = license; settings.logLevel = (onixs_ice::LogLevels::Enum)options.onixs_log_mode(); settings.logDirectory = - (boost::filesystem::path{config::get().data_path()} / - "logging" / "onixs_ice/") - .string(); + (spark::SparkDataDirectory{}.log_dir() / "onixs_ice/").string(); settings.connectivityConfiguration = me_->config_file; // diff --git a/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.h b/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.h index 2b5e716fb..ac2aa64ba 100644 --- a/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.h +++ b/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.h @@ -1,10 +1,7 @@ #pragma once #include "ICEListener.h" -#include "ICESecurityIndex.h" -#include "spark/md/coconut2.h" -#include "spark/utils/bts_no_copy.h" -#include "onixs.h" +#include "spark/config/protos/market_data.pb.h" #include "sparkd/md/platforms/IMDPlatform.h" #include diff --git a/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 82e27eb9e..6bfe2b7ee 100644 --- a/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -1,21 +1,15 @@ #include "MDP3Main.h" -#include "domain/protobufs/trading_status.pb.h" #include "spark/config/config.h" +#include "spark/config/protos/deployment_type.pb.h" #include "spark/config/protos/market_data.pb.h" #include "spark/md/MBOProcessor.h" #include "spark/md/SecurityDB.h" #include "spark/md/cme/MDP3Utils.h" #include "spark/md/cme/mdp3_data_directory.h" -#include "spark/md/coconut2.h" -#include "spark/md/protos/co2.pb.h" #include "spark/md/vendor/OnixsUtils.h" -#include "spark/utils/Dispatcher.h" #include "spark/utils/thread_manager.h" #include "system/logging/logging.h" -#include "system/os.h" -#include "system/protobufs/config/system_config.pb.h" -#include "system/protobufs/proto_helper.h" #include #include diff --git a/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index 9d455a951..0b5b33c07 100644 --- a/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -2,9 +2,9 @@ #include "CMESecurityIndex.h" #include "MDP3Listener.h" +#include "onixs.h" #include "spark/md/coconut2.h" #include "spark/utils/bts_no_copy.h" -#include "onixs.h" #include "sparkd/md/platforms/IMDPlatform.h" #include diff --git a/src/lib_system/system/spark_data_directory.cpp b/src/lib_spark/spark/config/spark_data_directory.cpp similarity index 83% rename from src/lib_system/system/spark_data_directory.cpp rename to src/lib_spark/spark/config/spark_data_directory.cpp index 7d7672769..1e2147df4 100644 --- a/src/lib_system/system/spark_data_directory.cpp +++ b/src/lib_spark/spark/config/spark_data_directory.cpp @@ -1,5 +1,7 @@ #include "spark_data_directory.h" +#include "spark/config/config.h" #include "system/os.h" +#include "system/protobufs/config/system_config.pb.h" #include #include @@ -7,6 +9,10 @@ namespace bts::spark { +SparkDataDirectory::SparkDataDirectory() + : SparkDataDirectory( + config::get().data_path()) {} + SparkDataDirectory::SparkDataDirectory(const std::string &root) : root_(root) {} boost::filesystem::path SparkDataDirectory::root() const { return root_; } diff --git a/src/lib_system/system/spark_data_directory.h b/src/lib_spark/spark/config/spark_data_directory.h similarity index 96% rename from src/lib_system/system/spark_data_directory.h rename to src/lib_spark/spark/config/spark_data_directory.h index 0b47384d3..705284c03 100644 --- a/src/lib_system/system/spark_data_directory.h +++ b/src/lib_spark/spark/config/spark_data_directory.h @@ -7,6 +7,7 @@ namespace bts::spark { class SparkDataDirectory { public: + SparkDataDirectory(); explicit SparkDataDirectory(const std::string &root); boost::filesystem::path root() const; diff --git a/src/lib_spark/spark/md/cme/MDP3Utils.cpp b/src/lib_spark/spark/md/cme/MDP3Utils.cpp index 8e45c3624..6c64636dd 100644 --- a/src/lib_spark/spark/md/cme/MDP3Utils.cpp +++ b/src/lib_spark/spark/md/cme/MDP3Utils.cpp @@ -5,18 +5,15 @@ #include "spark/config/protos/deployment_type.pb.h" #include "spark/config/protos/market_data.pb.h" #include "spark/md/SymbolLoader.h" -#include "spark/utils/bts_chrono.h" -#include "spark/utils/datetime.h" +#include "spark/md/cme/mdp3_data_directory.h" #include "spark/utils/download.h" #include "system/FMT.h" #include "system/logging/logging.h" -#include "system/protobufs/config/system_config.pb.h" #include -#include -#include #include #include +#include using namespace std; using namespace bts::api; @@ -25,16 +22,6 @@ namespace bts { namespace md { using bts::domain::SecurityCriteria; -using bts::system::protobufs::SystemConfig; - -string data_path() { - auto p = config::get().data_path() + "cme_mdp3/"; - os::normalize_path(p); - return p; -} -const string config_path() { return data_path() + "config.xml"; } -const string template_path() { return data_path() + "templates.xml"; } -const std::string log_path() { return logging::log_path().string() + "/onix"; } bool check_file_exists_with_instructions( const string &filename, const string &url, bool log) { @@ -75,8 +62,9 @@ void await_cme_config() { "templates_FixBinary.xml"; } - boost::filesystem::create_directories(data_path()); - boost::filesystem::create_directories(log_path()); + Mdp3DataDirectory data_dir; + boost::filesystem::create_directories(data_dir.data_dir()); + boost::filesystem::create_directories(data_dir.log_dir()); // Check config is present and give instructions for downloading it if not // If the file is not present log an error and wait a second. Continue to log @@ -84,13 +72,14 @@ void await_cme_config() { // killed. uint32_t count = 0; while (!check_file_exists_with_instructions( - config_path(), config_url, is_pow_two_or_0(count))) { + data_dir.config_file().string(), config_url, is_pow_two_or_0(count))) { ++count; std::this_thread::sleep_for(1s); } count = 0; while (!check_file_exists_with_instructions( - template_path(), templates_url, is_pow_two_or_0(count))) { + data_dir.templates_file().string(), templates_url, + is_pow_two_or_0(count))) { ++count; std::this_thread::sleep_for(1s); } @@ -139,8 +128,10 @@ const vector parse_channels(const vector &symbols) { bool skip_options = config::get().skip_options(); pugi::xml_document doc; + Mdp3DataDirectory data_dir{}; bts_assert_msg( - doc.load_file(config_path().c_str()), "Error parsing config.xml as xml"); + doc.load_file(data_dir.config_file().c_str()), + "Error parsing config.xml as xml"); auto configuration = doc.root().child("configuration"); bts_assert(configuration); diff --git a/src/lib_spark/spark/md/cme/MDP3Utils.h b/src/lib_spark/spark/md/cme/MDP3Utils.h index 6e1e241fe..d60817668 100644 --- a/src/lib_spark/spark/md/cme/MDP3Utils.h +++ b/src/lib_spark/spark/md/cme/MDP3Utils.h @@ -1,12 +1,8 @@ #pragma once +#include "spark/md/Shm/FixedSecurity.h" #include "spark/md/cme/CMEChannel.h" -#include "spark/md/cme/CMEConnection.h" -#include "spark/md/coconut2.h" -#include "system/FMT_IMPL.h" -#include -#include #include namespace bts::md { @@ -16,9 +12,6 @@ const std::vector get_configured_cme_channels(); const std::vector parse_channels( const std::vector &symbols); -const std::string config_path(); -const std::string template_path(); -const std::string log_path(); //https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP%203.0%20-%20CME%20Globex%20Pricing inline void decode_ticks(int vtt, double mintick, FixedSecurity &dest) { switch (vtt) { diff --git a/src/lib_spark/spark/md/cme/mdp3_data_directory.h b/src/lib_spark/spark/md/cme/mdp3_data_directory.h index d77f969b8..5823bd09e 100644 --- a/src/lib_spark/spark/md/cme/mdp3_data_directory.h +++ b/src/lib_spark/spark/md/cme/mdp3_data_directory.h @@ -1,8 +1,8 @@ #pragma once #include "spark/config/config.h" +#include "spark/config/spark_data_directory.h" #include "system/protobufs/config/system_config.pb.h" -#include "system/spark_data_directory.h" #include #include @@ -28,8 +28,7 @@ struct Mdp3DataDirectory { } private: - spark::SparkDataDirectory root{ - config::get().data_path()}; + spark::SparkDataDirectory root; }; } // namespace bts::md diff --git a/src/lib_spark/spark/md/ice/ICEUtils.cpp b/src/lib_spark/spark/md/ice/ICEUtils.cpp index 87b41e933..4f22e3605 100644 --- a/src/lib_spark/spark/md/ice/ICEUtils.cpp +++ b/src/lib_spark/spark/md/ice/ICEUtils.cpp @@ -4,6 +4,7 @@ #include "domain/security_criteria.h" #include "spark/config/config.h" #include "spark/md/SymbolLoader.h" +#include "spark/md/ice/ice_data_directory.h" #include "spark/utils/bts_chrono.h" #include "spark/utils/datetime.h" #include "spark/utils/download.h" @@ -24,25 +25,15 @@ namespace md { namespace ice { using bts::domain::SecurityCriteria; -using bts::system::protobufs::SystemConfig; - -std::string ICEChannel::to_string() const { - std::string mt = name; - for (const auto &x : market_types) - mt += " " + std::to_string(x); - for (const auto &x : uds_market_types) - mt += " " + std::to_string(x); - return mt; -} // We are sort of punting on symbols here because there is no symbol -> channel // mapping supplied by ice therefore we must 1) enumerate the channels we want // in config, and smash them together with the list of symbols we care about. // This has the unfortunate side effect of not knowing what channel should // heartbeat what symbol but Eric says this is ok -const vector parse_ice_channels(const vector &symbols) { +const vector parse_ice_channels( + const vector &symbols, const ICEOptions &ice_options) { vector result; - const auto &ice_options = config::get(); for (const auto &x : ice_options.channel()) { md::ice::ICEChannel chan; chan.name = x.name(); @@ -57,22 +48,6 @@ const vector parse_ice_channels(const vector &symbols) { return result; } -string ice_data_path() { - auto p = config::get().data_path() + "onixs_ice/"; - os::normalize_path(p); - return p; -} -const string ice_config_file() { - return config::get().ice_config_file(); -} -const string ice_config_path() { - return ice_data_path() + "/" + ice_config_file(); -} - -const std::string ice_log_path() { - return logging::log_path().string() + "/onix_ice"; -} - bool ice_check_file_exists_with_instructions( const string &filename, const string &url, bool log) { // if (!boost::filesystem::exists(filename)) { @@ -104,14 +79,15 @@ void await_ice_config() { config_url = "https://community.theice.com/docs/DOC-19989"; } - boost::filesystem::create_directories(ice_data_path()); - boost::filesystem::create_directories(ice_log_path()); + IceDataDirectory data_dir; + boost::filesystem::create_directories(data_dir.data_dir()); + boost::filesystem::create_directories(data_dir.log_dir()); // Check config is present and give instructions for downloading it if not // If the file is not present log an error and wait a second. Continue to log // the error ever power of 2 seconds until the file appears or the process is // killed. - auto configpath = ice_config_path(); + const auto configpath = data_dir.config_file().string(); uint32_t count = 0; while (!ice_check_file_exists_with_instructions( configpath, config_url, is_pow_two_or_0(count))) { @@ -127,9 +103,9 @@ void await_ice_config() { const std::vector get_configured_ice_channels() { // Get all ice symbols - const auto symbols = - SymbolLoader::load(SecurityCriteria::create({domain::protobufs::VENUE_ICE})); - return parse_ice_channels({symbols.begin(), symbols.end()}); + const auto symbols = SymbolLoader::load( + SecurityCriteria::create({domain::protobufs::VENUE_ICE})); + return ICEChannel::parse_all({symbols.begin(), symbols.end()}); } } // namespace ice diff --git a/src/lib_spark/spark/md/ice/ICEUtils.h b/src/lib_spark/spark/md/ice/ICEUtils.h index 8a1b418c5..653fedd93 100644 --- a/src/lib_spark/spark/md/ice/ICEUtils.h +++ b/src/lib_spark/spark/md/ice/ICEUtils.h @@ -1,6 +1,8 @@ #pragma once #include "spark/md/coconut2.h" +#include "spark/md/ice/ice_channel.h" + #include #include #include @@ -9,18 +11,18 @@ namespace bts { namespace md { namespace ice { -struct ICEChannel { - std::string name; - std::vector market_types; - std::vector uds_market_types; - std::set symbols; - std::string to_string() const; -}; +// struct ICEChannel { +// std::string name; +// std::vector market_types; +// std::vector uds_market_types; +// std::set symbols; +// std::string to_string() const; +// }; void await_ice_config(); const std::vector get_configured_ice_channels(); -const std::vector -parse_ice_channels(const std::vector &symbols); +// const std::vector parse_ice_channels( +// const std::vector &symbols); const std::string ice_config_path(); const std::string ice_log_path(); diff --git a/src/lib_spark/spark/md/ice/ice_channel.cpp b/src/lib_spark/spark/md/ice/ice_channel.cpp new file mode 100644 index 000000000..36ca63419 --- /dev/null +++ b/src/lib_spark/spark/md/ice/ice_channel.cpp @@ -0,0 +1,39 @@ +#include "ice_channel.h" +#include "spark/config/config.h" +#include "spark/config/protos/market_data.pb.h" + +namespace bts::md::ice { + +std::string ICEChannel::to_string() const { + std::string mt = name; + for (const auto &x : market_types) + mt += " " + std::to_string(x); + for (const auto &x : uds_market_types) + mt += " " + std::to_string(x); + return mt; +} + +// We are sort of punting on symbols here because there is no symbol -> channel +// mapping supplied by ice therefore we must 1) enumerate the channels we want +// in config, and smash them together with the list of symbols we care about. +// This has the unfortunate side effect of not knowing what channel should +// heartbeat what symbol but Eric says this is ok +std::vector ICEChannel::parse_all( + const std::vector &symbols) { + std::vector result; + const auto &ice_options = config::get(); + for (const auto &x : ice_options.channel()) { + md::ice::ICEChannel chan; + chan.name = x.name(); + chan.symbols.insert(symbols.begin(), symbols.end()); + for (const auto &xx : x.market_type()) + chan.market_types.push_back(xx); + for (const auto &xx : x.uds_market_type()) + chan.uds_market_types.push_back(xx); + + result.push_back(chan); + } + return result; +} + +} // namespace bts::md::ice diff --git a/src/lib_spark/spark/md/ice/ice_channel.h b/src/lib_spark/spark/md/ice/ice_channel.h new file mode 100644 index 000000000..47d077e6c --- /dev/null +++ b/src/lib_spark/spark/md/ice/ice_channel.h @@ -0,0 +1,23 @@ +#pragma once + +#include "spark/md/UserSymbol.h" + +#include +#include +#include +#include + +namespace bts::md::ice { + +struct ICEChannel { + std::string name; + std::vector market_types; + std::vector uds_market_types; + std::set symbols; + std::string to_string() const; + + static std::vector parse_all( + const std::vector &symbols); +}; + +} // namespace bts::md::ice diff --git a/src/lib_spark/spark/md/ice/ice_data_directory.h b/src/lib_spark/spark/md/ice/ice_data_directory.h new file mode 100644 index 000000000..d3506fb35 --- /dev/null +++ b/src/lib_spark/spark/md/ice/ice_data_directory.h @@ -0,0 +1,38 @@ +#pragma once + +#include "spark/config/config.h" +#include "spark/config/spark_data_directory.h" +#include "system/protobufs/config/system_config.pb.h" + +#include +#include + +namespace bts::md { + +struct IceDataDirectory { + boost::filesystem::path log_dir() const { + return root.log_dir() / "onixs_ice/"; + } + + boost::filesystem::path data_dir() const { + return root.root() / "onixs_ice/"; + } + + boost::filesystem::path config_file() const { + return data_dir() / "config.xml"; + } + + boost::filesystem::path templates_file() const { + return data_dir() / "templates.xml"; + } + + void create_dirs() { + boost::filesystem::create_directories(data_dir()); + boost::filesystem::create_directories(log_dir()); + } + + private: + spark::SparkDataDirectory root; +}; + +} // namespace bts::md diff --git a/src/lib_spark/spark/orderman/actors/MDSource.h b/src/lib_spark/spark/orderman/actors/MDSource.h index 29e02390b..839d6592e 100644 --- a/src/lib_spark/spark/orderman/actors/MDSource.h +++ b/src/lib_spark/spark/orderman/actors/MDSource.h @@ -360,7 +360,7 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor { struct ChannelRecord { int64_t cursor; - co2::MDConsumerChannel channel; + md::MDConsumerChannel channel; boost::container::small_vector securities; md::UserSymbol user_symbol; };