fix: Resolve onixs log & data dirs relative to app data root

When our logging code moved to lib_system, it lost access to the config
system. As a result, the "log_path" function, which was used by ICEMain
and MDP3Main, broke. I added a new class, SparkDataDirectory, to be the
central/common authority on paths to local storage.
This commit is contained in:
John Colvin committed 2024-11-29 15:05:12 +00:00
1 parent 4e41721e33
commit 1faef232aa
16 files changed
+154 -105

No files matched your search

@@ -1,7 +1,7 @@
#pragma once
#include "spark/md/coconut2.h"
#include "spark/md/ice/ICEUtils.h"
#include "spark/md/ice/ice_channel.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/helper_macros.h"
@@ -49,4 +49,4 @@ inline datetime::Date date_of_timestamp(const onixs_ice::DateTime &dt) {
} // namespace ice
} // namespace md
} // namespace bts
} // namespace bts
@@ -2,19 +2,11 @@
#include "spark/config/config.h"
#include "spark/config/protos/deployment_type.pb.h"
#include "spark/md/SecurityDB.h"
#include "spark/md/coconut2.h"
#include "spark/md/ice/ICEUtils.h"
#include "spark/md/protos/co2.pb.h"
#include "spark/md/vendor/OnixsUtils.h"
#include "spark/utils/Dispatcher.h"
#include "spark/utils/bts_chrono.h"
#include "spark/utils/helper_macros.h"
#include "spark/config/protos/market_data.pb.h"
#include "spark/utils/thread_manager.h"
#include "system/os.h"
#include "system/protobufs/proto_helper.h"
#include "spark/config/spark_data_directory.h"
#include "spark/md/ice/ICEUtils.h"
#include "spark/md/ice/ice_channel.h"
#include "spark/md/vendor/OnixsUtils.h"
#include "system/protobufs/config/system_config.pb.h"
#include <OnixS/ICE/iMpact/MarketData.h>
@@ -114,7 +106,7 @@ void ICEMain::start(const std::vector<UserSymbol> &symbols) {
validate_config_file(me_->config_file);
auto license = onixs_create_temporary_license();
const auto &channels = parse_ice_channels(symbols);
const auto &channels = ICEChannel::parse_all(symbols);
for (const auto &channel : channels) {
unique_ptr<onixs_ice::Handler> phandler =
prepare_handler(channel, license.path.string());
@@ -161,9 +153,7 @@ unique_ptr<onixs_ice::Handler> ICEMain::prepare_handler(
settings.licenseDirectory = license;
settings.logLevel = (onixs_ice::LogLevels::Enum)options.onixs_log_mode();
settings.logDirectory =
(boost::filesystem::path{config::get<SystemConfig>().data_path()} /
"logging" / "onixs_ice/")
.string();
(spark::SparkDataDirectory{}.log_dir() / "onixs_ice/").string();
settings.connectivityConfiguration = me_->config_file;
//
@@ -1,10 +1,7 @@
#pragma once
#include "ICEListener.h"
#include "ICESecurityIndex.h"
#include "spark/md/coconut2.h"
#include "spark/utils/bts_no_copy.h"
#include "onixs.h"
#include "spark/config/protos/market_data.pb.h"
#include "sparkd/md/platforms/IMDPlatform.h"
#include <memory>
@@ -1,21 +1,15 @@
#include "MDP3Main.h"
#include "domain/protobufs/trading_status.pb.h"
#include "spark/config/config.h"
#include "spark/config/protos/deployment_type.pb.h"
#include "spark/config/protos/market_data.pb.h"
#include "spark/md/MBOProcessor.h"
#include "spark/md/SecurityDB.h"
#include "spark/md/cme/MDP3Utils.h"
#include "spark/md/cme/mdp3_data_directory.h"
#include "spark/md/coconut2.h"
#include "spark/md/protos/co2.pb.h"
#include "spark/md/vendor/OnixsUtils.h"
#include "spark/utils/Dispatcher.h"
#include "spark/utils/thread_manager.h"
#include "system/logging/logging.h"
#include "system/os.h"
#include "system/protobufs/config/system_config.pb.h"
#include "system/protobufs/proto_helper.h"
#include <boost/filesystem.hpp>
#include <memory>
@@ -2,9 +2,9 @@
#include "CMESecurityIndex.h"
#include "MDP3Listener.h"
#include "onixs.h"
#include "spark/md/coconut2.h"
#include "spark/utils/bts_no_copy.h"
#include "onixs.h"
#include "sparkd/md/platforms/IMDPlatform.h"
#include <map>
@@ -1,5 +1,7 @@
#include "spark_data_directory.h"
#include "spark/config/config.h"
#include "system/os.h"
#include "system/protobufs/config/system_config.pb.h"
#include <boost/filesystem/path.hpp>
#include <fmt/format.h>
@@ -7,6 +9,10 @@
namespace bts::spark {
SparkDataDirectory::SparkDataDirectory()
: SparkDataDirectory(
config::get<system::protobufs::SystemConfig>().data_path()) {}
SparkDataDirectory::SparkDataDirectory(const std::string &root) : root_(root) {}
boost::filesystem::path SparkDataDirectory::root() const { return root_; }
@@ -7,6 +7,7 @@ namespace bts::spark {
class SparkDataDirectory {
public:
SparkDataDirectory();
explicit SparkDataDirectory(const std::string &root);
boost::filesystem::path root() const;
+11 -20
View File
@@ -5,18 +5,15 @@
#include "spark/config/protos/deployment_type.pb.h"
#include "spark/config/protos/market_data.pb.h"
#include "spark/md/SymbolLoader.h"
#include "spark/utils/bts_chrono.h"
#include "spark/utils/datetime.h"
#include "spark/md/cme/mdp3_data_directory.h"
#include "spark/utils/download.h"
#include "system/FMT.h"
#include "system/logging/logging.h"
#include "system/protobufs/config/system_config.pb.h"
#include <boost/filesystem.hpp>
#include <iostream>
#include <mutex>
#include <pugixml/pugixml.hpp>
#include <sstream>
#include <thread>
using namespace std;
using namespace bts::api;
@@ -25,16 +22,6 @@ namespace bts {
namespace md {
using bts::domain::SecurityCriteria;
using bts::system::protobufs::SystemConfig;
string data_path() {
auto p = config::get<SystemConfig>().data_path() + "cme_mdp3/";
os::normalize_path(p);
return p;
}
const string config_path() { return data_path() + "config.xml"; }
const string template_path() { return data_path() + "templates.xml"; }
const std::string log_path() { return logging::log_path().string() + "/onix"; }
bool check_file_exists_with_instructions(
const string &filename, const string &url, bool log) {
@@ -75,8 +62,9 @@ void await_cme_config() {
"templates_FixBinary.xml";
}
boost::filesystem::create_directories(data_path());
boost::filesystem::create_directories(log_path());
Mdp3DataDirectory data_dir;
boost::filesystem::create_directories(data_dir.data_dir());
boost::filesystem::create_directories(data_dir.log_dir());
// Check config is present and give instructions for downloading it if not
// If the file is not present log an error and wait a second. Continue to log
@@ -84,13 +72,14 @@ void await_cme_config() {
// killed.
uint32_t count = 0;
while (!check_file_exists_with_instructions(
config_path(), config_url, is_pow_two_or_0(count))) {
data_dir.config_file().string(), config_url, is_pow_two_or_0(count))) {
++count;
std::this_thread::sleep_for(1s);
}
count = 0;
while (!check_file_exists_with_instructions(
template_path(), templates_url, is_pow_two_or_0(count))) {
data_dir.templates_file().string(), templates_url,
is_pow_two_or_0(count))) {
++count;
std::this_thread::sleep_for(1s);
}
@@ -139,8 +128,10 @@ const vector<CMEChannel> parse_channels(const vector<UserSymbol> &symbols) {
bool skip_options = config::get<PublisherOptions>().skip_options();
pugi::xml_document doc;
Mdp3DataDirectory data_dir{};
bts_assert_msg(
doc.load_file(config_path().c_str()), "Error parsing config.xml as xml");
doc.load_file(data_dir.config_file().c_str()),
"Error parsing config.xml as xml");
auto configuration = doc.root().child("configuration");
bts_assert(configuration);
+1 -8
View File
@@ -1,12 +1,8 @@
#pragma once
#include "spark/md/Shm/FixedSecurity.h"
#include "spark/md/cme/CMEChannel.h"
#include "spark/md/cme/CMEConnection.h"
#include "spark/md/coconut2.h"
#include "system/FMT_IMPL.h"
#include <set>
#include <string>
#include <vector>
namespace bts::md {
@@ -16,9 +12,6 @@ const std::vector<CMEChannel> get_configured_cme_channels();
const std::vector<CMEChannel> parse_channels(
const std::vector<UserSymbol> &symbols);
const std::string config_path();
const std::string template_path();
const std::string log_path();
//https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP%203.0%20-%20CME%20Globex%20Pricing
inline void decode_ticks(int vtt, double mintick, FixedSecurity &dest) {
switch (vtt) {
@@ -1,8 +1,8 @@
#pragma once
#include "spark/config/config.h"
#include "spark/config/spark_data_directory.h"
#include "system/protobufs/config/system_config.pb.h"
#include "system/spark_data_directory.h"
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
@@ -28,8 +28,7 @@ struct Mdp3DataDirectory {
}
private:
spark::SparkDataDirectory root{
config::get<system::protobufs::SystemConfig>().data_path()};
spark::SparkDataDirectory root;
};
} // namespace bts::md
+10 -34
View File
@@ -4,6 +4,7 @@
#include "domain/security_criteria.h"
#include "spark/config/config.h"
#include "spark/md/SymbolLoader.h"
#include "spark/md/ice/ice_data_directory.h"
#include "spark/utils/bts_chrono.h"
#include "spark/utils/datetime.h"
#include "spark/utils/download.h"
@@ -24,25 +25,15 @@ namespace md {
namespace ice {
using bts::domain::SecurityCriteria;
using bts::system::protobufs::SystemConfig;
std::string ICEChannel::to_string() const {
std::string mt = name;
for (const auto &x : market_types)
mt += " " + std::to_string(x);
for (const auto &x : uds_market_types)
mt += " " + std::to_string(x);
return mt;
}
// We are sort of punting on symbols here because there is no symbol -> channel
// mapping supplied by ice therefore we must 1) enumerate the channels we want
// in config, and smash them together with the list of symbols we care about.
// This has the unfortunate side effect of not knowing what channel should
// heartbeat what symbol but Eric says this is ok
const vector<ICEChannel> parse_ice_channels(const vector<UserSymbol> &symbols) {
const vector<ICEChannel> parse_ice_channels(
const vector<UserSymbol> &symbols, const ICEOptions &ice_options) {
vector<ICEChannel> result;
const auto &ice_options = config::get<bts::md::ICEOptions>();
for (const auto &x : ice_options.channel()) {
md::ice::ICEChannel chan;
chan.name = x.name();
@@ -57,22 +48,6 @@ const vector<ICEChannel> parse_ice_channels(const vector<UserSymbol> &symbols) {
return result;
}
string ice_data_path() {
auto p = config::get<SystemConfig>().data_path() + "onixs_ice/";
os::normalize_path(p);
return p;
}
const string ice_config_file() {
return config::get<bts::md::ICEOptions>().ice_config_file();
}
const string ice_config_path() {
return ice_data_path() + "/" + ice_config_file();
}
const std::string ice_log_path() {
return logging::log_path().string() + "/onix_ice";
}
bool ice_check_file_exists_with_instructions(
const string &filename, const string &url, bool log) {
// if (!boost::filesystem::exists(filename)) {
@@ -104,14 +79,15 @@ void await_ice_config() {
config_url = "https://community.theice.com/docs/DOC-19989";
}
boost::filesystem::create_directories(ice_data_path());
boost::filesystem::create_directories(ice_log_path());
IceDataDirectory data_dir;
boost::filesystem::create_directories(data_dir.data_dir());
boost::filesystem::create_directories(data_dir.log_dir());
// Check config is present and give instructions for downloading it if not
// If the file is not present log an error and wait a second. Continue to log
// the error ever power of 2 seconds until the file appears or the process is
// killed.
auto configpath = ice_config_path();
const auto configpath = data_dir.config_file().string();
uint32_t count = 0;
while (!ice_check_file_exists_with_instructions(
configpath, config_url, is_pow_two_or_0(count))) {
@@ -127,9 +103,9 @@ void await_ice_config() {
const std::vector<ICEChannel> get_configured_ice_channels() {
// Get all ice symbols
const auto symbols =
SymbolLoader::load(SecurityCriteria::create({domain::protobufs::VENUE_ICE}));
return parse_ice_channels({symbols.begin(), symbols.end()});
const auto symbols = SymbolLoader::load(
SecurityCriteria::create({domain::protobufs::VENUE_ICE}));
return ICEChannel::parse_all({symbols.begin(), symbols.end()});
}
} // namespace ice
+11 -9
View File
@@ -1,6 +1,8 @@
#pragma once
#include "spark/md/coconut2.h"
#include "spark/md/ice/ice_channel.h"
#include <set>
#include <string>
#include <vector>
@@ -9,18 +11,18 @@ namespace bts {
namespace md {
namespace ice {
struct ICEChannel {
std::string name;
std::vector<uint32_t> market_types;
std::vector<uint32_t> uds_market_types;
std::set<UserSymbol> symbols;
std::string to_string() const;
};
// struct ICEChannel {
// std::string name;
// std::vector<uint32_t> market_types;
// std::vector<uint32_t> uds_market_types;
// std::set<UserSymbol> symbols;
// std::string to_string() const;
// };
void await_ice_config();
const std::vector<ICEChannel> get_configured_ice_channels();
const std::vector<ICEChannel>
parse_ice_channels(const std::vector<UserSymbol> &symbols);
// const std::vector<ICEChannel> parse_ice_channels(
// const std::vector<UserSymbol> &symbols);
const std::string ice_config_path();
const std::string ice_log_path();
@@ -0,0 +1,39 @@
#include "ice_channel.h"
#include "spark/config/config.h"
#include "spark/config/protos/market_data.pb.h"
namespace bts::md::ice {
std::string ICEChannel::to_string() const {
std::string mt = name;
for (const auto &x : market_types)
mt += " " + std::to_string(x);
for (const auto &x : uds_market_types)
mt += " " + std::to_string(x);
return mt;
}
// We are sort of punting on symbols here because there is no symbol -> channel
// mapping supplied by ice therefore we must 1) enumerate the channels we want
// in config, and smash them together with the list of symbols we care about.
// This has the unfortunate side effect of not knowing what channel should
// heartbeat what symbol but Eric says this is ok
std::vector<ICEChannel> ICEChannel::parse_all(
const std::vector<UserSymbol> &symbols) {
std::vector<ICEChannel> result;
const auto &ice_options = config::get<bts::md::ICEOptions>();
for (const auto &x : ice_options.channel()) {
md::ice::ICEChannel chan;
chan.name = x.name();
chan.symbols.insert(symbols.begin(), symbols.end());
for (const auto &xx : x.market_type())
chan.market_types.push_back(xx);
for (const auto &xx : x.uds_market_type())
chan.uds_market_types.push_back(xx);
result.push_back(chan);
}
return result;
}
} // namespace bts::md::ice
+23
View File
@@ -0,0 +1,23 @@
#pragma once
#include "spark/md/UserSymbol.h"
#include <cstdint>
#include <set>
#include <string>
#include <vector>
namespace bts::md::ice {
struct ICEChannel {
std::string name;
std::vector<uint32_t> market_types;
std::vector<uint32_t> uds_market_types;
std::set<UserSymbol> symbols;
std::string to_string() const;
static std::vector<ICEChannel> parse_all(
const std::vector<UserSymbol> &symbols);
};
} // namespace bts::md::ice
@@ -0,0 +1,38 @@
#pragma once
#include "spark/config/config.h"
#include "spark/config/spark_data_directory.h"
#include "system/protobufs/config/system_config.pb.h"
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
namespace bts::md {
struct IceDataDirectory {
boost::filesystem::path log_dir() const {
return root.log_dir() / "onixs_ice/";
}
boost::filesystem::path data_dir() const {
return root.root() / "onixs_ice/";
}
boost::filesystem::path config_file() const {
return data_dir() / "config.xml";
}
boost::filesystem::path templates_file() const {
return data_dir() / "templates.xml";
}
void create_dirs() {
boost::filesystem::create_directories(data_dir());
boost::filesystem::create_directories(log_dir());
}
private:
spark::SparkDataDirectory root;
};
} // namespace bts::md
@@ -360,7 +360,7 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
struct ChannelRecord {
int64_t cursor;
co2::MDConsumerChannel channel;
md::MDConsumerChannel channel;
boost::container::small_vector<TickerUpdateEvent, 5> securities;
md::UserSymbol user_symbol;
};