fix: Resolve onixs log & data dirs relative to app data root
When our logging code moved to lib_system, it lost access to the config system. As a result, the "log_path" function, which was used by ICEMain and MDP3Main, broke. I added a new class, SparkDataDirectory, to be the central/common authority on paths to local storage.
This commit is contained in:
1 parent
4e41721e33
commit
1faef232aa
16 files changed
+154
-105
No files matched your search
@@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/md/coconut2.h"
|
||||
#include "spark/md/ice/ICEUtils.h"
|
||||
#include "spark/md/ice/ice_channel.h"
|
||||
#include "spark/utils/bts_assert.h"
|
||||
#include "spark/utils/helper_macros.h"
|
||||
|
||||
@@ -49,4 +49,4 @@ inline datetime::Date date_of_timestamp(const onixs_ice::DateTime &dt) {
|
||||
|
||||
} // namespace ice
|
||||
} // namespace md
|
||||
} // namespace bts
|
||||
} // namespace bts
|
||||
@@ -2,19 +2,11 @@
|
||||
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/config/protos/deployment_type.pb.h"
|
||||
#include "spark/md/SecurityDB.h"
|
||||
#include "spark/md/coconut2.h"
|
||||
#include "spark/md/ice/ICEUtils.h"
|
||||
#include "spark/md/protos/co2.pb.h"
|
||||
#include "spark/md/vendor/OnixsUtils.h"
|
||||
#include "spark/utils/Dispatcher.h"
|
||||
#include "spark/utils/bts_chrono.h"
|
||||
#include "spark/utils/helper_macros.h"
|
||||
|
||||
#include "spark/config/protos/market_data.pb.h"
|
||||
#include "spark/utils/thread_manager.h"
|
||||
#include "system/os.h"
|
||||
#include "system/protobufs/proto_helper.h"
|
||||
#include "spark/config/spark_data_directory.h"
|
||||
#include "spark/md/ice/ICEUtils.h"
|
||||
#include "spark/md/ice/ice_channel.h"
|
||||
#include "spark/md/vendor/OnixsUtils.h"
|
||||
#include "system/protobufs/config/system_config.pb.h"
|
||||
|
||||
#include <OnixS/ICE/iMpact/MarketData.h>
|
||||
@@ -114,7 +106,7 @@ void ICEMain::start(const std::vector<UserSymbol> &symbols) {
|
||||
validate_config_file(me_->config_file);
|
||||
auto license = onixs_create_temporary_license();
|
||||
|
||||
const auto &channels = parse_ice_channels(symbols);
|
||||
const auto &channels = ICEChannel::parse_all(symbols);
|
||||
for (const auto &channel : channels) {
|
||||
unique_ptr<onixs_ice::Handler> phandler =
|
||||
prepare_handler(channel, license.path.string());
|
||||
@@ -161,9 +153,7 @@ unique_ptr<onixs_ice::Handler> ICEMain::prepare_handler(
|
||||
settings.licenseDirectory = license;
|
||||
settings.logLevel = (onixs_ice::LogLevels::Enum)options.onixs_log_mode();
|
||||
settings.logDirectory =
|
||||
(boost::filesystem::path{config::get<SystemConfig>().data_path()} /
|
||||
"logging" / "onixs_ice/")
|
||||
.string();
|
||||
(spark::SparkDataDirectory{}.log_dir() / "onixs_ice/").string();
|
||||
settings.connectivityConfiguration = me_->config_file;
|
||||
|
||||
//
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "ICEListener.h"
|
||||
#include "ICESecurityIndex.h"
|
||||
#include "spark/md/coconut2.h"
|
||||
#include "spark/utils/bts_no_copy.h"
|
||||
#include "onixs.h"
|
||||
#include "spark/config/protos/market_data.pb.h"
|
||||
#include "sparkd/md/platforms/IMDPlatform.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
@@ -1,21 +1,15 @@
|
||||
#include "MDP3Main.h"
|
||||
|
||||
#include "domain/protobufs/trading_status.pb.h"
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/config/protos/deployment_type.pb.h"
|
||||
#include "spark/config/protos/market_data.pb.h"
|
||||
#include "spark/md/MBOProcessor.h"
|
||||
#include "spark/md/SecurityDB.h"
|
||||
#include "spark/md/cme/MDP3Utils.h"
|
||||
#include "spark/md/cme/mdp3_data_directory.h"
|
||||
#include "spark/md/coconut2.h"
|
||||
#include "spark/md/protos/co2.pb.h"
|
||||
#include "spark/md/vendor/OnixsUtils.h"
|
||||
#include "spark/utils/Dispatcher.h"
|
||||
#include "spark/utils/thread_manager.h"
|
||||
#include "system/logging/logging.h"
|
||||
#include "system/os.h"
|
||||
#include "system/protobufs/config/system_config.pb.h"
|
||||
#include "system/protobufs/proto_helper.h"
|
||||
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <memory>
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
#include "CMESecurityIndex.h"
|
||||
#include "MDP3Listener.h"
|
||||
#include "onixs.h"
|
||||
#include "spark/md/coconut2.h"
|
||||
#include "spark/utils/bts_no_copy.h"
|
||||
#include "onixs.h"
|
||||
#include "sparkd/md/platforms/IMDPlatform.h"
|
||||
|
||||
#include <map>
|
||||
|
||||
+6
@@ -1,5 +1,7 @@
|
||||
#include "spark_data_directory.h"
|
||||
#include "spark/config/config.h"
|
||||
#include "system/os.h"
|
||||
#include "system/protobufs/config/system_config.pb.h"
|
||||
|
||||
#include <boost/filesystem/path.hpp>
|
||||
#include <fmt/format.h>
|
||||
@@ -7,6 +9,10 @@
|
||||
|
||||
namespace bts::spark {
|
||||
|
||||
SparkDataDirectory::SparkDataDirectory()
|
||||
: SparkDataDirectory(
|
||||
config::get<system::protobufs::SystemConfig>().data_path()) {}
|
||||
|
||||
SparkDataDirectory::SparkDataDirectory(const std::string &root) : root_(root) {}
|
||||
|
||||
boost::filesystem::path SparkDataDirectory::root() const { return root_; }
|
||||
+1
@@ -7,6 +7,7 @@ namespace bts::spark {
|
||||
|
||||
class SparkDataDirectory {
|
||||
public:
|
||||
SparkDataDirectory();
|
||||
explicit SparkDataDirectory(const std::string &root);
|
||||
|
||||
boost::filesystem::path root() const;
|
||||
@@ -5,18 +5,15 @@
|
||||
#include "spark/config/protos/deployment_type.pb.h"
|
||||
#include "spark/config/protos/market_data.pb.h"
|
||||
#include "spark/md/SymbolLoader.h"
|
||||
#include "spark/utils/bts_chrono.h"
|
||||
#include "spark/utils/datetime.h"
|
||||
#include "spark/md/cme/mdp3_data_directory.h"
|
||||
#include "spark/utils/download.h"
|
||||
#include "system/FMT.h"
|
||||
#include "system/logging/logging.h"
|
||||
#include "system/protobufs/config/system_config.pb.h"
|
||||
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <pugixml/pugixml.hpp>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
|
||||
using namespace std;
|
||||
using namespace bts::api;
|
||||
@@ -25,16 +22,6 @@ namespace bts {
|
||||
namespace md {
|
||||
|
||||
using bts::domain::SecurityCriteria;
|
||||
using bts::system::protobufs::SystemConfig;
|
||||
|
||||
string data_path() {
|
||||
auto p = config::get<SystemConfig>().data_path() + "cme_mdp3/";
|
||||
os::normalize_path(p);
|
||||
return p;
|
||||
}
|
||||
const string config_path() { return data_path() + "config.xml"; }
|
||||
const string template_path() { return data_path() + "templates.xml"; }
|
||||
const std::string log_path() { return logging::log_path().string() + "/onix"; }
|
||||
|
||||
bool check_file_exists_with_instructions(
|
||||
const string &filename, const string &url, bool log) {
|
||||
@@ -75,8 +62,9 @@ void await_cme_config() {
|
||||
"templates_FixBinary.xml";
|
||||
}
|
||||
|
||||
boost::filesystem::create_directories(data_path());
|
||||
boost::filesystem::create_directories(log_path());
|
||||
Mdp3DataDirectory data_dir;
|
||||
boost::filesystem::create_directories(data_dir.data_dir());
|
||||
boost::filesystem::create_directories(data_dir.log_dir());
|
||||
|
||||
// Check config is present and give instructions for downloading it if not
|
||||
// If the file is not present log an error and wait a second. Continue to log
|
||||
@@ -84,13 +72,14 @@ void await_cme_config() {
|
||||
// killed.
|
||||
uint32_t count = 0;
|
||||
while (!check_file_exists_with_instructions(
|
||||
config_path(), config_url, is_pow_two_or_0(count))) {
|
||||
data_dir.config_file().string(), config_url, is_pow_two_or_0(count))) {
|
||||
++count;
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
count = 0;
|
||||
while (!check_file_exists_with_instructions(
|
||||
template_path(), templates_url, is_pow_two_or_0(count))) {
|
||||
data_dir.templates_file().string(), templates_url,
|
||||
is_pow_two_or_0(count))) {
|
||||
++count;
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
@@ -139,8 +128,10 @@ const vector<CMEChannel> parse_channels(const vector<UserSymbol> &symbols) {
|
||||
bool skip_options = config::get<PublisherOptions>().skip_options();
|
||||
|
||||
pugi::xml_document doc;
|
||||
Mdp3DataDirectory data_dir{};
|
||||
bts_assert_msg(
|
||||
doc.load_file(config_path().c_str()), "Error parsing config.xml as xml");
|
||||
doc.load_file(data_dir.config_file().c_str()),
|
||||
"Error parsing config.xml as xml");
|
||||
|
||||
auto configuration = doc.root().child("configuration");
|
||||
bts_assert(configuration);
|
||||
|
||||
@@ -1,12 +1,8 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/md/Shm/FixedSecurity.h"
|
||||
#include "spark/md/cme/CMEChannel.h"
|
||||
#include "spark/md/cme/CMEConnection.h"
|
||||
#include "spark/md/coconut2.h"
|
||||
#include "system/FMT_IMPL.h"
|
||||
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace bts::md {
|
||||
@@ -16,9 +12,6 @@ const std::vector<CMEChannel> get_configured_cme_channels();
|
||||
const std::vector<CMEChannel> parse_channels(
|
||||
const std::vector<UserSymbol> &symbols);
|
||||
|
||||
const std::string config_path();
|
||||
const std::string template_path();
|
||||
const std::string log_path();
|
||||
//https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP%203.0%20-%20CME%20Globex%20Pricing
|
||||
inline void decode_ticks(int vtt, double mintick, FixedSecurity &dest) {
|
||||
switch (vtt) {
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/config/spark_data_directory.h"
|
||||
#include "system/protobufs/config/system_config.pb.h"
|
||||
#include "system/spark_data_directory.h"
|
||||
|
||||
#include <boost/filesystem/operations.hpp>
|
||||
#include <boost/filesystem/path.hpp>
|
||||
@@ -28,8 +28,7 @@ struct Mdp3DataDirectory {
|
||||
}
|
||||
|
||||
private:
|
||||
spark::SparkDataDirectory root{
|
||||
config::get<system::protobufs::SystemConfig>().data_path()};
|
||||
spark::SparkDataDirectory root;
|
||||
};
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "domain/security_criteria.h"
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/md/SymbolLoader.h"
|
||||
#include "spark/md/ice/ice_data_directory.h"
|
||||
#include "spark/utils/bts_chrono.h"
|
||||
#include "spark/utils/datetime.h"
|
||||
#include "spark/utils/download.h"
|
||||
@@ -24,25 +25,15 @@ namespace md {
|
||||
namespace ice {
|
||||
|
||||
using bts::domain::SecurityCriteria;
|
||||
using bts::system::protobufs::SystemConfig;
|
||||
|
||||
std::string ICEChannel::to_string() const {
|
||||
std::string mt = name;
|
||||
for (const auto &x : market_types)
|
||||
mt += " " + std::to_string(x);
|
||||
for (const auto &x : uds_market_types)
|
||||
mt += " " + std::to_string(x);
|
||||
return mt;
|
||||
}
|
||||
|
||||
// We are sort of punting on symbols here because there is no symbol -> channel
|
||||
// mapping supplied by ice therefore we must 1) enumerate the channels we want
|
||||
// in config, and smash them together with the list of symbols we care about.
|
||||
// This has the unfortunate side effect of not knowing what channel should
|
||||
// heartbeat what symbol but Eric says this is ok
|
||||
const vector<ICEChannel> parse_ice_channels(const vector<UserSymbol> &symbols) {
|
||||
const vector<ICEChannel> parse_ice_channels(
|
||||
const vector<UserSymbol> &symbols, const ICEOptions &ice_options) {
|
||||
vector<ICEChannel> result;
|
||||
const auto &ice_options = config::get<bts::md::ICEOptions>();
|
||||
for (const auto &x : ice_options.channel()) {
|
||||
md::ice::ICEChannel chan;
|
||||
chan.name = x.name();
|
||||
@@ -57,22 +48,6 @@ const vector<ICEChannel> parse_ice_channels(const vector<UserSymbol> &symbols) {
|
||||
return result;
|
||||
}
|
||||
|
||||
string ice_data_path() {
|
||||
auto p = config::get<SystemConfig>().data_path() + "onixs_ice/";
|
||||
os::normalize_path(p);
|
||||
return p;
|
||||
}
|
||||
const string ice_config_file() {
|
||||
return config::get<bts::md::ICEOptions>().ice_config_file();
|
||||
}
|
||||
const string ice_config_path() {
|
||||
return ice_data_path() + "/" + ice_config_file();
|
||||
}
|
||||
|
||||
const std::string ice_log_path() {
|
||||
return logging::log_path().string() + "/onix_ice";
|
||||
}
|
||||
|
||||
bool ice_check_file_exists_with_instructions(
|
||||
const string &filename, const string &url, bool log) {
|
||||
// if (!boost::filesystem::exists(filename)) {
|
||||
@@ -104,14 +79,15 @@ void await_ice_config() {
|
||||
config_url = "https://community.theice.com/docs/DOC-19989";
|
||||
}
|
||||
|
||||
boost::filesystem::create_directories(ice_data_path());
|
||||
boost::filesystem::create_directories(ice_log_path());
|
||||
IceDataDirectory data_dir;
|
||||
boost::filesystem::create_directories(data_dir.data_dir());
|
||||
boost::filesystem::create_directories(data_dir.log_dir());
|
||||
|
||||
// Check config is present and give instructions for downloading it if not
|
||||
// If the file is not present log an error and wait a second. Continue to log
|
||||
// the error ever power of 2 seconds until the file appears or the process is
|
||||
// killed.
|
||||
auto configpath = ice_config_path();
|
||||
const auto configpath = data_dir.config_file().string();
|
||||
uint32_t count = 0;
|
||||
while (!ice_check_file_exists_with_instructions(
|
||||
configpath, config_url, is_pow_two_or_0(count))) {
|
||||
@@ -127,9 +103,9 @@ void await_ice_config() {
|
||||
|
||||
const std::vector<ICEChannel> get_configured_ice_channels() {
|
||||
// Get all ice symbols
|
||||
const auto symbols =
|
||||
SymbolLoader::load(SecurityCriteria::create({domain::protobufs::VENUE_ICE}));
|
||||
return parse_ice_channels({symbols.begin(), symbols.end()});
|
||||
const auto symbols = SymbolLoader::load(
|
||||
SecurityCriteria::create({domain::protobufs::VENUE_ICE}));
|
||||
return ICEChannel::parse_all({symbols.begin(), symbols.end()});
|
||||
}
|
||||
|
||||
} // namespace ice
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/md/coconut2.h"
|
||||
#include "spark/md/ice/ice_channel.h"
|
||||
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
@@ -9,18 +11,18 @@ namespace bts {
|
||||
namespace md {
|
||||
namespace ice {
|
||||
|
||||
struct ICEChannel {
|
||||
std::string name;
|
||||
std::vector<uint32_t> market_types;
|
||||
std::vector<uint32_t> uds_market_types;
|
||||
std::set<UserSymbol> symbols;
|
||||
std::string to_string() const;
|
||||
};
|
||||
// struct ICEChannel {
|
||||
// std::string name;
|
||||
// std::vector<uint32_t> market_types;
|
||||
// std::vector<uint32_t> uds_market_types;
|
||||
// std::set<UserSymbol> symbols;
|
||||
// std::string to_string() const;
|
||||
// };
|
||||
|
||||
void await_ice_config();
|
||||
const std::vector<ICEChannel> get_configured_ice_channels();
|
||||
const std::vector<ICEChannel>
|
||||
parse_ice_channels(const std::vector<UserSymbol> &symbols);
|
||||
// const std::vector<ICEChannel> parse_ice_channels(
|
||||
// const std::vector<UserSymbol> &symbols);
|
||||
|
||||
const std::string ice_config_path();
|
||||
const std::string ice_log_path();
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
#include "ice_channel.h"
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/config/protos/market_data.pb.h"
|
||||
|
||||
namespace bts::md::ice {
|
||||
|
||||
std::string ICEChannel::to_string() const {
|
||||
std::string mt = name;
|
||||
for (const auto &x : market_types)
|
||||
mt += " " + std::to_string(x);
|
||||
for (const auto &x : uds_market_types)
|
||||
mt += " " + std::to_string(x);
|
||||
return mt;
|
||||
}
|
||||
|
||||
// We are sort of punting on symbols here because there is no symbol -> channel
|
||||
// mapping supplied by ice therefore we must 1) enumerate the channels we want
|
||||
// in config, and smash them together with the list of symbols we care about.
|
||||
// This has the unfortunate side effect of not knowing what channel should
|
||||
// heartbeat what symbol but Eric says this is ok
|
||||
std::vector<ICEChannel> ICEChannel::parse_all(
|
||||
const std::vector<UserSymbol> &symbols) {
|
||||
std::vector<ICEChannel> result;
|
||||
const auto &ice_options = config::get<bts::md::ICEOptions>();
|
||||
for (const auto &x : ice_options.channel()) {
|
||||
md::ice::ICEChannel chan;
|
||||
chan.name = x.name();
|
||||
chan.symbols.insert(symbols.begin(), symbols.end());
|
||||
for (const auto &xx : x.market_type())
|
||||
chan.market_types.push_back(xx);
|
||||
for (const auto &xx : x.uds_market_type())
|
||||
chan.uds_market_types.push_back(xx);
|
||||
|
||||
result.push_back(chan);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
} // namespace bts::md::ice
|
||||
@@ -0,0 +1,23 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/md/UserSymbol.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace bts::md::ice {
|
||||
|
||||
struct ICEChannel {
|
||||
std::string name;
|
||||
std::vector<uint32_t> market_types;
|
||||
std::vector<uint32_t> uds_market_types;
|
||||
std::set<UserSymbol> symbols;
|
||||
std::string to_string() const;
|
||||
|
||||
static std::vector<ICEChannel> parse_all(
|
||||
const std::vector<UserSymbol> &symbols);
|
||||
};
|
||||
|
||||
} // namespace bts::md::ice
|
||||
@@ -0,0 +1,38 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/config/spark_data_directory.h"
|
||||
#include "system/protobufs/config/system_config.pb.h"
|
||||
|
||||
#include <boost/filesystem/operations.hpp>
|
||||
#include <boost/filesystem/path.hpp>
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
struct IceDataDirectory {
|
||||
boost::filesystem::path log_dir() const {
|
||||
return root.log_dir() / "onixs_ice/";
|
||||
}
|
||||
|
||||
boost::filesystem::path data_dir() const {
|
||||
return root.root() / "onixs_ice/";
|
||||
}
|
||||
|
||||
boost::filesystem::path config_file() const {
|
||||
return data_dir() / "config.xml";
|
||||
}
|
||||
|
||||
boost::filesystem::path templates_file() const {
|
||||
return data_dir() / "templates.xml";
|
||||
}
|
||||
|
||||
void create_dirs() {
|
||||
boost::filesystem::create_directories(data_dir());
|
||||
boost::filesystem::create_directories(log_dir());
|
||||
}
|
||||
|
||||
private:
|
||||
spark::SparkDataDirectory root;
|
||||
};
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -360,7 +360,7 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
|
||||
|
||||
struct ChannelRecord {
|
||||
int64_t cursor;
|
||||
co2::MDConsumerChannel channel;
|
||||
md::MDConsumerChannel channel;
|
||||
boost::container::small_vector<TickerUpdateEvent, 5> securities;
|
||||
md::UserSymbol user_symbol;
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user