Put cme config url in GlbxMarketDataConfig proto

Instead of using magic strings
This commit is contained in:
John Colvin committed 2024-11-29 15:05:12 +00:00
1 parent 5604eabde4
commit ccc71c5a65
12 files changed
+133 -56

No files matched your search

@@ -1,11 +1,38 @@
#include "Channels.h"
#include "domain/protobufs/venue.pb.h"
#include "domain/security_criteria.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/config/config.h"
#include "spark/md/SymbolLoader.h"
#include "spark/md/cme/MDP3Utils.h"
#include "spark/md/cme/mdp3_data_directory.h"
namespace bts::cmds::md::cmds {
using bts::domain::SecurityCriteria;
using bts::md::await_cme_config;
using bts::md::Mdp3DataDirectory;
using bts::md::parse_channels;
using bts::md::SymbolLoader;
Channels::Channels() : CLI::App("List configured cme channels", "channels") {
callback([] {
auto channels = bts::md::get_configured_cme_channels();
Mdp3DataDirectory data_dir;
auto md_cfg = bts::config::get<api::md::MarketDataConfig>();
if (md_cfg.glbx().empty()) {
std::cerr << "'MarketDataConfig.glbx' not defined (check spark.conf)."
<< std::endl;
}
const auto cfg_file = data_dir.config_file().string();
auto glbx_cfg = md_cfg.glbx()[0];
const auto templates_file = data_dir.templates_file().string();
await_cme_config(
glbx_cfg.cme_config_url(), cfg_file, glbx_cfg.cme_templates_url(),
templates_file);
const auto symbols = SymbolLoader::load(
SecurityCriteria::create({domain::protobufs::VENUE_GLBX}));
const auto channels =
parse_channels(cfg_file, {symbols.begin(), symbols.end()});
for (const auto &ch : channels) {
std::cout << ch.id << " - ";
std::set<std::string> symbols;
@@ -122,7 +122,7 @@ void OnixsGlbxSubscription::start(
auto cme_symbols = SymbolLoader::load(
bts::domain::SecurityCriteria::create({domain::protobufs::VENUE_GLBX}));
if (!cme_symbols.empty()) {
auto *main = new bts::md::MDP3Main();
auto *main = new bts::md::MDP3Main(me_->config);
main->start({cme_symbols.begin(), cme_symbols.end()}, mbo, tssd);
platforms.emplace_back(main);
}
@@ -1,5 +1,6 @@
#include "MDP3Main.h"
#include "spark/api/md/GlbxMarketDataConfig.pb.h"
#include "spark/config/config.h"
#include "spark/config/protos/deployment_type.pb.h"
#include "spark/config/protos/market_data.pb.h"
@@ -28,6 +29,9 @@ namespace md {
using Path = boost::filesystem::path;
struct MDP3Main::PImpl {
PImpl(api::md::GlbxMarketDataConfig config) : config(std::move(config)) {}
api::md::GlbxMarketDataConfig config;
Mdp3DataDirectory data_dir;
CMESecurityIndex index;
SecurityIDRecorder recorder;
@@ -46,8 +50,8 @@ struct MDP3Main::PImpl {
producer(recorder), engines(), handlers(), listeners(), recovery() {}
};
MDP3Main::MDP3Main()
: me_(std::make_unique<PImpl>()) {}
MDP3Main::MDP3Main(api::md::GlbxMarketDataConfig config)
: me_(std::make_unique<PImpl>(std::move(config))) {}
MDP3Main::~MDP3Main() = default;
@@ -180,10 +184,19 @@ void MDP3Main::start(
oss << us << " ";
}
LOG(md, Info, "MDP3Main::start symbols: {}", oss.str());
await_cme_config();
const auto cfg_file = me_->data_dir.config_file().string();
const auto &cfg_url = me_->config.cme_config_url();
const auto templates_file = me_->data_dir.templates_file().string();
const auto &templates_url = me_->config.cme_templates_url();
if (!await_cme_config(cfg_url, cfg_file, templates_url, templates_file)) {
LOG(md, Error,
"MDP3Main::start exiting early because cme config.xml and/or "
"templates.xml files are missing");
return;
}
auto license = onixs_create_temporary_license();
initialize(license.path.string());
const auto &channels = parse_channels(symbols);
const auto &channels = parse_channels(cfg_file, symbols);
me_->engines = create_feed_engines(channels);
for (const auto &channel : channels) {
unique_ptr<onixs::Handler> phandler = prepare_handler(channel.id, mbo);
@@ -29,7 +29,7 @@ struct CMEChannel;
*/
class MDP3Main : public IMDPlatform {
public:
explicit MDP3Main();
explicit MDP3Main(api::md::GlbxMarketDataConfig config);
MDP3Main(const MDP3Main &) = delete;
MDP3Main &operator=(const MDP3Main &) = delete;
~MDP3Main();
@@ -11,6 +11,8 @@ option csharp_namespace = "bts.api.md";
message GlbxMarketDataConfig {
uint32 unique_check_value = 1;
bts.domain.protobufs.SecurityCriteria security_criteria = 3;
string cme_config_url = 4;
string cme_templates_url = 5;
bool use_databento = 10;
DatabentoMarketDataConfig databento_settings =20;
//uint32 num_depth_levels = 1;
@@ -84,6 +84,11 @@ MarketDataConfig {
protocol_pong_timeout_ms: 30000
}
glbx {
cme_config_url: "ftp://ftp.cmegroup.com/SBEFix/Cert/Configuration/config.xml"
cme_templates_url: "ftp://ftp.cmegroup.com/SBEFix/Cert/Templates/templates_FixBinary.xml"
}
recording {
num_of_seconds_to_run: 300
venue: VENUE_DERIBIT
@@ -71,6 +71,11 @@ MarketDataConfig {
protocol_pong_timeout_ms: 30000
}
glbx {
cme_config_url: "ftp://ftp.cmegroup.com/sbefix/production/configuration/config.xml"
cme_templates_url: "ftp://ftp.cmegroup.com/SBEFix/Production/Templates/templates_FixBinary.xml"
}
recording {
num_of_seconds_to_run: 300
venue: VENUE_DERIBIT
+36 -33
View File
@@ -44,42 +44,47 @@ bool check_file_exists_with_instructions(
// For exponential backoff on logging this return true if x is a power of 2 or 0
bool is_pow_two_or_0(uint32_t x) { return (x & (~x + 1)) == x; }
void await_cme_config() {
bool await_cme_config(
const std::string &config_url,
const std::string &config_file,
const std::string &templates_url,
const std::string &templates_file,
core::TimeSpan timeout) {
static const core::TimeSpan max_timeout = core::TimeSpan::from_seconds(120);
bts_assert_msgf(
timeout <= max_timeout,
"await_cme_config timeout ({}) is suspiciously long (max timeout is {})",
timeout, max_timeout);
static std::mutex config_mutex;
std::lock_guard<std::mutex> guard(config_mutex);
std::string config_url;
std::string templates_url;
if (config::get<config::DeploymentType>().type() != "dev") {
config_url =
"ftp://ftp.cmegroup.com/SBEFix/Production/Configuration/config.xml";
templates_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Templates/"
"templates_FixBinary.xml";
} else {
config_url =
"ftp://ftp.cmegroup.com/SBEFix/Cert/Configuration/config.xml";
templates_url = "ftp://ftp.cmegroup.com/SBEFix/Cert/Templates/"
"templates_FixBinary.xml";
boost::filesystem::path config_path{config_file};
boost::filesystem::path templates_path{templates_file};
if (config_path.has_parent_path()) {
boost::filesystem::create_directories(config_path.parent_path());
}
if (templates_path.has_parent_path()) {
boost::filesystem::create_directories(templates_path.parent_path());
}
Mdp3DataDirectory data_dir;
boost::filesystem::create_directories(data_dir.data_dir());
boost::filesystem::create_directories(data_dir.log_dir());
// Check config is present and give instructions for downloading it if not
// If the file is not present log an error and wait a second. Continue to log
// the error ever power of 2 seconds until the file appears or the process is
// killed.
uint32_t count = 0;
while (!check_file_exists_with_instructions(
data_dir.config_file().string(), config_url, is_pow_two_or_0(count))) {
// There is no reason we should wait longer than 2 minutes for config
auto start_time = core::TimeStamp::now();
while ((core::TimeStamp::now() - start_time) < timeout &&
!check_file_exists_with_instructions(
config_file, config_url, is_pow_two_or_0(count))) {
++count;
std::this_thread::sleep_for(1s);
}
count = 0;
while (!check_file_exists_with_instructions(
data_dir.templates_file().string(), templates_url,
is_pow_two_or_0(count))) {
// There is no reason we should wait longer than 2 minutes for config
start_time = core::TimeStamp::now();
while ((core::TimeStamp::now() - start_time) < timeout &&
!check_file_exists_with_instructions(
templates_file, templates_url, is_pow_two_or_0(count))) {
++count;
std::this_thread::sleep_for(1s);
}
@@ -89,6 +94,8 @@ void await_cme_config() {
if (count > 0) {
std::this_thread::sleep_for(1s);
}
return boost::filesystem::exists(config_file) &&
boost::filesystem::exists(templates_file);
}
SecurityType infer_security_type(const string &label) {
@@ -108,12 +115,6 @@ SecurityType infer_security_type(const string &label) {
return SECURITYTYPE_OPTION;
}
const std::vector<CMEChannel> get_configured_cme_channels() {
// Get all glbx symbols
const auto symbols = SymbolLoader::load(
SecurityCriteria::create({domain::protobufs::VENUE_GLBX}));
return parse_channels({symbols.begin(), symbols.end()});
}
template <class T>
std::string to_str(const std::vector<T> &vec) {
std::stringstream ss;
@@ -121,17 +122,19 @@ std::string to_str(const std::vector<T> &vec) {
ss << t << " ";
return ss.str();
}
const vector<CMEChannel> parse_channels(const vector<UserSymbol> &symbols) {
const vector<CMEChannel> parse_channels(
const std::string &config_file, const vector<UserSymbol> &symbols) {
LOG(md, Info, "parse_channels symbols: {}", FMT_JOIN(symbols, ", "));
await_cme_config();
if (symbols.empty()) {
return {};
}
vector<CMEChannel> result;
bool skip_options = config::get<PublisherOptions>().skip_options();
pugi::xml_document doc;
Mdp3DataDirectory data_dir{};
bts_assert_msg(
doc.load_file(data_dir.config_file().c_str()),
"Error parsing config.xml as xml");
doc.load_file(config_file.c_str()), "Error parsing config.xml as xml");
auto configuration = doc.root().child("configuration");
bts_assert(configuration);
+12 -3
View File
@@ -7,10 +7,19 @@
namespace bts::md {
void await_cme_config();
const std::vector<CMEChannel> get_configured_cme_channels();
/**
* Check if cme config.xml and templates.xml files are present. If they're not, and @p timeout > 0,
* download them from cme.
*/
bool await_cme_config(
const std::string &config_url,
const std::string &config_dest,
const std::string &templates_url,
const std::string &templates_dest,
core::TimeSpan timeout = core::TimeSpan::from_seconds(5));
const std::vector<CMEChannel> parse_channels(
const std::vector<UserSymbol> &symbols);
const std::string &config_file, const std::vector<UserSymbol> &symbols);
//https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP%203.0%20-%20CME%20Globex%20Pricing
inline void decode_ticks(int vtt, double mintick, FixedSecurity &dest) {
@@ -12,6 +12,7 @@
#include "spark/domain_model/common.h"
#include "spark/md/MDClient.h"
#include "spark/md/MDConsumer.h"
#include "spark/md/UserSymbol.h"
#include "spark/mercury/actors/Timer.h"
#include "spark/mercury/executor/Executor.h"
#include "spark/metrics/metrics.h"
@@ -171,7 +172,8 @@ OrderServiceServer::OrderServiceServer(
DynamicOrderRegistry const &d,
metrics::MetricsContainer &stats,
trace::PerfLog &log,
md::TradeSummarySignalDispatcher &tssd)
md::TradeSummarySignalDispatcher &tssd,
std::map<md::UserSymbol, std::string> symbol_to_channel)
: api::orderman::OrderServiceBase(ctx),
me(new PImpl(
ctx,
@@ -202,7 +204,8 @@ OrderServiceServer::OrderServiceServer(
thread, "Could not find a configured thread for MDSource. See the "
"OrdermanCfg key");
thread->executor.spawn<MDSource>(me->md_consumer, me->stats, log, &tssd);
thread->executor.spawn<MDSource>(
me->md_consumer, me->stats, log, &tssd, std::move(symbol_to_channel));
// OrderManager
thread = find_thread_for_actor("OrderManager", me->actor_threads);
@@ -3,9 +3,11 @@
#include "spark/api/services/OrderService.pb.h"
#include "spark/api/services/OrderService.server.pb.h"
#include "spark/md/MDClient.h"
#include "spark/md/UserSymbol.h"
#include "spark/mercury/actors/IPollableActor.h"
#include "spark/orderman/events/ExchangeOpenOrdersEvent.h"
#include <memory>
#include <string>
namespace bts {
namespace md {
@@ -24,13 +26,24 @@ class ExchangePositionEvent;
class OrderServiceServer : public api::orderman::OrderServiceBase {
public:
/**
* Creates the OrderServiceServer.
*
* @param context Service context for grpc services and quickbus.
* @param md_client Market data client
* @param dyn_order_reg Dynamic order registry
* @param stats Metrics container
* @param perf_log Performance logger
* @param md_channels A map of symbol to channel id (for trace logs).
*/
OrderServiceServer(
bts::api::ServiceContext &context,
md::MDClient &,
DynamicOrderRegistry const &,
metrics::MetricsContainer &,
md::MDClient &md_client,
DynamicOrderRegistry const &dyn_order_reg,
metrics::MetricsContainer &stats,
trace::PerfLog &,
md::TradeSummarySignalDispatcher &);
md::TradeSummarySignalDispatcher &,
std::map<md::UserSymbol, std::string> md_channels = {});
~OrderServiceServer();
void create_order(
@@ -34,17 +34,14 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
md::MDConsumer &consumer,
metrics::MetricsContainer &stats,
trace::PerfLog &log,
md::TradeSummarySignalDispatcher *trade_signal_dispatcher)
md::TradeSummarySignalDispatcher *trade_signal_dispatcher,
std::map<md::UserSymbol, std::string> symbol_to_channel = {})
: _handles(), _trade_signal_dispatcher(trade_signal_dispatcher),
_pending_handles(), _status_subs(), _md_consumer(consumer),
_book_update(), _ticker_update(), _status_update(), _perf_log(log),
_stats(stats), _loop_timer(), _counter(0) {
_stats(stats), _loop_timer(),
_symbol_to_channel(std::move(symbol_to_channel)), _counter(0) {
_trades_buffer.Reserve(1000);
auto cme_channels = md::get_configured_cme_channels();
for (auto const &c : cme_channels)
for (auto const &s : c.symbols)
_symbol_to_channel[s] = c.id;
}
void init() override {