Put cme config url in GlbxMarketDataConfig proto
Instead of using magic strings
This commit is contained in:
1 parent
5604eabde4
commit
ccc71c5a65
12 files changed
+133
-56
No files matched your search
@@ -1,11 +1,38 @@
|
||||
#include "Channels.h"
|
||||
#include "domain/protobufs/venue.pb.h"
|
||||
#include "domain/security_criteria.h"
|
||||
#include "spark/api/md/MarketDataConfig.pb.h"
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/md/SymbolLoader.h"
|
||||
#include "spark/md/cme/MDP3Utils.h"
|
||||
#include "spark/md/cme/mdp3_data_directory.h"
|
||||
|
||||
namespace bts::cmds::md::cmds {
|
||||
|
||||
using bts::domain::SecurityCriteria;
|
||||
using bts::md::await_cme_config;
|
||||
using bts::md::Mdp3DataDirectory;
|
||||
using bts::md::parse_channels;
|
||||
using bts::md::SymbolLoader;
|
||||
|
||||
Channels::Channels() : CLI::App("List configured cme channels", "channels") {
|
||||
callback([] {
|
||||
auto channels = bts::md::get_configured_cme_channels();
|
||||
Mdp3DataDirectory data_dir;
|
||||
auto md_cfg = bts::config::get<api::md::MarketDataConfig>();
|
||||
if (md_cfg.glbx().empty()) {
|
||||
std::cerr << "'MarketDataConfig.glbx' not defined (check spark.conf)."
|
||||
<< std::endl;
|
||||
}
|
||||
const auto cfg_file = data_dir.config_file().string();
|
||||
auto glbx_cfg = md_cfg.glbx()[0];
|
||||
const auto templates_file = data_dir.templates_file().string();
|
||||
await_cme_config(
|
||||
glbx_cfg.cme_config_url(), cfg_file, glbx_cfg.cme_templates_url(),
|
||||
templates_file);
|
||||
const auto symbols = SymbolLoader::load(
|
||||
SecurityCriteria::create({domain::protobufs::VENUE_GLBX}));
|
||||
const auto channels =
|
||||
parse_channels(cfg_file, {symbols.begin(), symbols.end()});
|
||||
for (const auto &ch : channels) {
|
||||
std::cout << ch.id << " - ";
|
||||
std::set<std::string> symbols;
|
||||
|
||||
@@ -122,7 +122,7 @@ void OnixsGlbxSubscription::start(
|
||||
auto cme_symbols = SymbolLoader::load(
|
||||
bts::domain::SecurityCriteria::create({domain::protobufs::VENUE_GLBX}));
|
||||
if (!cme_symbols.empty()) {
|
||||
auto *main = new bts::md::MDP3Main();
|
||||
auto *main = new bts::md::MDP3Main(me_->config);
|
||||
main->start({cme_symbols.begin(), cme_symbols.end()}, mbo, tssd);
|
||||
platforms.emplace_back(main);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "MDP3Main.h"
|
||||
|
||||
#include "spark/api/md/GlbxMarketDataConfig.pb.h"
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/config/protos/deployment_type.pb.h"
|
||||
#include "spark/config/protos/market_data.pb.h"
|
||||
@@ -28,6 +29,9 @@ namespace md {
|
||||
using Path = boost::filesystem::path;
|
||||
|
||||
struct MDP3Main::PImpl {
|
||||
PImpl(api::md::GlbxMarketDataConfig config) : config(std::move(config)) {}
|
||||
|
||||
api::md::GlbxMarketDataConfig config;
|
||||
Mdp3DataDirectory data_dir;
|
||||
CMESecurityIndex index;
|
||||
SecurityIDRecorder recorder;
|
||||
@@ -46,8 +50,8 @@ struct MDP3Main::PImpl {
|
||||
producer(recorder), engines(), handlers(), listeners(), recovery() {}
|
||||
};
|
||||
|
||||
MDP3Main::MDP3Main()
|
||||
: me_(std::make_unique<PImpl>()) {}
|
||||
MDP3Main::MDP3Main(api::md::GlbxMarketDataConfig config)
|
||||
: me_(std::make_unique<PImpl>(std::move(config))) {}
|
||||
|
||||
MDP3Main::~MDP3Main() = default;
|
||||
|
||||
@@ -180,10 +184,19 @@ void MDP3Main::start(
|
||||
oss << us << " ";
|
||||
}
|
||||
LOG(md, Info, "MDP3Main::start symbols: {}", oss.str());
|
||||
await_cme_config();
|
||||
const auto cfg_file = me_->data_dir.config_file().string();
|
||||
const auto &cfg_url = me_->config.cme_config_url();
|
||||
const auto templates_file = me_->data_dir.templates_file().string();
|
||||
const auto &templates_url = me_->config.cme_templates_url();
|
||||
if (!await_cme_config(cfg_url, cfg_file, templates_url, templates_file)) {
|
||||
LOG(md, Error,
|
||||
"MDP3Main::start exiting early because cme config.xml and/or "
|
||||
"templates.xml files are missing");
|
||||
return;
|
||||
}
|
||||
auto license = onixs_create_temporary_license();
|
||||
initialize(license.path.string());
|
||||
const auto &channels = parse_channels(symbols);
|
||||
const auto &channels = parse_channels(cfg_file, symbols);
|
||||
me_->engines = create_feed_engines(channels);
|
||||
for (const auto &channel : channels) {
|
||||
unique_ptr<onixs::Handler> phandler = prepare_handler(channel.id, mbo);
|
||||
|
||||
@@ -29,7 +29,7 @@ struct CMEChannel;
|
||||
*/
|
||||
class MDP3Main : public IMDPlatform {
|
||||
public:
|
||||
explicit MDP3Main();
|
||||
explicit MDP3Main(api::md::GlbxMarketDataConfig config);
|
||||
MDP3Main(const MDP3Main &) = delete;
|
||||
MDP3Main &operator=(const MDP3Main &) = delete;
|
||||
~MDP3Main();
|
||||
|
||||
@@ -11,6 +11,8 @@ option csharp_namespace = "bts.api.md";
|
||||
message GlbxMarketDataConfig {
|
||||
uint32 unique_check_value = 1;
|
||||
bts.domain.protobufs.SecurityCriteria security_criteria = 3;
|
||||
string cme_config_url = 4;
|
||||
string cme_templates_url = 5;
|
||||
bool use_databento = 10;
|
||||
DatabentoMarketDataConfig databento_settings =20;
|
||||
//uint32 num_depth_levels = 1;
|
||||
|
||||
@@ -84,6 +84,11 @@ MarketDataConfig {
|
||||
protocol_pong_timeout_ms: 30000
|
||||
}
|
||||
|
||||
glbx {
|
||||
cme_config_url: "ftp://ftp.cmegroup.com/SBEFix/Cert/Configuration/config.xml"
|
||||
cme_templates_url: "ftp://ftp.cmegroup.com/SBEFix/Cert/Templates/templates_FixBinary.xml"
|
||||
}
|
||||
|
||||
recording {
|
||||
num_of_seconds_to_run: 300
|
||||
venue: VENUE_DERIBIT
|
||||
|
||||
@@ -71,6 +71,11 @@ MarketDataConfig {
|
||||
protocol_pong_timeout_ms: 30000
|
||||
}
|
||||
|
||||
glbx {
|
||||
cme_config_url: "ftp://ftp.cmegroup.com/sbefix/production/configuration/config.xml"
|
||||
cme_templates_url: "ftp://ftp.cmegroup.com/SBEFix/Production/Templates/templates_FixBinary.xml"
|
||||
}
|
||||
|
||||
recording {
|
||||
num_of_seconds_to_run: 300
|
||||
venue: VENUE_DERIBIT
|
||||
|
||||
@@ -44,42 +44,47 @@ bool check_file_exists_with_instructions(
|
||||
// For exponential backoff on logging this return true if x is a power of 2 or 0
|
||||
bool is_pow_two_or_0(uint32_t x) { return (x & (~x + 1)) == x; }
|
||||
|
||||
void await_cme_config() {
|
||||
bool await_cme_config(
|
||||
const std::string &config_url,
|
||||
const std::string &config_file,
|
||||
const std::string &templates_url,
|
||||
const std::string &templates_file,
|
||||
core::TimeSpan timeout) {
|
||||
static const core::TimeSpan max_timeout = core::TimeSpan::from_seconds(120);
|
||||
bts_assert_msgf(
|
||||
timeout <= max_timeout,
|
||||
"await_cme_config timeout ({}) is suspiciously long (max timeout is {})",
|
||||
timeout, max_timeout);
|
||||
static std::mutex config_mutex;
|
||||
std::lock_guard<std::mutex> guard(config_mutex);
|
||||
std::string config_url;
|
||||
std::string templates_url;
|
||||
|
||||
if (config::get<config::DeploymentType>().type() != "dev") {
|
||||
config_url =
|
||||
"ftp://ftp.cmegroup.com/SBEFix/Production/Configuration/config.xml";
|
||||
templates_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Templates/"
|
||||
"templates_FixBinary.xml";
|
||||
} else {
|
||||
config_url =
|
||||
"ftp://ftp.cmegroup.com/SBEFix/Cert/Configuration/config.xml";
|
||||
templates_url = "ftp://ftp.cmegroup.com/SBEFix/Cert/Templates/"
|
||||
"templates_FixBinary.xml";
|
||||
boost::filesystem::path config_path{config_file};
|
||||
boost::filesystem::path templates_path{templates_file};
|
||||
if (config_path.has_parent_path()) {
|
||||
boost::filesystem::create_directories(config_path.parent_path());
|
||||
}
|
||||
if (templates_path.has_parent_path()) {
|
||||
boost::filesystem::create_directories(templates_path.parent_path());
|
||||
}
|
||||
|
||||
Mdp3DataDirectory data_dir;
|
||||
boost::filesystem::create_directories(data_dir.data_dir());
|
||||
boost::filesystem::create_directories(data_dir.log_dir());
|
||||
|
||||
// Check config is present and give instructions for downloading it if not
|
||||
// If the file is not present log an error and wait a second. Continue to log
|
||||
// the error ever power of 2 seconds until the file appears or the process is
|
||||
// killed.
|
||||
uint32_t count = 0;
|
||||
while (!check_file_exists_with_instructions(
|
||||
data_dir.config_file().string(), config_url, is_pow_two_or_0(count))) {
|
||||
// There is no reason we should wait longer than 2 minutes for config
|
||||
auto start_time = core::TimeStamp::now();
|
||||
while ((core::TimeStamp::now() - start_time) < timeout &&
|
||||
!check_file_exists_with_instructions(
|
||||
config_file, config_url, is_pow_two_or_0(count))) {
|
||||
++count;
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
count = 0;
|
||||
while (!check_file_exists_with_instructions(
|
||||
data_dir.templates_file().string(), templates_url,
|
||||
is_pow_two_or_0(count))) {
|
||||
// There is no reason we should wait longer than 2 minutes for config
|
||||
start_time = core::TimeStamp::now();
|
||||
while ((core::TimeStamp::now() - start_time) < timeout &&
|
||||
!check_file_exists_with_instructions(
|
||||
templates_file, templates_url, is_pow_two_or_0(count))) {
|
||||
++count;
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
@@ -89,6 +94,8 @@ void await_cme_config() {
|
||||
if (count > 0) {
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
return boost::filesystem::exists(config_file) &&
|
||||
boost::filesystem::exists(templates_file);
|
||||
}
|
||||
|
||||
SecurityType infer_security_type(const string &label) {
|
||||
@@ -108,12 +115,6 @@ SecurityType infer_security_type(const string &label) {
|
||||
return SECURITYTYPE_OPTION;
|
||||
}
|
||||
|
||||
const std::vector<CMEChannel> get_configured_cme_channels() {
|
||||
// Get all glbx symbols
|
||||
const auto symbols = SymbolLoader::load(
|
||||
SecurityCriteria::create({domain::protobufs::VENUE_GLBX}));
|
||||
return parse_channels({symbols.begin(), symbols.end()});
|
||||
}
|
||||
template <class T>
|
||||
std::string to_str(const std::vector<T> &vec) {
|
||||
std::stringstream ss;
|
||||
@@ -121,17 +122,19 @@ std::string to_str(const std::vector<T> &vec) {
|
||||
ss << t << " ";
|
||||
return ss.str();
|
||||
}
|
||||
const vector<CMEChannel> parse_channels(const vector<UserSymbol> &symbols) {
|
||||
const vector<CMEChannel> parse_channels(
|
||||
const std::string &config_file, const vector<UserSymbol> &symbols) {
|
||||
LOG(md, Info, "parse_channels symbols: {}", FMT_JOIN(symbols, ", "));
|
||||
await_cme_config();
|
||||
if (symbols.empty()) {
|
||||
return {};
|
||||
}
|
||||
vector<CMEChannel> result;
|
||||
bool skip_options = config::get<PublisherOptions>().skip_options();
|
||||
|
||||
pugi::xml_document doc;
|
||||
Mdp3DataDirectory data_dir{};
|
||||
bts_assert_msg(
|
||||
doc.load_file(data_dir.config_file().c_str()),
|
||||
"Error parsing config.xml as xml");
|
||||
doc.load_file(config_file.c_str()), "Error parsing config.xml as xml");
|
||||
|
||||
auto configuration = doc.root().child("configuration");
|
||||
bts_assert(configuration);
|
||||
|
||||
@@ -7,10 +7,19 @@
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
void await_cme_config();
|
||||
const std::vector<CMEChannel> get_configured_cme_channels();
|
||||
/**
|
||||
* Check if cme config.xml and templates.xml files are present. If they're not, and @p timeout > 0,
|
||||
* download them from cme.
|
||||
*/
|
||||
bool await_cme_config(
|
||||
const std::string &config_url,
|
||||
const std::string &config_dest,
|
||||
const std::string &templates_url,
|
||||
const std::string &templates_dest,
|
||||
core::TimeSpan timeout = core::TimeSpan::from_seconds(5));
|
||||
|
||||
const std::vector<CMEChannel> parse_channels(
|
||||
const std::vector<UserSymbol> &symbols);
|
||||
const std::string &config_file, const std::vector<UserSymbol> &symbols);
|
||||
|
||||
//https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP%203.0%20-%20CME%20Globex%20Pricing
|
||||
inline void decode_ticks(int vtt, double mintick, FixedSecurity &dest) {
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#include "spark/domain_model/common.h"
|
||||
#include "spark/md/MDClient.h"
|
||||
#include "spark/md/MDConsumer.h"
|
||||
#include "spark/md/UserSymbol.h"
|
||||
#include "spark/mercury/actors/Timer.h"
|
||||
#include "spark/mercury/executor/Executor.h"
|
||||
#include "spark/metrics/metrics.h"
|
||||
@@ -171,7 +172,8 @@ OrderServiceServer::OrderServiceServer(
|
||||
DynamicOrderRegistry const &d,
|
||||
metrics::MetricsContainer &stats,
|
||||
trace::PerfLog &log,
|
||||
md::TradeSummarySignalDispatcher &tssd)
|
||||
md::TradeSummarySignalDispatcher &tssd,
|
||||
std::map<md::UserSymbol, std::string> symbol_to_channel)
|
||||
: api::orderman::OrderServiceBase(ctx),
|
||||
me(new PImpl(
|
||||
ctx,
|
||||
@@ -202,7 +204,8 @@ OrderServiceServer::OrderServiceServer(
|
||||
thread, "Could not find a configured thread for MDSource. See the "
|
||||
"OrdermanCfg key");
|
||||
|
||||
thread->executor.spawn<MDSource>(me->md_consumer, me->stats, log, &tssd);
|
||||
thread->executor.spawn<MDSource>(
|
||||
me->md_consumer, me->stats, log, &tssd, std::move(symbol_to_channel));
|
||||
|
||||
// OrderManager
|
||||
thread = find_thread_for_actor("OrderManager", me->actor_threads);
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
#include "spark/api/services/OrderService.pb.h"
|
||||
#include "spark/api/services/OrderService.server.pb.h"
|
||||
#include "spark/md/MDClient.h"
|
||||
#include "spark/md/UserSymbol.h"
|
||||
#include "spark/mercury/actors/IPollableActor.h"
|
||||
#include "spark/orderman/events/ExchangeOpenOrdersEvent.h"
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
namespace bts {
|
||||
namespace md {
|
||||
@@ -24,13 +26,24 @@ class ExchangePositionEvent;
|
||||
|
||||
class OrderServiceServer : public api::orderman::OrderServiceBase {
|
||||
public:
|
||||
/**
|
||||
* Creates the OrderServiceServer.
|
||||
*
|
||||
* @param context Service context for grpc services and quickbus.
|
||||
* @param md_client Market data client
|
||||
* @param dyn_order_reg Dynamic order registry
|
||||
* @param stats Metrics container
|
||||
* @param perf_log Performance logger
|
||||
* @param md_channels A map of symbol to channel id (for trace logs).
|
||||
*/
|
||||
OrderServiceServer(
|
||||
bts::api::ServiceContext &context,
|
||||
md::MDClient &,
|
||||
DynamicOrderRegistry const &,
|
||||
metrics::MetricsContainer &,
|
||||
md::MDClient &md_client,
|
||||
DynamicOrderRegistry const &dyn_order_reg,
|
||||
metrics::MetricsContainer &stats,
|
||||
trace::PerfLog &,
|
||||
md::TradeSummarySignalDispatcher &);
|
||||
md::TradeSummarySignalDispatcher &,
|
||||
std::map<md::UserSymbol, std::string> md_channels = {});
|
||||
~OrderServiceServer();
|
||||
|
||||
void create_order(
|
||||
|
||||
@@ -34,17 +34,14 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
|
||||
md::MDConsumer &consumer,
|
||||
metrics::MetricsContainer &stats,
|
||||
trace::PerfLog &log,
|
||||
md::TradeSummarySignalDispatcher *trade_signal_dispatcher)
|
||||
md::TradeSummarySignalDispatcher *trade_signal_dispatcher,
|
||||
std::map<md::UserSymbol, std::string> symbol_to_channel = {})
|
||||
: _handles(), _trade_signal_dispatcher(trade_signal_dispatcher),
|
||||
_pending_handles(), _status_subs(), _md_consumer(consumer),
|
||||
_book_update(), _ticker_update(), _status_update(), _perf_log(log),
|
||||
_stats(stats), _loop_timer(), _counter(0) {
|
||||
|
||||
_stats(stats), _loop_timer(),
|
||||
_symbol_to_channel(std::move(symbol_to_channel)), _counter(0) {
|
||||
_trades_buffer.Reserve(1000);
|
||||
auto cme_channels = md::get_configured_cme_channels();
|
||||
for (auto const &c : cme_channels)
|
||||
for (auto const &s : c.symbols)
|
||||
_symbol_to_channel[s] = c.id;
|
||||
}
|
||||
|
||||
void init() override {
|
||||
|
||||
Reference in new issue
Block a user