#ifndef _WIN32 #include "MDP3Utils.h" #include "lib/config/config.h" #include "lib/logging/logging.h" #include "lib/md/get_symbols.h" #include "lib/utils/bts_chrono.h" #include "lib/utils/datetime.h" #include "lib/utils/download.h" #include "lib/config/protos/market_data.pb.h" #include "lib/config/protos/system.pb.h" #include #include #include #include using namespace std; using namespace bts::api; namespace bts { namespace md { std::ostream &operator<<(std::ostream &out, md::CMEConnection const &c) { out << "feed_type: " << c.feed_type << " ip: " << c.ip << " port: " << c.port << " feed: " << c.feed; return out; } std::ostream &operator<<(std::ostream &out, md::CMEChannel const &ch) { out << "id: " << ch.id << "\n"; out << "symbols: \n"; for (auto s : ch.symbols) out << "\t" << s.symbol() << ":" << api::SecurityType_Name(s.type()) << "\n"; out << "Connections:\n"; for (auto const &[st, c] : ch.connections) out << "\t" << st << " -> {" << c << "}\n"; out << "\n"; return out; } string data_path() { auto p = config::get().data_path() + "cme_mdp3/"; #ifdef _WIN32 // OnixS requires win32 paths to look like Win32 paths - everyone else in the // world is ok with '/' os::normalize_path(p, '\\'); #else os::normalize_path(p); #endif return p; } const string config_path() { return data_path() + "config.xml"; } const string template_path() { return data_path() + "templates.xml"; } const std::string log_path() { return log::log_path().string() + "/onix"; } bool check_file_exists_with_instructions( const string &filename, const string &url, bool log) { if (!boost::filesystem::exists(filename)) { download_file(url.c_str(), filename.c_str()); } if (!boost::filesystem::exists(filename)) { if (log) { LOG(md, Error, "File {} is not present on this system. " "Please download it from the CME ftp site. " "I will wait until the file is present.\n$ curl {} > {} ", filename, url, filename); } return false; } return true; } // For exponential backoff on logging this return true if x is a power of 2 or 0 bool is_pow_two_or_0(uint32_t x) { return (x & (~x + 1)) == x; } void await_cme_config() { static std::mutex config_mutex; std::lock_guard guard(config_mutex); std::string config_url; std::string templates_url; if (config::get().type() != "dev") { config_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Configuration/config.xml"; templates_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Templates/" "templates_FixBinary.xml"; } else { config_url = "ftp://ftp.cmegroup.com/SBEFix/Cert/Configuration/config.xml"; templates_url = "ftp://ftp.cmegroup.com/SBEFix/Cert/Templates/" "templates_FixBinary.xml"; } boost::filesystem::create_directories(data_path()); boost::filesystem::create_directories(log_path()); // Check config is present and give instructions for downloading it if not // If the file is not present log an error and wait a second. Continue to log // the error ever power of 2 seconds until the file appears or the process is // killed. uint32_t count = 0; while (!check_file_exists_with_instructions( config_path(), config_url, is_pow_two_or_0(count))) { ++count; std::this_thread::sleep_for(1s); } count = 0; while (!check_file_exists_with_instructions( template_path(), templates_url, is_pow_two_or_0(count))) { ++count; std::this_thread::sleep_for(1s); } // If we waited for someone to download the file give it a second to be // finished downloading if (count > 0) { std::this_thread::sleep_for(1s); } } SecurityType infer_security_type(const string &label) { if (label.find("Option") != std::string::npos) { return SECURITYTYPE_OPTION; } if (label.find("Future") != std::string::npos) { return SECURITYTYPE_FUTURE; } // TODO: Allow configuration to override this? LOG(md, Warn, "Unable to guess security type from label: '{}" "' Treating as OPTION.", label); return SECURITYTYPE_OPTION; } const std::vector get_configured_cme_channels() { std::vector symbols; // Get all glbx symbols get_system_md_symbols(api::VENUE_GLBX, api::SECURITYTYPE_UNSET, &symbols); return parse_channels(symbols); } const vector parse_channels(const vector &symbols) { await_cme_config(); vector result; bool skip_options = config::get().skip_options(); pugi::xml_document doc; bts_assert_msg( doc.load_file(config_path().c_str()), "Error parsing config.xml as xml"); auto configuration = doc.root().child("configuration"); bts_assert(configuration); LOG(md, Info, "CME: Loaded '{}' config. Last updated: {}", std::string(configuration.attribute("environment").as_string("??")), std::string(configuration.attribute("updated").as_string("??"))); auto channels = configuration.children("channel"); for (const auto &chan : channels) { CMEChannel channel; channel.id = chan.attribute("id").as_string(); auto channel_type = infer_security_type(chan.attribute("label").as_string()); if (channel_type == SECURITYTYPE_OPTION && skip_options) continue; if (!(channel.id.empty())) { auto product_list = chan.child("products").children("product"); for (const auto &product : product_list) { auto symbol = product.attribute("code").as_string(); for (auto s : symbols) { if ((s.type() == channel_type || s.type() == SECURITYTYPE_SPREAD) && s.symbol() == symbol) { channel.symbols.insert(s); } } } if (!channel.symbols.empty()) { auto connections = chan.child("connections").children("connection"); for (const auto &srccon : connections) { std::string id = srccon.attribute("id").as_string(); if (!id.empty()) { CMEConnection connection; connection.feed_type = srccon.child("type").attribute("feed-type").as_string(); connection.ip = srccon.child_value("ip"); connection.port = srccon.child("port").text().as_uint(); connection.feed = srccon.child_value("feed")[0]; channel.connections.insert(std::make_pair(id, connection)); } } result.push_back(channel); } } } return result; } } // namespace md } // namespace bts #endif