#ifndef _WIN32 #include "MDP3Utils.h" #include "lib/config/config.h" #include "lib/utils/datetime.h" #include "lib/utils/bts_chrono.h" #include "lib/utils/download.h" #include "lib/logging/logging.h" #include "lib/md/get_symbols.h" #include "lib/config/protos/MarketData.pb.h" #include "lib/config/protos/system.pb.h" #include #include #include using namespace std; using namespace bts::api; namespace bts { namespace md { string data_path() { auto p = config::get().data_path() + "cme_mdp3/"; #ifdef _WIN32 // OnixS requires win32 paths to look like Win32 paths - everyone else in the world is ok with '/' os::normalize_path(p, '\\'); #else os::normalize_path(p); #endif return p; } const string config_path() { return data_path() + "config.xml"; } const string template_path() { return data_path() + "templates.xml"; } const std::string log_path() {return log:: log_path().string() + "/onix";} bool check_file_exists_with_instructions(const string& filename, const string& url, bool log) { if (!boost::filesystem::exists(filename)) { download_file(url.c_str(), filename.c_str()); } if (!boost::filesystem::exists(filename)) { if (log) { LOG(md,Error, "File {} is not present on this system. " "Please download it from the CME ftp site. " "I will wait until the file is present.\n$ curl {} > {} ", filename, url, filename); } return false; } return true; } // For exponential backoff on logging this return true if x is a power of 2 or 0 bool is_pow_two_or_0(uint32_t x) { return (x & (~x + 1)) == x; } void await_cme_config() { static std::mutex config_mutex; std::lock_guard guard(config_mutex); std::string config_url; std::string templates_url; if (config::get().type() != "dev") { config_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Configuration/config.xml"; templates_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Templates/templates_FixBinary.xml"; } else { config_url = "ftp://ftp.cmegroup.com/SBEFix/Cert/Configuration/config.xml"; templates_url = "ftp://ftp.cmegroup.com/SBEFix/Cert/Templates/templates_FixBinary.xml"; } boost::filesystem::create_directories(data_path()); boost::filesystem::create_directories(log_path()); // Check config is present and give instructions for downloading it if not // If the file is not present log an error and wait a second. Continue to log the error ever power of 2 seconds until the file appears or the process is killed. uint32_t count = 0; while(!check_file_exists_with_instructions(config_path(), config_url, is_pow_two_or_0(count))) { ++count; std::this_thread::sleep_for(1s); } count = 0; while(!check_file_exists_with_instructions(template_path(), templates_url, is_pow_two_or_0(count))) { ++count; std::this_thread::sleep_for(1s); } // If we waited for someone to download the file give it a second to be finished downloading if (count > 0) { std::this_thread::sleep_for(1s); } } SecurityType infer_security_type(const string& label) { if (label.find("Option") != std::string::npos) { return SECURITYTYPE_OPTION; } if (label.find("Future") != std::string::npos) { return SECURITYTYPE_FUTURE; } // TODO: Allow configuration to override this? LOG(md,Warn,"Unable to guess security type from label: '{}" "' Treating as OPTION.",label); return SECURITYTYPE_OPTION; } const std::vector get_configured_cme_channels() { std::vector symbols; // Get all glbx symbols get_system_md_symbols(api::VENUE_GLBX, api::SECURITYTYPE_UNSET, &symbols); return parse_channels(symbols); } const vector parse_channels(const vector& symbols) { await_cme_config(); vector result; bool skip_options = config::get().skip_options(); pugi::xml_document doc; bts_assert_msg(doc.load_file(config_path().c_str()), "Error parsing config.xml as xml"); auto configuration = doc.root().child("configuration"); bts_assert(configuration); LOG(md,Info,"CME: Loaded '{}' config. Last updated: {}" , std::string(configuration.attribute("environment").as_string("??")), std::string(configuration.attribute("updated").as_string("??"))); auto channels = configuration.children("channel"); for (const auto& chan : channels) { CMEChannel channel; channel.id = chan.attribute("id").as_string(); auto channel_type = infer_security_type(chan.attribute("label").as_string()); if (channel_type == SECURITYTYPE_OPTION && skip_options) continue; if (!(channel.id.empty())) { auto product_list = chan.child("products").children("product"); for (const auto& product : product_list) { auto symbol = product.attribute("code").as_string(); for (auto s : symbols) { if ((s.type() == channel_type || s.type() == SECURITYTYPE_SPREAD) && s.symbol() == symbol) { channel.symbols.insert(s); } } } if (!channel.symbols.empty()) { auto connections = chan.child("connections").children("connection"); for (const auto& srccon : connections) { std::string id = srccon.attribute("id").as_string(); if (!id.empty()) { CMEConnection connection; connection.feed_type = srccon.child("type").attribute("feed-type").as_string(); connection.ip = srccon.child_value("ip"); connection.port = srccon.child("port").text().as_uint(); connection.feed = srccon.child_value("feed")[0]; channel.connections.insert(std::make_pair(id, connection)); } } result.push_back(channel); } } } return result; } }} //namespace #endif