#include "run_md.h" #include "lib/md/relay/RelayRecorder.h" #include "lib/md/PlatformLoader.h" #include "lib/utils/bts_assert.h" #include "lib/config/config.h" #include "lib/config/protos/MarketData.pb.h" #include "lib/logging/logging.h" #include "lib/md/get_symbols.h" #include "lib/md/UserSymbol.h" #include "lib/md/Diag/ShmDumper.h" #include "lib/md/Diag/MDPeriodicDumper.h" #include "lib/md/cme/MDP3Utils.h" #ifdef USE_ONIX #include "platforms/mdp3_onixs/MDP3Main.h" #include "platforms/ice_onixs/ICEMain.h" #endif #include "platforms/IMDPlatform.h" #include "lib/md/SecurityDB.h" #include "server/sparkd/exchange_sim/ExchangeSim.h" #include "lib/utils/thread_manager.h" #include "lib/utils/parse_enum.h" #include #include #include #include #include #include using namespace bts; using namespace std; namespace bts { namespace md { string default_arg(const vector& args, size_t index, const char* default_value="") { if (index < args.size()) return args[index]; return default_value; } void clean_dir(const std::string base_path) { LOG(md, Info, "Cleaning shared memory"); namespace fs = boost::filesystem; const fs::path p(base_path); auto file_prefix = "bts_edge_" + config::get().shared_memory_namespace() + "_"; if (fs::is_directory(p)) { fs::directory_iterator end_iter; for (fs::directory_iterator dir_iter(p); dir_iter != end_iter; ++dir_iter) { if (fs::is_regular_file(dir_iter->status())) { auto name = dir_iter->path().filename().string(); if (name.find(file_prefix) != std::string::npos) { boost::system::error_code err; fs::remove(dir_iter->path(), err); if (!err) LOG(md, Info, "Cleaned up {}", dir_iter->path()); } } else if (fs::is_directory(dir_iter->status())) { std::string path = dir_iter->path().string(); clean_dir(path); } } } } void run_replay() { auto naggle_ms = config::get().naggle_ms(); const auto& options = config::get(); auto file = options.url(); auto fallback_path = options.fallback(); replay_recording(file, naggle_ms, fallback_path); } void run_onixs(MBOWriter* mbo) { (void) mbo; #ifdef USE_ONIX std::vector> platforms; // get all glbx symbols std::vector cme_symbols; get_system_md_symbols(api::VENUE_GLBX, api::SECURITYTYPE_UNSET, &cme_symbols); if(cme_symbols.size() > 0) { MDP3Main* main = new MDP3Main(); std::shared_ptr ptr(main); main->start(cme_symbols, mbo); platforms.push_back(ptr); } std::vector ice_symbols; get_system_md_symbols(api::VENUE_ICE, api::SECURITYTYPE_UNSET, &ice_symbols); if(ice_symbols.size() > 0) { ice::ICEMain* imain = new ice::ICEMain(); std::shared_ptr ptr(imain); imain->start(ice_symbols); platforms.push_back(ptr); } // // Timers auto& disp = utils::Dispatcher::current(); utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() { for (auto& p: platforms) p->send_heartbeats(); }); heartbeat_timer.start(); disp.run(); #endif } void run_cleanup() { clean_dir("/dev/shm"); } std::optional _md_thread; void run_market_data_publishers_forever(MBOWriter* mbo) { const auto& conf = config::get(); if (conf.use_replay()) { run_replay(); } else if (conf.use_onixs()) { run_onixs(mbo); } } void boot_sleep() { auto conf = config::get(); const auto* schedule = &conf.standard(); if (datetime::is_local_time_sunday()) { schedule = &conf.sunday(); } bts_assert(schedule); int hh_start = 0, mm_start = 0, hh_end = 0, mm_end = 0; sscanf(schedule->start().c_str(), "%i:%i", &hh_start, &mm_start); sscanf(schedule->end().c_str(), "%i:%i", &hh_end, &mm_end); if (datetime::is_local_time_between(hh_start, mm_start, hh_end, mm_end)) { std::this_thread::sleep_for( std::chrono::seconds( conf.delay_seconds())); } else { std::this_thread::sleep_for(2s); } } void run_market_datapublishers(MBOWriter* mbo) { bts_assert(_md_thread == std::nullopt); run_cleanup(); if (!config::get().use_md()) return; // start a thread for the md _md_thread = std::optional( threading::make_thread("md publisher", [mbo]() { run_market_data_publishers_forever(mbo); })); if (config::get().use_sim()) { threading::make_thread("ExchangeSimGLBX", [mbo]() { sim::start_exchange_sim(api::Venue::VENUE_SIM, mbo); }).detach(); //threading::make_thread("ExchangeSimICE", [mbo]() { sim::start_exchange_sim(api::Venue::VENUE_SIMICE, mbo); }).detach(); //TODO I'll need Eric's help to disambiguate replay / relay here } boot_sleep(); } void run_record(const std::string& arg){ boost::filesystem::path p(arg); record_everything(p.parent_path().string(), p.filename().string()); } string& uppercase(string& s) { transform(s.begin(), s.end(), s.begin(), ::toupper); return s; } template bool parse_enum(const string& value, const string& prefix, T2 parse, T default_value, T* result) { auto opt = ::bts::parse_enum(value, prefix, parse); if (opt) { *result = opt.value(); return true; } else { *result = default_value; return false; } } bool int_of_string(const std::string& str, long* result) { try { size_t pos = 0; int i = std::stol(str, &pos); if (pos == str.size()) { *result = i; return true; } } catch(const std::exception&) { } return false; } typedef std::string Symbol; struct MDArgs { api::Venue venue; co2::Platform platform; api::SecurityType type; Symbol symbol; bool hot; long number; }; MDArgs parse_md_args(const vector& args) { using namespace api; using namespace co2; auto venue = VENUE_UNSET; auto platform = PLATFORM_UNSET; auto type = SECURITYTYPE_UNSET; bool hot = false; long number = 0; std::string symbol; for (auto str : args) { // Iterate through each argument and see if it parses to one of these enums uppercase(str); bool used = (venue == VENUE_UNSET && parse_enum(str, "VENUE", Venue_Parse, venue, &venue)) || (platform == PLATFORM_UNSET && parse_enum(str, "PLATFORM", Platform_Parse, platform, &platform)) || parse_enum(str, "SECURITYTYPE", SecurityType_Parse, type, &type); if (str == "HOT") { hot = true; used = true; } else if (str == "COLD") { hot = false; used = true; } else { used |= int_of_string(str, &number); } if (!used) { symbol = str; } } if (venue == VENUE_UNSET) venue = VENUE_GLBX; if (platform == PLATFORM_UNSET) platform = platform_for_exchange(venue); return { venue, platform, type, symbol, hot, number }; } void run_dump(const vector& args) { using std::get; auto p = parse_md_args(args); stringstream ss; ss << "bts_edge_" << config::get().shared_memory_namespace() << "_" << uppercase(p.symbol) << "_" << SecurityType_Name(p.type) << "_" << Venue_Name(p.venue) << "_" << Platform_Name(p.platform); ShmDumper dumper(ss.str()); if (p.hot) { dumper.dump_hot(); } else { dumper.dump_cold(); } } void run_rectool(const vector args) { if (args.size() < 2) { cout << "rectool use: " << endl; cout << "\tbtscmd md rectool print " << endl; cout << "\tbtscmd md rectool unprint " << endl; cout << "\tbtscmd md rectool trace " << endl; return; } if (args[0] == "print") { print_recording(args[1]); } else if (args[0] == "unprint") { unprint_recording(args[1], args[2]); } else if (args[0] == "trace") { trace_recording(args[1], atoi(args[2].c_str())); } } void dump_db(const vector& args) { set keys; bool all = false; for (const auto& arg : args) { if (arg == "all") { all = true; break; } auto key = std::stoul(arg); if (key > 0) { keys.insert((uint)key); } } std::set symbols; md::get_system_md_symbols(symbols); md::db::load_securities(symbols, [&](const api::Security& security) { if (all || keys.find(security.id()) != keys.end()) { security.PrintDebugString(); if (md::securitytype_of(security) == api::SECURITYTYPE_SYNTHETIC) { auto metadata = md::db::load_metadata(); for (const auto &m : metadata) { if(m.security_key() == security.id()) { m.PrintDebugString(); break; } } } } }); } void run_tail(const vector& args) { std::set products; auto conf = parse_md_args(args); bool monitor_all = false; for (const auto& b : args) { if (b=="all") monitor_all = true; } PlatformConfig config; if (!monitor_all) config.set_type(api::SecurityType_Name(api::SECURITYTYPE_FUTURE)); config.set_platform(co2::Platform_Name(conf.platform)); get_system_md_symbols(config, products); auto mod = conf.number > 0 ? conf.number : 1000; PrintFormat fmt = PrintFormat::CSV; for (const auto& a : args) { if (a == "longform") fmt = PrintFormat::Human; } md::run_periodic_dumper(products, mod, fmt, monitor_all); } void run_channels() { auto channels = md::get_configured_cme_channels(); for (const auto& ch : channels) { std::cout << ch.id << " - " ; std::set symbols; for (const auto& s : ch.symbols) { symbols.insert(s.symbol()); } for (const auto& s : symbols) { std::cout << s << " "; } std::cout << std::endl; } } string help() { return "Market data tools:\n" " record \t-- Record market data to a file.\n" " rectool \t-- Manipulate recorded market data. \n" " cleanup \t-- Remove all persisted market data shared memory sections\n" " dump [exchange] [platform] [hot]\n" " replay \t-- replay recorded market data.\n" " run \t\t-- run market data publishers.\n" " relay \t-- relay market data from another system.\n" " dumpsecurity \t-- print information about a specific security or all securities.\n" " tail [count] \t-- print out books when sequence%count==0.\n" " channels \t-- list configured cme channels.\n" ; } void printhelp() { cout << help() << endl; } void md_tool(std::vector const &args) { const string& cmd = args.empty() ? "" : args[0]; if (args.empty()) printhelp(); else if (cmd == "record") run_record(args.size() > 1 ? args[1] : ""); else if (cmd == "rectool") run_rectool(vector(args.begin()+1, args.end())); else if (cmd == "cleanup") run_cleanup(); else if (cmd == "dump") run_dump(vector(args.begin()+1, args.end())); else if (cmd == "replay") run_replay(); else if (cmd == "run") run_market_data_publishers_forever(nullptr); else if (cmd == "relay") relay_everything(); else if (cmd == "test_relay") relay_everything(); else if (cmd == "dumpsecurity" || cmd == "dumpsecurities") dump_db(vector(args.begin()+1, args.end())); else if (cmd == "tail") run_tail(args); else if (cmd == "channels") run_channels(); else printhelp(); } }}//namespace