#include "run_md.h" #include "lib/md/relay/RelayRecorder.h" #include "lib/md/PlatformLoader.h" #include "lib/utils/bts_assert.h" #include "lib/config/config.h" #include "lib/config/protos/MarketData.pb.h" #include "lib/logging/logging.h" #include "lib/md/get_symbols.h" #include "lib/md/UserSymbol.h" #include "lib/md/Diag/ShmDumper.h" #include "lib/md/Diag/MDPeriodicDumper.h" #include "platforms/mdp3_onixs/MDP3Main.h" #include "lib/md/SecurityDB.h" #include "server/sparkd/exchange_sim/ExchangeSim.h" #include "lib/utils/thread_manager.h" #include #include #include #include using namespace bts; using namespace std; namespace bts { namespace md { string default_arg(const vector& args, size_t index, const char* default_value="") { if (index < args.size()) return args[index]; return default_value; } void clean_dir(const std::string base_path) { LOG(md, Info, "Cleaning shared memory"); namespace fs = boost::filesystem; const fs::path p(base_path); auto file_prefix = "bts_edge_" + config::get().shared_memory_namespace() + "_"; if (fs::is_directory(p)) { fs::directory_iterator end_iter; for (fs::directory_iterator dir_iter(p); dir_iter != end_iter; ++dir_iter) { if (fs::is_regular_file(dir_iter->status())) { auto name = dir_iter->path().filename().string(); if (name.find(file_prefix) != std::string::npos) { boost::system::error_code err; fs::remove(dir_iter->path(), err); if (!err) LOG(md, Info, "Cleaned up {}", dir_iter->path()); } } else if (fs::is_directory(dir_iter->status())) { std::string path = dir_iter->path().string(); clean_dir(path); } } } } void run_replay() { auto naggle_ms = config::get().naggle_ms(); auto file = "http://mdreplay.bluetradesys.local/spark/futures.coconut"; replay_recording(file, naggle_ms); } void run_onixs() { // get all glbx symbols std::vector symbols; get_system_md_symbols(api::VENUE_GLBX, api::SECURITYTYPE_UNSET, &symbols); MDP3Main main; main.run(symbols); } void run_cleanup() { clean_dir("/dev/shm"); } std::thread* _md_thread; void run_market_data_publishers_forever() { os::set_thread_name("md publisher"); if (config::get().use_replay()) { run_replay(); } else { run_onixs(); } } void run_market_datapublishers() { bts_assert(_md_thread == nullptr); run_cleanup(); if (!config::get().use_md()) return; // start a thread for the md _md_thread = new std::thread([]() { run_market_data_publishers_forever(); }); if (config::get().use_sim()) { threading::make_thread("ExchangeSim", []() { sim::start_exchange_sim(); }) .detach(); } std::this_thread::sleep_for(2s); } void run_record(const std::string& arg){ boost::filesystem::path p(arg); record_everything(p.parent_path().string(), p.filename().string()); } string& uppercase(string& s) { transform(s.begin(), s.end(), s.begin(), ::toupper); return s; } template bool parse_enum(const string& value, const string& prefix, T2 parse, T default_value, T* presult) { string temp = value; if (temp.find(prefix) == string::npos) { temp = prefix + "_" + temp; } if(parse(temp, presult)) { return true; } else { *presult = default_value; return false; } } bool int_of_string(const std::string& str, long* result) { try { size_t pos = 0; int i = std::stol(str, &pos); if (pos == str.size()) { *result = i; return true; } } catch(const std::exception&) { } return false; } typedef std::string Symbol; struct MDArgs { api::Venue venue; co2::Platform platform; api::SecurityType type; Symbol symbol; bool hot; long number; }; MDArgs parse_md_args(const vector& args) { using namespace api; using namespace co2; auto venue = VENUE_UNSET; // Default to glbx. If you change this change below (used) also auto platform = PLATFORM_UNSET; auto type = SECURITYTYPE_UNSET; bool hot = false; long number = 0; std::string symbol; for (auto str : args) { uppercase(str); bool used = (venue == VENUE_UNSET && parse_enum(str, "VENUE", Venue_Parse, venue, &venue)) || (platform == PLATFORM_UNSET && parse_enum(str, "PLATFORM", Platform_Parse, platform, &platform)) || parse_enum(str, "SECURITYTYPE", SecurityType_Parse, type, &type); if (str == "HOT") { hot = true; used = true; } else if (str == "COLD") { hot = false; used = true; } else { used |= int_of_string(str, &number); } if (!used) { symbol = str; } } if (venue == VENUE_UNSET) venue = VENUE_GLBX; if (platform == PLATFORM_UNSET) platform = platform_for_exchange(venue); return { venue, platform, type, symbol, hot, number }; } void run_dump(const vector& args) { using std::get; auto p = parse_md_args(args); stringstream ss; ss << "bts_edge_" << config::get().shared_memory_namespace() << "_" << uppercase(p.symbol) << "_" << SecurityType_Name(p.type) << "_" << Venue_Name(p.venue) << "_" << Platform_Name(p.platform); ShmDumper dumper(ss.str()); if (p.hot) { dumper.dump_hot(); } else { dumper.dump_cold(); } } void run_rectool(const vector args) { if (args.size() < 2) { cout << "rectool use: " << endl; cout << "\tbtscmd md rectool print " << endl; cout << "\tbtscmd md rectool unprint " << endl; cout << "\tbtscmd md rectool trace " << endl; return; } if (args[0] == "print") { print_recording(args[1]); } else if (args[0] == "unprint") { unprint_recording(args[1], args[2]); } else if (args[0] == "trace") { trace_recording(args[1], atoi(args[2].c_str())); } } void dump_db(const vector& args) { set keys; bool all = false; for (const auto& arg : args) { if (arg == "all") { all = true; break; } auto key = std::stoi(arg); if (key > 0) { keys.insert(key); } } std::set symbols; md::get_system_md_symbols(symbols); md::db::load_securities(symbols, [&](const api::Security& security) { if (all || keys.find(security.id()) != keys.end()) { security.PrintDebugString(); if (md::securitytype_of(security) == api::SECURITYTYPE_SYNTHETIC) { auto metadata = md::db::load_metadata(); for (const auto &m : metadata) { if(m.security_key() == security.id()) { m.PrintDebugString(); break; } } } } }); } void run_tail(const vector& args) { std::set products; auto conf = parse_md_args(args); PlatformConfig config; config.set_type(api::SecurityType_Name(api::SECURITYTYPE_FUTURE)); config.set_platform(co2::Platform_Name(conf.platform)); get_system_md_symbols(config, products); auto mod = conf.number > 0 ? conf.number : 1000; PrintFormat fmt = PrintFormat::CSV; for (const auto& a : args) { if (a == "longform") fmt = PrintFormat::Human; } md::run_periodic_dumper(products, mod, fmt); } string help() { return "Coconut 2.0 market data tools:\n" ; } void printhelp() { cout << help() << endl; } void md_tool(std::vector const &args) { const string& cmd = args.empty() ? "" : args[0]; if (args.empty()) printhelp(); else if (cmd == "record") run_record(args.size() > 1 ? args[1] : ""); else if (cmd == "rectool") run_rectool(vector(args.begin()+1, args.end())); else if (cmd == "cleanup") run_cleanup(); else if (cmd == "dump") run_dump(vector(args.begin()+1, args.end())); else if (cmd == "replay") run_replay(); else if (cmd == "run") run_market_data_publishers_forever(); else if (cmd == "relay") relay_everything(); else if (cmd == "test_relay") relay_everything(); else if (cmd == "dumpsecurity" || cmd == "dumpsecurities") dump_db(vector(args.begin()+1, args.end())); else if (cmd == "tail") run_tail(args); else printhelp(); } }}//namespace