373 lines
12 KiB
C++
373 lines
12 KiB
C++
#include "run_md.h"
|
|
#include "lib/md/relay/RelayRecorder.h"
|
|
#include "lib/md/PlatformLoader.h"
|
|
#include "lib/utils/bts_assert.h"
|
|
#include "lib/config/config.h"
|
|
#include "lib/config/protos/MarketData.pb.h"
|
|
#include "lib/logging/logging.h"
|
|
#include "lib/md/get_symbols.h"
|
|
#include "lib/md/UserSymbol.h"
|
|
#include "lib/md/Diag/ShmDumper.h"
|
|
#include "lib/md/Diag/MDPeriodicDumper.h"
|
|
#include "lib/md/cme/MDP3Utils.h"
|
|
#include "platforms/mdp3_onixs/MDP3Main.h"
|
|
#include "platforms/ice_onixs/ICEMain.h"
|
|
#include "platforms/IMDPlatform.h"
|
|
#include "lib/md/SecurityDB.h"
|
|
#include "server/sparkd/exchange_sim/ExchangeSim.h"
|
|
#include "lib/utils/thread_manager.h"
|
|
|
|
|
|
#include <boost/filesystem.hpp>
|
|
#include <iostream>
|
|
#include <thread>
|
|
#include <memory>
|
|
#include <set>
|
|
#include <optional>
|
|
|
|
using namespace bts;
|
|
using namespace std;
|
|
|
|
namespace bts {
|
|
namespace md {
|
|
|
|
|
|
string default_arg(const vector<std::string>& args, size_t index, const char* default_value="") {
|
|
if (index < args.size()) return args[index];
|
|
return default_value;
|
|
}
|
|
|
|
void clean_dir(const std::string base_path) {
|
|
LOG(md, Info, "Cleaning shared memory");
|
|
namespace fs = boost::filesystem;
|
|
const fs::path p(base_path);
|
|
auto file_prefix = "bts_edge_" + config::get<PublisherNamespace>().shared_memory_namespace() + "_";
|
|
if (fs::is_directory(p)) {
|
|
fs::directory_iterator end_iter;
|
|
for (fs::directory_iterator dir_iter(p); dir_iter != end_iter; ++dir_iter) {
|
|
if (fs::is_regular_file(dir_iter->status())) {
|
|
auto name = dir_iter->path().filename().string();
|
|
if (name.find(file_prefix) != std::string::npos) {
|
|
boost::system::error_code err;
|
|
fs::remove(dir_iter->path(), err);
|
|
if (!err) LOG(md, Info, "Cleaned up {}", dir_iter->path());
|
|
}
|
|
}
|
|
else if (fs::is_directory(dir_iter->status())) {
|
|
std::string path = dir_iter->path().string();
|
|
clean_dir(path);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void run_replay() {
|
|
auto naggle_ms = config::get<RelayOptions>().naggle_ms();
|
|
const auto& options = config::get<ReplayOptions>();
|
|
auto file = options.url();
|
|
auto fallback_path = options.fallback();
|
|
replay_recording(file, naggle_ms, fallback_path);
|
|
}
|
|
|
|
void run_onixs(MBOWriter* mbo) {
|
|
std::vector<std::shared_ptr<IMDPlatform>> platforms;
|
|
// get all glbx symbols
|
|
std::vector<UserSymbol> cme_symbols;
|
|
get_system_md_symbols(api::VENUE_GLBX, api::SECURITYTYPE_UNSET, &cme_symbols);
|
|
if(cme_symbols.size() > 0)
|
|
{
|
|
MDP3Main* main = new MDP3Main();
|
|
std::shared_ptr<IMDPlatform> ptr(main);
|
|
main->start(cme_symbols, mbo);
|
|
platforms.push_back(ptr);
|
|
}
|
|
|
|
std::vector<UserSymbol> ice_symbols;
|
|
get_system_md_symbols(api::VENUE_ICE, api::SECURITYTYPE_UNSET, &ice_symbols);
|
|
if(ice_symbols.size() > 0)
|
|
{
|
|
ice::ICEMain* imain = new ice::ICEMain();
|
|
std::shared_ptr<IMDPlatform> ptr(imain);
|
|
imain->start(ice_symbols);
|
|
platforms.push_back(ptr);
|
|
}
|
|
|
|
//
|
|
// Timers
|
|
auto& disp = utils::Dispatcher::current();
|
|
utils::DispatcherTimer heartbeat_timer(disp, 250, [&]()
|
|
{
|
|
for (auto& p: platforms)
|
|
p->send_heartbeats();
|
|
});
|
|
heartbeat_timer.start();
|
|
|
|
disp.run();
|
|
}
|
|
|
|
void run_cleanup() {
|
|
clean_dir("/dev/shm");
|
|
}
|
|
|
|
|
|
std::optional<std::thread> _md_thread;
|
|
void run_market_data_publishers_forever(MBOWriter* mbo) {
|
|
const auto& conf = config::get<MarketDataConfig>();
|
|
if (conf.use_replay()) {
|
|
run_replay();
|
|
}
|
|
else if (conf.use_onixs()) {
|
|
run_onixs(mbo);
|
|
}
|
|
}
|
|
|
|
void boot_sleep() {
|
|
auto conf = config::get<MDBootDelayWindow>();
|
|
const auto* schedule = &conf.standard();
|
|
if (datetime::is_local_time_sunday()) {
|
|
schedule = &conf.sunday();
|
|
}
|
|
bts_assert(schedule);
|
|
int hh_start = 0, mm_start = 0, hh_end = 0, mm_end = 0;
|
|
sscanf(schedule->start().c_str(), "%i:%i", &hh_start, &mm_start);
|
|
sscanf(schedule->end().c_str(), "%i:%i", &hh_end, &mm_end);
|
|
if (datetime::is_local_time_between(hh_start, mm_start, hh_end, mm_end)) {
|
|
std::this_thread::sleep_for(
|
|
std::chrono::seconds(
|
|
conf.delay_seconds()));
|
|
}
|
|
else {
|
|
std::this_thread::sleep_for(2s);
|
|
}
|
|
}
|
|
|
|
void run_market_datapublishers(MBOWriter* mbo) {
|
|
bts_assert(_md_thread == std::nullopt);
|
|
run_cleanup();
|
|
if (!config::get<MarketDataConfig>().use_md()) return;
|
|
// start a thread for the md
|
|
_md_thread = std::optional(
|
|
threading::make_thread("md publisher", [mbo]() {
|
|
run_market_data_publishers_forever(mbo);
|
|
}));
|
|
|
|
if (config::get<MarketDataConfig>().use_sim()) {
|
|
threading::make_thread("ExchangeSimGLBX", [mbo]() { sim::start_exchange_sim(api::Venue::VENUE_SIM, mbo); }).detach();
|
|
//threading::make_thread("ExchangeSimICE", [mbo]() { sim::start_exchange_sim(api::Venue::VENUE_SIMICE, mbo); }).detach();
|
|
//TODO I'll need Eric's help to disambiguate replay / relay here
|
|
}
|
|
boot_sleep();
|
|
}
|
|
|
|
|
|
void run_record(const std::string& arg){
|
|
boost::filesystem::path p(arg);
|
|
record_everything(p.parent_path().string(), p.filename().string());
|
|
}
|
|
|
|
string& uppercase(string& s) {
|
|
transform(s.begin(), s.end(), s.begin(), ::toupper);
|
|
return s;
|
|
}
|
|
template <class T, class T2>
|
|
bool parse_enum(const string& value, const string& prefix, T2 parse, T default_value, T* presult) {
|
|
string temp = value;
|
|
if (temp.find(prefix) == string::npos) {
|
|
temp = prefix + "_" + temp;
|
|
}
|
|
if(parse(temp, presult)) {
|
|
return true;
|
|
}
|
|
else {
|
|
*presult = default_value;
|
|
return false;
|
|
}
|
|
}
|
|
|
|
bool int_of_string(const std::string& str, long* result) {
|
|
try {
|
|
size_t pos = 0;
|
|
int i = std::stol(str, &pos);
|
|
if (pos == str.size()) {
|
|
*result = i;
|
|
return true;
|
|
}
|
|
}
|
|
catch(const std::exception&) { }
|
|
return false;
|
|
}
|
|
|
|
typedef std::string Symbol;
|
|
struct MDArgs {
|
|
api::Venue venue;
|
|
co2::Platform platform;
|
|
api::SecurityType type;
|
|
Symbol symbol;
|
|
bool hot;
|
|
long number;
|
|
};
|
|
MDArgs parse_md_args(const vector<string>& args) {
|
|
using namespace api;
|
|
using namespace co2;
|
|
auto venue = VENUE_UNSET; // Default to glbx. If you change this change below (used) also
|
|
auto platform = PLATFORM_UNSET;
|
|
auto type = SECURITYTYPE_UNSET;
|
|
bool hot = false;
|
|
long number = 0;
|
|
std::string symbol;
|
|
for (auto str : args) {
|
|
uppercase(str);
|
|
bool used =
|
|
(venue == VENUE_UNSET && parse_enum(str, "VENUE", Venue_Parse, venue, &venue)) ||
|
|
(platform == PLATFORM_UNSET && parse_enum(str, "PLATFORM", Platform_Parse, platform, &platform)) ||
|
|
parse_enum(str, "SECURITYTYPE", SecurityType_Parse, type, &type);
|
|
if (str == "HOT") { hot = true; used = true; }
|
|
else if (str == "COLD") { hot = false; used = true; }
|
|
else {
|
|
used |= int_of_string(str, &number);
|
|
}
|
|
if (!used) {
|
|
symbol = str;
|
|
}
|
|
}
|
|
if (venue == VENUE_UNSET) venue = VENUE_GLBX;
|
|
if (platform == PLATFORM_UNSET) platform = platform_for_exchange(venue);
|
|
return { venue, platform, type, symbol, hot, number };
|
|
}
|
|
|
|
void run_dump(const vector<string>& args) {
|
|
using std::get;
|
|
|
|
auto p = parse_md_args(args);
|
|
stringstream ss;
|
|
ss << "bts_edge_" << config::get<PublisherNamespace>().shared_memory_namespace() << "_"
|
|
<< uppercase(p.symbol) << "_"
|
|
<< SecurityType_Name(p.type) << "_"
|
|
<< Venue_Name(p.venue) << "_"
|
|
<< Platform_Name(p.platform);
|
|
|
|
ShmDumper dumper(ss.str());
|
|
if (p.hot) {
|
|
dumper.dump_hot();
|
|
}
|
|
else {
|
|
dumper.dump_cold();
|
|
}
|
|
}
|
|
|
|
|
|
void run_rectool(const vector<string> args) {
|
|
if (args.size() < 2) {
|
|
cout << "rectool use: " << endl;
|
|
cout << "\tbtscmd md rectool print <filepath>" << endl;
|
|
cout << "\tbtscmd md rectool unprint <input> <output>" << endl;
|
|
cout << "\tbtscmd md rectool trace <filepath> <security_key>" << endl;
|
|
return;
|
|
}
|
|
if (args[0] == "print") {
|
|
print_recording(args[1]);
|
|
} else
|
|
if (args[0] == "unprint") {
|
|
unprint_recording(args[1], args[2]);
|
|
} else
|
|
if (args[0] == "trace") {
|
|
trace_recording(args[1], atoi(args[2].c_str()));
|
|
}
|
|
}
|
|
|
|
void dump_db(const vector<string>& args) {
|
|
set<security_key> keys;
|
|
bool all = false;
|
|
for (const auto& arg : args) {
|
|
if (arg == "all") {
|
|
all = true; break;
|
|
}
|
|
auto key = std::stoul(arg);
|
|
if (key > 0) {
|
|
keys.insert((uint)key);
|
|
}
|
|
}
|
|
std::set<md::UserSymbol> symbols;
|
|
md::get_system_md_symbols(symbols);
|
|
md::db::load_securities(symbols, [&](const api::Security& security) {
|
|
if (all || keys.find(security.id()) != keys.end()) {
|
|
security.PrintDebugString();
|
|
if (md::securitytype_of(security) == api::SECURITYTYPE_SYNTHETIC) {
|
|
auto metadata = md::db::load_metadata();
|
|
for (const auto &m : metadata) {
|
|
if(m.security_key() == security.id()) {
|
|
m.PrintDebugString();
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
void run_tail(const vector<string>& args) {
|
|
std::set<UserSymbol> products;
|
|
auto conf = parse_md_args(args);
|
|
bool monitor_all = false;
|
|
for (const auto& b : args) { if (b=="all") monitor_all = true; }
|
|
|
|
PlatformConfig config;
|
|
if (!monitor_all) config.set_type(api::SecurityType_Name(api::SECURITYTYPE_FUTURE));
|
|
config.set_platform(co2::Platform_Name(conf.platform));
|
|
get_system_md_symbols(config, products);
|
|
auto mod = conf.number > 0 ? conf.number : 1000;
|
|
PrintFormat fmt = PrintFormat::CSV;
|
|
for (const auto& a : args) { if (a == "longform") fmt = PrintFormat::Human; }
|
|
md::run_periodic_dumper(products, mod, fmt, monitor_all);
|
|
}
|
|
|
|
void run_channels() {
|
|
auto channels = md::get_configured_cme_channels();
|
|
for (const auto& ch : channels) {
|
|
std::cout << ch.id << " - " ;
|
|
std::set<std::string> symbols;
|
|
for (const auto& s : ch.symbols) { symbols.insert(s.symbol()); }
|
|
for (const auto& s : symbols) {
|
|
std::cout << s << " ";
|
|
}
|
|
std::cout << std::endl;
|
|
}
|
|
}
|
|
|
|
string help() {
|
|
return "Market data tools:\n"
|
|
" record \t-- Record market data to a file.\n"
|
|
" rectool \t-- Manipulate recorded market data. \n"
|
|
" cleanup \t-- Remove all persisted market data shared memory sections\n"
|
|
" dump <symbol> <type> [exchange] [platform] [hot]\n"
|
|
" replay \t-- replay recorded market data.\n"
|
|
" run \t\t-- run market data publishers.\n"
|
|
" relay \t-- relay market data from another system.\n"
|
|
" dumpsecurity <id|all> \t-- print information about a specific security or all securities.\n"
|
|
" tail [count] \t-- print out books when sequence%count==0.\n"
|
|
" channels \t-- list configured cme channels.\n"
|
|
;
|
|
}
|
|
void printhelp() {
|
|
cout << help() << endl;
|
|
}
|
|
|
|
void md_tool(std::vector<std::string> const &args) {
|
|
const string& cmd = args.empty() ? "" : args[0];
|
|
if (args.empty()) printhelp();
|
|
else if (cmd == "record") run_record(args.size() > 1 ? args[1] : "");
|
|
else if (cmd == "rectool") run_rectool(vector<string>(args.begin()+1, args.end()));
|
|
else if (cmd == "cleanup") run_cleanup();
|
|
else if (cmd == "dump") run_dump(vector<string>(args.begin()+1, args.end()));
|
|
else if (cmd == "replay") run_replay();
|
|
else if (cmd == "run") run_market_data_publishers_forever(nullptr);
|
|
else if (cmd == "relay") relay_everything();
|
|
else if (cmd == "test_relay") relay_everything();
|
|
else if (cmd == "dumpsecurity" || cmd == "dumpsecurities") dump_db(vector<string>(args.begin()+1, args.end()));
|
|
else if (cmd == "tail") run_tail(args);
|
|
else if (cmd == "channels") run_channels();
|
|
else printhelp();
|
|
}
|
|
|
|
}}//namespace
|