Files
spark/src/server/sparkd/md/run_md.cpp
T

289 lines
8.6 KiB
C++

#include "run_md.h"
#include "lib/md/relay/RelayRecorder.h"
#include "lib/md/PlatformLoader.h"
#include "lib/utils/bts_assert.h"
#include "lib/config/config.h"
#include "lib/config/protos/MarketData.pb.h"
#include "lib/logging/logging.h"
#include "lib/md/get_symbols.h"
#include "lib/md/UserSymbol.h"
#include "lib/md/Diag/ShmDumper.h"
#include "lib/md/Diag/MDPeriodicDumper.h"
#include "platforms/mdp3_onixs/MDP3Main.h"
#include "lib/md/SecurityDB.h"
#include "server/sparkd/exchange_sim/ExchangeSim.h"
#include "lib/utils/thread_manager.h"
#include <boost/filesystem.hpp>
#include <iostream>
#include <thread>
#include <set>
using namespace bts;
using namespace std;
namespace bts {
namespace md {
string default_arg(const vector<std::string>& args, size_t index, const char* default_value="") {
if (index < args.size()) return args[index];
return default_value;
}
void clean_dir(const std::string base_path) {
LOG(md, Info, "Cleaning shared memory");
namespace fs = boost::filesystem;
const fs::path p(base_path);
auto file_prefix = "bts_edge_" + config::get<PublisherNamespace>().shared_memory_namespace() + "_";
if (fs::is_directory(p)) {
fs::directory_iterator end_iter;
for (fs::directory_iterator dir_iter(p); dir_iter != end_iter; ++dir_iter) {
if (fs::is_regular_file(dir_iter->status())) {
auto name = dir_iter->path().filename().string();
if (name.find(file_prefix) != std::string::npos) {
boost::system::error_code err;
fs::remove(dir_iter->path(), err);
if (!err) LOG(md, Info, "Cleaned up {}", dir_iter->path());
}
}
else if (fs::is_directory(dir_iter->status())) {
std::string path = dir_iter->path().string();
clean_dir(path);
}
}
}
}
void run_replay() {
auto naggle_ms = config::get<RelayOptions>().naggle_ms();
auto file = "http://mdreplay.bluetradesys.local/spark/futures.coconut";
replay_recording(file, naggle_ms);
}
void run_onixs() {
// get all glbx symbols
std::vector<UserSymbol> symbols;
get_system_md_symbols(api::VENUE_GLBX, api::SECURITYTYPE_UNSET, &symbols);
MDP3Main main;
main.run(symbols);
}
void run_cleanup() {
clean_dir("/dev/shm");
}
std::thread* _md_thread;
void run_market_data_publishers_forever() {
os::set_thread_name("md publisher");
if (config::get<MarketDataConfig>().use_replay()) {
run_replay();
}
else {
run_onixs();
}
}
void run_market_datapublishers() {
bts_assert(_md_thread == nullptr);
run_cleanup();
if (!config::get<MarketDataConfig>().use_md()) return;
// start a thread for the md
_md_thread = new std::thread([]() {
run_market_data_publishers_forever();
});
if (config::get<MarketDataConfig>().use_sim()) {
threading::make_thread("ExchangeSim", []() { sim::start_exchange_sim(); })
.detach();
}
std::this_thread::sleep_for(2s);
}
void run_record(const std::string& arg){
boost::filesystem::path p(arg);
record_everything(p.parent_path().string(), p.filename().string());
}
string& uppercase(string& s) {
transform(s.begin(), s.end(), s.begin(), ::toupper);
return s;
}
template <class T, class T2>
bool parse_enum(const string& value, const string& prefix, T2 parse, T default_value, T* presult) {
string temp = value;
if (temp.find(prefix) == string::npos) {
temp = prefix + "_" + temp;
}
if(parse(temp, presult)) {
return true;
}
else {
*presult = default_value;
return false;
}
}
bool int_of_string(const std::string& str, long* result) {
try {
size_t pos = 0;
int i = std::stol(str, &pos);
if (pos == str.size()) {
*result = i;
return true;
}
}
catch(const std::exception&) { }
return false;
}
typedef std::string Symbol;
struct MDArgs {
api::Venue venue;
co2::Platform platform;
api::SecurityType type;
Symbol symbol;
bool hot;
long number;
};
MDArgs parse_md_args(const vector<string>& args) {
using namespace api;
using namespace co2;
auto venue = VENUE_UNSET; // Default to glbx. If you change this change below (used) also
auto platform = PLATFORM_UNSET;
auto type = SECURITYTYPE_UNSET;
bool hot = false;
long number = 0;
std::string symbol;
for (auto str : args) {
uppercase(str);
bool used =
(venue == VENUE_UNSET && parse_enum(str, "VENUE", Venue_Parse, venue, &venue)) ||
(platform == PLATFORM_UNSET && parse_enum(str, "PLATFORM", Platform_Parse, platform, &platform)) ||
parse_enum(str, "SECURITYTYPE", SecurityType_Parse, type, &type);
if (str == "HOT") { hot = true; used = true; }
else if (str == "COLD") { hot = false; used = true; }
else {
used |= int_of_string(str, &number);
}
if (!used) {
symbol = str;
}
}
if (venue == VENUE_UNSET) venue = VENUE_GLBX;
if (platform == PLATFORM_UNSET) platform = platform_for_exchange(venue);
return { venue, platform, type, symbol, hot, number };
}
void run_dump(const vector<string>& args) {
using std::get;
auto p = parse_md_args(args);
stringstream ss;
ss << "bts_edge_" << config::get<PublisherNamespace>().shared_memory_namespace() << "_"
<< uppercase(p.symbol) << "_"
<< SecurityType_Name(p.type) << "_"
<< Venue_Name(p.venue) << "_"
<< Platform_Name(p.platform);
ShmDumper dumper(ss.str());
if (p.hot) {
dumper.dump_hot();
}
else {
dumper.dump_cold();
}
}
void run_rectool(const vector<string> args) {
if (args.size() < 2) {
cout << "rectool use: " << endl;
cout << "\tbtscmd md rectool print <filepath>" << endl;
cout << "\tbtscmd md rectool unprint <input> <output>" << endl;
cout << "\tbtscmd md rectool trace <filepath> <security_key>" << endl;
return;
}
if (args[0] == "print") {
print_recording(args[1]);
} else
if (args[0] == "unprint") {
unprint_recording(args[1], args[2]);
} else
if (args[0] == "trace") {
trace_recording(args[1], atoi(args[2].c_str()));
}
}
void dump_db(const vector<string>& args) {
set<security_key> keys;
bool all = false;
for (const auto& arg : args) {
if (arg == "all") {
all = true; break;
}
auto key = std::stoi(arg);
if (key > 0) {
keys.insert(key);
}
}
std::set<md::UserSymbol> symbols;
md::get_system_md_symbols(symbols);
md::db::load_securities(symbols, [&](const api::Security& security) {
if (all || keys.find(security.id()) != keys.end()) {
security.PrintDebugString();
if (md::securitytype_of(security) == api::SECURITYTYPE_SYNTHETIC) {
auto metadata = md::db::load_metadata();
for (const auto &m : metadata) {
if(m.security_key() == security.id()) {
m.PrintDebugString();
break;
}
}
}
}
});
}
void run_tail(const vector<string>& args) {
std::set<UserSymbol> products;
auto conf = parse_md_args(args);
PlatformConfig config;
config.set_type(api::SecurityType_Name(api::SECURITYTYPE_FUTURE));
config.set_platform(co2::Platform_Name(conf.platform));
get_system_md_symbols(config, products);
auto mod = conf.number > 0 ? conf.number : 1000;
PrintFormat fmt = PrintFormat::CSV;
for (const auto& a : args) { if (a == "longform") fmt = PrintFormat::Human; }
md::run_periodic_dumper(products, mod, fmt);
}
string help() {
return "Coconut 2.0 market data tools:\n" ;
}
void printhelp() {
cout << help() << endl;
}
void md_tool(std::vector<std::string> const &args) {
const string& cmd = args.empty() ? "" : args[0];
if (args.empty()) printhelp();
else if (cmd == "record") run_record(args.size() > 1 ? args[1] : "");
else if (cmd == "rectool") run_rectool(vector<string>(args.begin()+1, args.end()));
else if (cmd == "cleanup") run_cleanup();
else if (cmd == "dump") run_dump(vector<string>(args.begin()+1, args.end()));
else if (cmd == "replay") run_replay();
else if (cmd == "run") run_market_data_publishers_forever();
else if (cmd == "relay") relay_everything();
else if (cmd == "test_relay") relay_everything();
else if (cmd == "dumpsecurity" || cmd == "dumpsecurities") dump_db(vector<string>(args.begin()+1, args.end()));
else if (cmd == "tail") run_tail(args);
else printhelp();
}
}}//namespace