added some diagnostic spartan cmds to track down sim_books mismatches

This commit is contained in:
John Colvin committed 2020-12-03 17:11:34 -06:00
1 parent e4c36c8470
commit fff7faab90
1 file changed
+71 -2
+71 -2
View File
@@ -97,6 +97,37 @@ int sim(std::string const &input_file) {
return 0;
}
// The cme occassionally recycles order keys within the same trading day
// (despite having 64 bit keys), so we are forced remap the order keys
// prevent collisions during simulations, as the lifetime of an order
// may be longer in a simualtion than it was during the actual trading day.
// This method simply prints our internal mapping to a csv file,
// for reference in the sqlite db. The first row contains the column names.
int sim_key_map_to_csv(const std::string& sim_input, const std::string& out_csv) {
std::ifstream f(sim_input, std::ios::binary);
spartan::protos::SimInput input_proto;
if (!input_proto.ParseFromIstream(&f)) {
std::cerr << "Failed to deserialize simulator input from " << sim_input
<< std::endl;
return 1;
} else {
auto sim_input = make_sim_input_from_proto(input_proto);
std::unordered_map<MDP3OrderKey, std::unordered_set<MDP3OrderKey>> key_map;
for (auto &rev : sim_input.transactions) {
for (auto& ch : rev.order_changes)
key_map.emplace(ch.original_key, std::unordered_set<MDP3OrderKey>())
.first->second.emplace(ch.order.key);
}
auto output = std::ofstream{out_csv};
output << "exch_ok," << "sim_ok";
for (const auto& p : key_map) {
for (auto k : p.second)
output << std::endl << p.first.value() << "," << k.value();
}
return 0;
}
}
void find_polys(std::vector<std::string> const &args) {
PolygonFinder::Options options;
if (args.size() > 2)
@@ -203,6 +234,33 @@ void sim_books(std::string const &sim_file) {
}
}
int icebergs_to_csv(const std::unordered_map<MDP3OrderKey, IcebergInput>& ibs, const std::string& out_csv) {
const char* eid_col = "eid";
const char* ok_col = "ok";
const char* total_col = "total";
const char* show_qty_col = "show_qty";
const char* fills_col = "aggr_fills";
//const char* log_header = "IcebergsCSV";
// the nth leg col denotes the last event in which the nth leg's book changed
// validate args
if (out_csv.empty())
return 1;
// write icebergs to csv
auto ofs = std::ofstream(out_csv);
// write cols
ofs << eid_col << "," << ok_col << "," << show_qty_col << "," << total_col << "," << fills_col;
// write rows
for (auto& p : ibs) {
for (auto [eid, show_qty] : p.second.show_quantities()) {
ofs << std::endl
<< eid << "," << p.first << "," << show_qty << "," << p.second.total_qty()
<< "," << p.second.count_fills_by(eid);
}
}
ofs.flush();
return 0;
}
int polygon_books(const std::string& pcap, const std::string& db, const std::string& id_string, const std::string& out_csv) {
const char* eid_col = "eid";
const char* pid_col = "pid";
@@ -358,7 +416,13 @@ int Spartan::execute(std::vector<std::string> const &args) {
std::atoi(args[3].c_str()));
} else if (args[0] == "compress" && args.size() == 3)
error = spartan::mdp3::compress(args[1], args[2]);
else if (args[0] == "csv" && args.size() == 2)
else if (args[0] == "icebergs_csv" && args.size() >= 2) {
auto out_csv = args.size() > 2 ? args[2]
: bts::spartan::file::change_file_ext(
args[1], "icebergs.csv", "pcap.xz");
auto ibs = iceberg_file_cache::find_icebergs(args[1]);
return icebergs_to_csv(ibs, out_csv);
} else if (args[0] == "csv" && args.size() == 2)
error = spartan::mdp3::to_csv(args[1]);
else if (args[0] == "sql" && args.size() == 2)
error = spartan::mdp3::to_sql(args[1]);
@@ -367,6 +431,11 @@ int Spartan::execute(std::vector<std::string> const &args) {
: bts::spartan::file::change_file_ext(
args[1], "polygon_books.csv", "pcap.xz");
error = polygon_books(args[1], args[2], args[3], out_csv);
} else if (args[0] == "sim_key_map_csv" && args.size() >= 2) {
auto out_csv = args.size() > 2 ? args[2]
: bts::spartan::file::change_file_ext(args[1], "sim_key_map.csv");
error = sim_key_map_to_csv(args[1], out_csv);
return error;
} else if (extensions.count(args[0]) && args.size() >= 2) {
return extensions[args[0]](args);
}
@@ -374,7 +443,7 @@ int Spartan::execute(std::vector<std::string> const &args) {
if (error)
std::cout << help();
return error;
} // namespace cmd
}
std::string Spartan::help() {
auto str = fmt::format("\tjson <pcap>[.xz] [start_packet packet_count]\n"