From fff7faab908dc0704da108f73b019d6016b8cc3f Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 2 Dec 2020 17:47:56 -0600 Subject: [PATCH] added some diagnostic spartan cmds to track down sim_books mismatches --- src/server/sparkd/cmds/Spartan.cpp | 73 +++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index 9c8016b44..810305a46 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -97,6 +97,37 @@ int sim(std::string const &input_file) { return 0; } +// The cme occassionally recycles order keys within the same trading day +// (despite having 64 bit keys), so we are forced remap the order keys +// prevent collisions during simulations, as the lifetime of an order +// may be longer in a simualtion than it was during the actual trading day. +// This method simply prints our internal mapping to a csv file, +// for reference in the sqlite db. The first row contains the column names. +int sim_key_map_to_csv(const std::string& sim_input, const std::string& out_csv) { + std::ifstream f(sim_input, std::ios::binary); + spartan::protos::SimInput input_proto; + if (!input_proto.ParseFromIstream(&f)) { + std::cerr << "Failed to deserialize simulator input from " << sim_input + << std::endl; + return 1; + } else { + auto sim_input = make_sim_input_from_proto(input_proto); + std::unordered_map> key_map; + for (auto &rev : sim_input.transactions) { + for (auto& ch : rev.order_changes) + key_map.emplace(ch.original_key, std::unordered_set()) + .first->second.emplace(ch.order.key); + } + auto output = std::ofstream{out_csv}; + output << "exch_ok," << "sim_ok"; + for (const auto& p : key_map) { + for (auto k : p.second) + output << std::endl << p.first.value() << "," << k.value(); + } + return 0; + } +} + void find_polys(std::vector const &args) { PolygonFinder::Options options; if (args.size() > 2) @@ -203,6 +234,33 @@ void sim_books(std::string const &sim_file) { } } +int icebergs_to_csv(const std::unordered_map& ibs, const std::string& out_csv) { + const char* eid_col = "eid"; + const char* ok_col = "ok"; + const char* total_col = "total"; + const char* show_qty_col = "show_qty"; + const char* fills_col = "aggr_fills"; + //const char* log_header = "IcebergsCSV"; + // the nth leg col denotes the last event in which the nth leg's book changed + // validate args + if (out_csv.empty()) + return 1; + // write icebergs to csv + auto ofs = std::ofstream(out_csv); + // write cols + ofs << eid_col << "," << ok_col << "," << show_qty_col << "," << total_col << "," << fills_col; + // write rows + for (auto& p : ibs) { + for (auto [eid, show_qty] : p.second.show_quantities()) { + ofs << std::endl + << eid << "," << p.first << "," << show_qty << "," << p.second.total_qty() + << "," << p.second.count_fills_by(eid); + } + } + ofs.flush(); + return 0; +} + int polygon_books(const std::string& pcap, const std::string& db, const std::string& id_string, const std::string& out_csv) { const char* eid_col = "eid"; const char* pid_col = "pid"; @@ -358,7 +416,13 @@ int Spartan::execute(std::vector const &args) { std::atoi(args[3].c_str())); } else if (args[0] == "compress" && args.size() == 3) error = spartan::mdp3::compress(args[1], args[2]); - else if (args[0] == "csv" && args.size() == 2) + else if (args[0] == "icebergs_csv" && args.size() >= 2) { + auto out_csv = args.size() > 2 ? args[2] + : bts::spartan::file::change_file_ext( + args[1], "icebergs.csv", "pcap.xz"); + auto ibs = iceberg_file_cache::find_icebergs(args[1]); + return icebergs_to_csv(ibs, out_csv); + } else if (args[0] == "csv" && args.size() == 2) error = spartan::mdp3::to_csv(args[1]); else if (args[0] == "sql" && args.size() == 2) error = spartan::mdp3::to_sql(args[1]); @@ -367,6 +431,11 @@ int Spartan::execute(std::vector const &args) { : bts::spartan::file::change_file_ext( args[1], "polygon_books.csv", "pcap.xz"); error = polygon_books(args[1], args[2], args[3], out_csv); + } else if (args[0] == "sim_key_map_csv" && args.size() >= 2) { + auto out_csv = args.size() > 2 ? args[2] + : bts::spartan::file::change_file_ext(args[1], "sim_key_map.csv"); + error = sim_key_map_to_csv(args[1], out_csv); + return error; } else if (extensions.count(args[0]) && args.size() >= 2) { return extensions[args[0]](args); } @@ -374,7 +443,7 @@ int Spartan::execute(std::vector const &args) { if (error) std::cout << help(); return error; -} // namespace cmd +} std::string Spartan::help() { auto str = fmt::format("\tjson [.xz] [start_packet packet_count]\n"