#include "Spartan.h" #include "server/sparkd/cmds/spartan/algo/RunTAS.h" #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/PickedFeed.h" #include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h" #include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h" #include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" #include "server/sparkd/cmds/spartan/util/PCapReader.h" #include #include #include #include #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h" #include "server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h" #include "server/sparkd/cmds/spartan/util/PCapReader.h" namespace bts { namespace cmd { using namespace spartan::mdp3; std::map const &)>> extensions = { {"validate_fifo", [](auto const &args) { struct BackTesterHarness : IChannelObserver { std::unordered_map orders; std::unordered_map icebergs; size_t passed_tests = 0; BackTesterHarness(std::string const &pcap) : icebergs(find_icebergs(pcap)) {} void on_recovery_complete(InitialSnapshot const &snap) override { for (auto s : snap) { auto annotated_orders = s->orders(); add_iceberg_annotation(annotated_orders); orders.emplace(s->security_id(), annotated_orders); } } void on_real_time_event(MDP3Event const &ev) override { for (auto s : ev.affected_securities) { auto it = orders.find(s->security_id()); if (it == orders.end()) { bts_assert(s->orders().empty()); it = orders.emplace(s->security_id(), s->orders()).first; } auto &old_orders = it->second; auto deduced = s->deduced_order_changes(); // We have a hard time simulating icebergs, // for now skip them auto found_ice = false; for (auto &f : s->fills()) if (icebergs.count(f.key)) { found_ice = true; break; } if (found_ice) continue; if (deduced.size()) { // We have deduced orders, lets see what the matcher // says add_iceberg_annotation(deduced); FifoSecurityMatcher matcher(ev.event_id,old_orders); auto current_orders = s->orders(); add_iceberg_annotation(current_orders); for (auto &o : deduced) matcher.on_order_change(o); auto error = matcher.has_same_orders(current_orders); if (error) { std::cout << "Failing on event:\n " << ev.event_id << std::endl << "Error: " << error.value() << std::endl << "Deduced orders:\n" << deduced << std::endl << "Old Orders:\n" << old_orders << std::endl << "Order Changes:\n" << s->order_changes() << std::endl << "Fills:\n" << s->fills() << std::endl; bts_assert(false); } old_orders = current_orders; } else { it->second = s->orders(); add_iceberg_annotation(it->second); } } } void add_iceberg_annotation(MDP3Order &order) { auto it = icebergs.find(order.key); if (it != icebergs.end()) order.iceberg_quantity = it->second.estimated_quantity; } void add_iceberg_annotation(Orders &orders) { for (auto &o : orders) add_iceberg_annotation(o.second); } void add_iceberg_annotation(OrderChanges &changes) { for (auto &o : changes) { add_iceberg_annotation(o.previous_order); add_iceberg_annotation(o.order); } } void on_real_time_channel_loss() override { bts_assert(false); } } harness(args[1]); observe_channel(args[1], harness); return 0; }}, {"deduce_orders", [](auto const &args) { struct OrderGen : IChannelObserver { std::ofstream out; OrderGen(std::string const &out_path) : out(out_path) {} void on_recovery_complete(InitialSnapshot const &) override {} void on_real_time_event(MDP3Event const &ev) override { for (auto s : ev.affected_securities) { auto deduced = s->deduced_order_changes(); if (deduced.size()) { out << "sym: " << s->definition().symbol() << " eid: " << ev.event_id << " sn: " << ev.first_packet_sn << " to " << ev.last_packet_sn << " ts: " << to_time_string(ev.transact_time) << std::endl << "Order changes: " << std::endl << s->order_changes() << "Fills: " << std::endl << s->fills() << "Deduced changes: " << std::endl << s->deduced_order_changes() << std::endl << std::endl; } } } void on_real_time_channel_loss() override { bts_assert(false); } } gen(args[2]); observe_channel(args[1], gen); return 0; }}, {"find_icebergs", [](auto const &args) { std::ofstream out(args[2]); for (auto const &[k, ice] : find_icebergs(args[1])) out << "ok: " << k << " sz: " << ice.estimated_quantity.value() << std::endl; return 0; }}, {"gen_sim_input", [](auto const &args) { auto input = make_simulator_input(args[1]); std::cout << "Secs: " << input.instruments.size() << " events: " << input.events.size() << " max_order_id: " << input.max_order_id << std::endl; return 0; }}, }; int Spartan::execute(std::vector const &args) { auto error = 1; if (args.size() > 0) { if (args[0] == "json" && args.size() >= 2) { if (args.size() == 2) error = spartan::mdp3::to_json(args[1]); else if (args.size() == 4) error = spartan::mdp3::to_json(args[1], std::atoi(args[2].c_str()), std::atoi(args[3].c_str())); } else if (args[0] == "compress" && args.size() == 3) error = spartan::mdp3::compress(args[1], args[2]); else if (args[0] == "csv" && args.size() == 3) error = spartan::mdp3::to_csv(args[1], args[2]); else if (args[0] == "sql" && args.size() == 3) error = spartan::mdp3::to_sql(args[1], args[2]); else if (extensions.count(args[0]) && args.size() >= 2) { return extensions[args[0]](args); } } if (error) std::cout << help(); return error; } // namespace cmd std::string Spartan::help() { auto str = fmt::format("\tjson [.xz] [start_packet packet_count]\n" "\tcompress [.xz] \n" "\tsql [.xz] \n" "\tcsv [.xz] \n"); for (auto const &p : extensions) str += fmt::format("\t{} [.xz] ...\n", p.first); return str; } } // namespace cmd } // namespace bts