diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index b90703bfb..c9fdeb846 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/SimAlgo.cpp;cmds/spartan/md/mdp3/back_testing/gen_sim_input.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index ddba644cb..cd6b10b79 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -1,6 +1,6 @@ #include "Spartan.h" #include "server/sparkd/cmds/spartan/algo/RunTAS.h" -#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" @@ -15,7 +15,7 @@ #include #include -#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h" #include "server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h" #include "server/sparkd/cmds/spartan/util/PCapReader.h" @@ -27,149 +27,151 @@ using namespace spartan::mdp3; std::map const &)>> extensions = { - {"validate_fifo", - [](auto const &args) { - struct BackTesterHarness : IChannelObserver { - std::unordered_map orders; - std::unordered_map icebergs; - size_t passed_tests = 0; + {"validate_fifo", + [](auto const &args) { + struct BackTesterHarness : IChannelObserver { + std::unordered_map orders; + std::unordered_map icebergs; + size_t passed_tests = 0; - BackTesterHarness(std::string const &pcap) - : icebergs(find_icebergs(pcap)) {} + BackTesterHarness(std::string const &pcap) + : icebergs(find_icebergs(pcap)) {} - void - on_recovery_complete(InitialSnapshot const &snap) override { - for (auto s : snap) { - auto annotated_orders = s->orders(); - add_iceberg_annotation(annotated_orders); - orders.emplace(s->security_id(), annotated_orders); - } - } + void on_recovery_complete(InitialSnapshot const &snap) override { + for (auto s : snap) { + auto annotated_orders = s->orders(); + add_iceberg_annotation(annotated_orders); + orders.emplace(s->security_id(), annotated_orders); + } + } - void on_real_time_event(MDP3Event const &ev) override { - bool ignore_ibs = true; - for (auto s : ev.affected_securities) { - auto it = orders.find(s->security_id()); - if (it == orders.end()) { - bts_assert(s->orders().empty()); - it = orders.emplace(s->security_id(), s->orders()) - .first; - } - auto &old_orders = it->second; - auto deduced = s->deduced_order_changes(); + void on_real_time_event(MDP3Event const &ev) override { + for (auto s : ev.affected_securities) { + auto it = orders.find(s->security_id()); + if (it == orders.end()) { + bts_assert(s->orders().empty()); + it = + orders.emplace(s->security_id(), s->orders()).first; + } + auto &old_orders = it->second; + auto deduced = s->deduced_order_changes(); - // We have a hard time simulating icebergs, - // for now skip them - auto found_ice = false; - for (auto &f : s->fills()) - if (icebergs.count(f.key)) { - found_ice = true; - break; - } - if (found_ice) - if(ignore_ibs) - continue; + // We have a hard time simulating icebergs, + // for now skip them + auto found_ice = false; + for (auto &f : s->fills()) + if (icebergs.count(f.key)) { + found_ice = true; + break; + } + if (found_ice) + continue; - if (deduced.size()) { + if (deduced.size()) { - // We have deduced orders, lets see what the matcher - // says - add_iceberg_annotation(deduced); + // We have deduced orders, lets see what the matcher + // says + add_iceberg_annotation(deduced); FifoSecurityMatcher matcher(ev.event_id,old_orders); - auto current_orders = s->orders(); - add_iceberg_annotation(current_orders); - for (auto &o : deduced) - matcher.on_order_change(o); - auto error = matcher.has_same_orders(current_orders); - if (error) { - std::cout << "Failing on event:\n " << ev.event_id - << std::endl - << "Error: " << error.value() << std::endl - << "Deduced orders:\n" - << deduced << std::endl - << "Old Orders:\n" - << old_orders << std::endl - << "Order Changes:\n" - << s->order_changes() << std::endl - << "Fills:\n" - << s->fills() << std::endl; - bts_assert(false); - } - old_orders = current_orders; - } else { - it->second = s->orders(); - add_iceberg_annotation(it->second); - } - } - } - void add_iceberg_annotation(MDP3Order &order) { - auto it = icebergs.find(order.key); - if (it != icebergs.end()) - order.iceberg_quantity = it->second.estimated_quantity; - } - void add_iceberg_annotation(Orders &orders) { - for (auto &o : orders) - add_iceberg_annotation(o.second); - } - void add_iceberg_annotation(OrderChanges &changes) { - for (auto &o : changes) { - add_iceberg_annotation(o.previous_order); - add_iceberg_annotation(o.order); - } - } + auto current_orders = s->orders(); + add_iceberg_annotation(current_orders); + for (auto &o : deduced) + matcher.on_order_change(o); + auto error = matcher.has_same_orders(current_orders); + if (error) { + std::cout << "Failing on event:\n " << ev.event_id + << std::endl + << "Error: " << error.value() << std::endl + << "Deduced orders:\n" + << deduced << std::endl + << "Old Orders:\n" + << old_orders << std::endl + << "Order Changes:\n" + << s->order_changes() << std::endl + << "Fills:\n" + << s->fills() << std::endl; + bts_assert(false); + } + old_orders = current_orders; + } else { + it->second = s->orders(); + add_iceberg_annotation(it->second); + } + } + } - void on_real_time_channel_loss() override { bts_assert(false); } - } harness(args[1]); - observe_channel(args[1], harness); - return 0; - }}, + void add_iceberg_annotation(MDP3Order &order) { + auto it = icebergs.find(order.key); + if (it != icebergs.end()) + order.iceberg_quantity = it->second.estimated_quantity; + } - {"deduce_orders", - [](auto const &args) { - struct OrderGen : IChannelObserver { - std::ofstream out; - OrderGen(std::string const &out_path) : out(out_path) {} + void add_iceberg_annotation(Orders &orders) { + for (auto &o : orders) + add_iceberg_annotation(o.second); + } - void on_recovery_complete(InitialSnapshot const &) override {} - void on_real_time_event(MDP3Event const &ev) override { - for (auto s : ev.affected_securities) { - auto deduced = s->deduced_order_changes(); - if (deduced.size()) { - out << "sym: " << s->definition().symbol() - << " eid: " << ev.event_id - << " sn: " << ev.first_packet_sn << " to " - << ev.last_packet_sn - << " ts: " << to_time_string(ev.transact_time) - << std::endl - << "Order changes: " << std::endl - << s->order_changes() << "Fills: " << std::endl - << s->fills() << "Deduced changes: " << std::endl - << s->deduced_order_changes() << std::endl - << std::endl; - } - } - } - void on_real_time_channel_loss() override { bts_assert(false); } - } gen(args[2]); + void add_iceberg_annotation(OrderChanges &changes) { + for (auto &o : changes) { + add_iceberg_annotation(o.previous_order); + add_iceberg_annotation(o.order); + } + } - observe_channel(args[1], gen); - return 0; - }}, - - {"find_icebergs", [](auto const &args) { - std::ofstream out(args[2]); - for (auto const &[k, ice] : find_icebergs(args[1])) - out << "ok: " << k << " sz: " << ice.estimated_quantity.value() - << std::endl; - return 0; + void on_real_time_channel_loss() override { bts_assert(false); } + } harness(args[1]); + observe_channel(args[1], harness); + return 0; }}, - {"gen_sim_input", [](auto const &args) { - auto input = gen_sim_input(args[1]); + {"deduce_orders", + [](auto const &args) { + struct OrderGen : IChannelObserver { + std::ofstream out; + OrderGen(std::string const &out_path) : out(out_path) {} + + void on_recovery_complete(InitialSnapshot const &) override {} + void on_real_time_event(MDP3Event const &ev) override { + for (auto s : ev.affected_securities) { + auto deduced = s->deduced_order_changes(); + if (deduced.size()) { + out << "sym: " << s->definition().symbol() + << " eid: " << ev.event_id + << " sn: " << ev.first_packet_sn << " to " + << ev.last_packet_sn + << " ts: " << to_time_string(ev.transact_time) + << std::endl + << "Order changes: " << std::endl + << s->order_changes() << "Fills: " << std::endl + << s->fills() << "Deduced changes: " << std::endl + << s->deduced_order_changes() << std::endl + << std::endl; + } + } + } + void on_real_time_channel_loss() override { bts_assert(false); } + } gen(args[2]); + + observe_channel(args[1], gen); + return 0; + }}, + + {"find_icebergs", + [](auto const &args) { + std::ofstream out(args[2]); + for (auto const &[k, ice] : find_icebergs(args[1])) + out << "ok: " << k << " sz: " << ice.estimated_quantity.value() + << std::endl; + return 0; + }}, + + {"gen_sim_input", + [](auto const &args) { + auto input = make_simulator_input(args[1]); std::cout << "Secs: " << input.instruments.size() << " events: " << input.events.size() - << std::endl; + << " max_order_id: " << input.max_order_id << std::endl; return 0; }}, }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.cpp deleted file mode 100644 index b5e661685..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "SimAlgo.h" diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.h deleted file mode 100644 index 01898f2ae..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.h +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include "SimEvent.h" -#include "../common/InstrumentDefinition.h" -#include "../common/mdp3_common.h" - -namespace bts::spartan::mdp3 { - -class SimAlgo { - public: - // All securities - InstrumentDefinitions const &instruments() const; - - // Order management - MDP3OrderKey create_order(MDP3SecurityID, api::Side, md::IntPrice, - MDP3Quantity, size_t latency_ns = 0); - void update_order(MDP3OrderKey, md::IntPrice, MDP3Quantity, - size_t latency_ns = 0); - void cancel_order(MDP3OrderKey, size_t latency_ns = 0); - // PnL as of now - int64_t pandl() const; - - // Event handling - virtual void init(){}; - virtual void finit(){}; - virtual void on_time(MDP3TimeStamp) {} - virtual void on_event(SimEvent const &) = 0; - - private: - struct PImpl; - std::unique_ptr me; -}; - -} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimEvent.h deleted file mode 100644 index ea4742047..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimEvent.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include "../common/mdp3_common.h" - -namespace bts::spartan::mdp3 { - -class InstrumentEvent { - public: - ~InstrumentEvent(); - MDP3SecurityID security_id() const; - Book const &book(api::Side) const; - Fills const &fills() const; - - private: - struct PImpl; - std::unique_ptr me; -}; - -using SimEvent = std::vector; - -} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.h deleted file mode 100644 index a10c2ba36..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once -#include "../common/mdp3_common.h" -#include "../common/InstrumentDefinition.h" - -namespace bts::spartan::mdp3 { - -struct SimExchangeEvent { - size_t event_id; - MDP3TimeStamp transact_time; - MDP3TimeStamp first_packet_send_time; - MDP3TimeStamp last_packet_send_time; - uint32_t first_packet_sn; - uint32_t last_packet_sn; - std::vector order_changes; -}; - -struct SimInput { - std::vector instruments; - std::vector events; -}; - -SimInput gen_sim_input(std::string const&pcap); - -} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp similarity index 99% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp index 7d7cb5e6c..1b0d74a26 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp @@ -1,5 +1,5 @@ #include "FifoPriceLevelMatcher.h" -#include "../common/InstrumentSnapshot.h" +#include "../../common/InstrumentSnapshot.h" namespace bts::spartan::mdp3 { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.h similarity index 98% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.h rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.h index 825fc6d6a..97762a471 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.h @@ -2,8 +2,8 @@ // did not want to fwd decl everything specially // the templated classes (will fix later) -#include "../common/InstrumentSnapshot.h" -#include "MatchingCommon.h" +#include "../MatchingCommon.h" +#include "../../common/InstrumentSnapshot.h" namespace bts::spartan::mdp3 { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp similarity index 99% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp index d032391cd..ccec80dd1 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp @@ -1,5 +1,5 @@ #include "FifoSecurityMatcher.h" -#include "../common/InstrumentSnapshot.h" +#include "../../common/InstrumentSnapshot.h" namespace bts::spartan::mdp3 { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h similarity index 98% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.h rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h index 03cdf9e80..b5f866ce3 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h @@ -2,7 +2,7 @@ // did not want to fwd decl everything specially // the templated classes (will fix later) -#include "../common/InstrumentSnapshot.h" +#include "../../common/InstrumentSnapshot.h" #include "FifoPriceLevelMatcher.h" #include diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h new file mode 100644 index 000000000..9e903bd2b --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h @@ -0,0 +1,30 @@ +#pragma once + +#include "../simulator/AlgoAction.h" +#include "../simulator/AlgoEvent.h" +#include "../../common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +class InstrumentDefinition; + +class Matcher { +public: + Matcher () {} + Matcher (std::vector const&) {} + Matcher (Matcher&&) {} + Matcher &operator=(Matcher&&) {return *this;} + + using MatchEffect = + std::variant; + + enum ChangeSrc { + ALGO, SIMULATOR_INPUT + }; + + void match(ChangeSrc, + std::vector const&, + std::vector *) {} +}; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoAction.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoAction.h new file mode 100644 index 000000000..75ac8750e --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoAction.h @@ -0,0 +1,28 @@ +#pragma once + +#include "../../common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +struct CreateOrder { + MDP3OrderKey ok; + MDP3SecurityID security; + api::Side side; + MDP3OrderPriority priority; + md::IntPrice price; + MDP3Quantity quantity; +}; + +struct UpdateOrder { + MDP3OrderKey ok; + md::IntPrice price; + MDP3Quantity quantity; +}; + +struct CancelOrder { + MDP3OrderKey ok; +}; + +using AlgoAction = std::variant; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h new file mode 100644 index 000000000..816e30fd6 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h @@ -0,0 +1,33 @@ +#pragma once + +#include "../../common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +struct MBPUpdate { + MDP3SecurityID security_id; + Book mbp_buy; + Book mbp_sell; + Book const &mbp_side(api::Side side) const { + if (side == api::SIDE_BUY) + return mbp_buy; + else + return mbp_sell; + } +}; + +struct OrderUpdate { + MDP3OrderKey key; + enum Status { FILLED, PARTIALLY_FILLED, CANCELED }; + Status status; + MDP3Quantity last_fills; + md::IntPrice last_price; +}; + +struct AlgoEvent { + MDP3TimeStamp transact_time; + MDP3TimeStamp send_time; + std::vector> parts; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgo.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgo.h new file mode 100644 index 000000000..3ebe2b736 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgo.h @@ -0,0 +1,17 @@ +#pragma once + +#include "../../common/mdp3_common.h" +#include "AlgoEvent.h" + +namespace bts::spartan::mdp3 { + +class ISimulatorServices; + +class IAlgo { + public: + virtual ~IAlgo(); + virtual void init(ISimulatorServices &) = 0; + virtual void on_event(AlgoEvent const&) {} +}; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp new file mode 100644 index 000000000..dfbd6557a --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp @@ -0,0 +1,7 @@ +#include "IAlgoModel.h" + +namespace bts::spartan::mdp3 { + +IAlgoModel::~IAlgoModel() {} + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h new file mode 100644 index 000000000..54155c7ac --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h @@ -0,0 +1,18 @@ +#pragma once + +#include "../../common/mdp3_common.h" +#include "AlgoEvent.h" + +namespace bts::spartan::mdp3 { + +class ISimulatorServices; +class SimulatorEvent; + +class IAlgoModel { + public: + virtual ~IAlgoModel(); + virtual void init(ISimulatorServices &) = 0; + virtual void on_event(AlgoEvent const&) {} +}; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp new file mode 100644 index 000000000..cf742d9a6 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp @@ -0,0 +1,8 @@ +#include "ISimulatorServices.h" + + +namespace bts::spartan::mdp3 { + +ISimulatorServices::~ISimulatorServices() {} + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h new file mode 100644 index 000000000..0f19be3b2 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h @@ -0,0 +1,18 @@ +#pragma once + +#include "../../common/mdp3_common.h" +#include "AlgoAction.h" + +namespace bts::spartan::mdp3 { +class InstrumentDefinition; + +class ISimulatorServices { + public: + virtual ~ISimulatorServices(); + virtual std::vector const &instruments() const = 0; + virtual MDP3TimeStamp time() const = 0; + virtual MDP3OrderKey generate_order_key() = 0; + virtual void send(AlgoAction const &, size_t latency_ns = 0) = 0; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp new file mode 100644 index 000000000..79c5f6033 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp @@ -0,0 +1,187 @@ +#include "Simulator.h" +#include "../matcher/Matcher.h" +#include "AlgoEvent.h" +#include "IAlgoModel.h" +#include "ISimulatorServices.h" +#include "SimulatorInput.h" +#include "lib/utils/overloaded.h" + +#include + +namespace bts::spartan::mdp3 { + +struct Simulator::AlgoActionRec { + MDP3TimeStamp transact_time; + AlgoRecord *algo_rec; + AlgoAction action; + bool operator<(AlgoActionRec const &rhs) const { + return rhs.transact_time.value() < transact_time.value(); + } +}; + +struct Simulator::AlgoEventRec { + MDP3TimeStamp send_time; + AlgoEvent event; + bool operator<(AlgoEventRec const &rhs) const { + return rhs.send_time.value() < send_time.value(); + } +}; + +struct Simulator::PImpl { + PImpl(SimulatorInput const &i) : input(i) {} + SimulatorInput const &input; + void reset() { + next_input_event = 0; + last_order_key = input.max_order_id; + algos.clear(); + matcher = Matcher{input.instruments}; + pending_algo_actions = {}; + order_to_algo = {}; + } + size_t next_input_event; + size_t last_order_key; + std::vector> algos; + Matcher matcher; + std::priority_queue pending_algo_actions; + std::unordered_map order_to_algo; +}; + +struct Simulator::AlgoRecord : ISimulatorServices { + Simulator &owner; + IAlgoModel &algo; + MDP3TimeStamp current_time; + RunOptions options; + uint64_t pandl; + + std::priority_queue pending_events; + + AlgoRecord(Simulator &owner, IAlgoModel &algo, const RunOptions &options) + : owner(owner), algo(algo), current_time(0), options(options), pandl(0) { + algo.init(*this); + } + + std::vector const &instruments() const override { + return owner.me->input.instruments; + } + + MDP3OrderKey generate_order_key() override { + return MDP3OrderKey{++owner.me->last_order_key}; + } + + MDP3TimeStamp time() const override { return current_time; } + + void send(AlgoAction const &action, size_t latency_ns) override { + if (latency_ns == 0) + latency_ns = options.default_algo_latency_ns; + bts_assert(pending_events.size()); + auto ¤t_event = pending_events.top(); + owner.me->pending_algo_actions.push( + {MDP3TimeStamp{current_event.send_time.value() + latency_ns}, this, + action}); + } + + void deliver_events() { + while (pending_events.size()) { + auto &event = pending_events.top().event; + current_time = event.send_time; + algo.on_event(event); + pending_events.pop(); + } + } +}; + +Simulator::Simulator(SimulatorInput const &input) : me(new PImpl{input}) {} + +Simulator::~Simulator() {} + +int64_t Simulator::run(IAlgoModel &algo, const RunOptions &options) { + return run({&algo}, options)[0]; +} + +std::vector Simulator::run(std::vector const &algos, + const RunOptions &options) { + me->reset(); + + for (auto algo : algos) + me->algos.push_back(std::make_unique(*this, *algo, options)); + + while (has_work()) + do_work(); + std::vector pandl; + for (auto &ar : me->algos) + pandl.push_back(ar->pandl); + return pandl; +} + +bool Simulator::has_work() const { + if (me->next_input_event != me->input.events.size() || + me->pending_algo_actions.size()) + return true; + else { + for (auto &a : me->algos) + if (a->pending_events.size()) + return true; + return false; + } +} + +void Simulator::do_work() { + if (me->next_input_event != me->input.events.size() && + me->pending_algo_actions.size()) { + + // Both input events and algo actions are available, pick the + // earliest + auto &top_input_event = me->input.events[me->next_input_event]; + auto top_algo_action = me->pending_algo_actions.top(); + + if (top_input_event.last_packet_send_time.value() <= + top_algo_action.transact_time.value()) { + // Input queue has it + me->next_input_event++; + on_simulator_input_event(top_input_event); + } else { + me->pending_algo_actions.pop(); + on_algo_action(top_algo_action); + } + } else if (me->next_input_event != me->input.events.size()) { + // Only have simulator events + auto &top_input_event = me->input.events[me->next_input_event]; + me->next_input_event++; + on_simulator_input_event(top_input_event); + } else if (me->pending_algo_actions.size()) { + // Only have algo actions + auto top_algo_action = me->pending_algo_actions.top(); + me->pending_algo_actions.pop(); + on_algo_action(top_algo_action); + } + + for (auto &algo : me->algos) + algo->deliver_events(); +} + +void Simulator::on_algo_action(AlgoActionRec const&) { +} + +void Simulator::on_simulator_input_event(SimulatorInputEvent const &) { + // std::vector effects; + // me->matcher.match(Matcher::SIMULATOR_INPUT, ev.order_changes, &effects); + + // for (auto &e : effects) + // for (auto &a : algo) { + // std::visit( + // overloaded{ + + // [&](MBPUpdate const &mbp_update) { + // algo->add_to_pending_mbp_event( + // ev.transact_time, + // ev.last_packet_send_time, + // mbp_update); + // }, + + + // [&](OrderUpdate const &) {}}, + // e); + // } +} +} // namespace bts::spartan::mdp3 + diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h new file mode 100644 index 000000000..e260794aa --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include "../../common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +struct SimulatorInput; +class IAlgoModel; +struct SimulatorInputEvent; + +class Simulator { + public: + Simulator(SimulatorInput const &); + ~Simulator(); + + struct RunOptions { + RunOptions() : default_algo_latency_ns(0), + ilink_delta_ns(0), + estimated_exchange_latency_ns(0) + {} + + // The default latency added to all algo actions + size_t default_algo_latency_ns = 0; + + // The delta between the market data and the corresponding + // execution report (can be negative) + int64_t ilink_delta_ns = 0; + + // The latency we use when calculating how long it takes to + // process our own algo events + size_t estimated_exchange_latency_ns = 50000; + }; + + int64_t run(IAlgoModel &, const RunOptions & = RunOptions()); + std::vector run(std::vector const &, const RunOptions & = RunOptions()); + + private: + struct AlgoRecord; + struct AlgoActionRec; + struct AlgoEventRec; + struct PImpl; + void enqueue_action(AlgoActionRec const&); + bool has_work() const; + void do_work(); + void on_simulator_input_event(SimulatorInputEvent const&); + void on_algo_action(AlgoActionRec const&); + MDP3TimeStamp next_simulator_event_send_time() const; + + MDP3OrderKey next_order_id(); + std::unique_ptr me; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp similarity index 76% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.cpp rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp index 4bf5433fe..30547b00a 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp @@ -1,17 +1,17 @@ -#include "gen_sim_input.h" -#include "../common/InstrumentSnapshot.h" -#include "../handler/IChannelObserver.h" -#include "../utils/mdp3_utils.h" +#include "../../common/InstrumentSnapshot.h" +#include "../../handler/IChannelObserver.h" +#include "../../utils/mdp3_utils.h" +#include "SimulatorInput.h" namespace bts::spartan::mdp3 { -SimInput gen_sim_input(std::string const &pcap) { +SimulatorInput make_simulator_input(std::string const &pcap) { struct Obs : IChannelObserver { - SimInput result; + SimulatorInput result; std::unordered_map instruments; - std::vector events; + std::vector events; std::unordered_set ready_secs; void on_recovery_complete( @@ -38,7 +38,7 @@ SimInput gen_sim_input(std::string const &pcap) { if (!sec_ready && s->trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade) { - // Transitioned to ready, boostrap book with existing orders + // Transitioned to ready, boostrap book with existing orders ready_secs.insert(s->security_id()); for (auto const &p : s->orders()) { @@ -47,6 +47,8 @@ SimInput gen_sim_input(std::string const &pcap) { change.order = p.second; change.change = OrderChange::added; this_event.order_changes.push_back(change); + result.max_order_id = + std::max(change.order.key.value(), result.max_order_id); } } else if (sec_ready && s->trading_status() == @@ -55,11 +57,14 @@ SimInput gen_sim_input(std::string const &pcap) { auto changes = s->deduced_order_changes(); if (changes.empty()) changes = s->order_changes(); - for (auto &c : changes) + for (auto &c : changes) { this_event.order_changes.push_back(c); - } - else if (sec_ready && s->trading_status() && s->trading_status() != - ::mdp3::SecurityTradingStatus::ReadyToTrade) { + result.max_order_id = + std::max(c.order.key.value(), result.max_order_id); + } + } else if (sec_ready && s->trading_status() && + s->trading_status() != + ::mdp3::SecurityTradingStatus::ReadyToTrade) { ready_secs.erase(s->security_id()); } } @@ -69,7 +74,7 @@ SimInput gen_sim_input(std::string const &pcap) { for (auto const &p : instruments) result.instruments.push_back(p.second); } - + void on_real_time_channel_loss() override { bts_assert(false); } } obs; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.h new file mode 100644 index 000000000..4bbec766d --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.h @@ -0,0 +1,16 @@ +#pragma once + +#include "../../common/InstrumentDefinition.h" +#include "SimulatorInputEvent.h" + +namespace bts::spartan::mdp3 { + +struct SimulatorInput { + std::vector instruments; + std::vector events; + size_t max_order_id = 0; +}; + +SimulatorInput make_simulator_input(std::string const&pcap); + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInputEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInputEvent.h new file mode 100644 index 000000000..9b5ef9297 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInputEvent.h @@ -0,0 +1,15 @@ +#include "../../common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +struct SimulatorInputEvent { + size_t event_id; + MDP3TimeStamp transact_time; + MDP3TimeStamp first_packet_send_time; + MDP3TimeStamp last_packet_send_time; + uint32_t first_packet_sn; + uint32_t last_packet_sn; + std::vector order_changes; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp index c4ff96865..6e0f91425 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp @@ -5,8 +5,8 @@ #include "lib/utils/TempFile.h" #include "lib/utils/overloaded.h" #include "lib/utils/type_name.h" -#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.h" -#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" @@ -100,12 +100,12 @@ TEST_CASE("spartan_md", "[spartan_md]") { auto buy_ok = is->build_book_from_orders(api::SIDE_BUY) == is->book(api::SIDE_BUY); if (!buy_ok) - std::cout << "shit"; + debug_break(); REQUIRE(buy_ok); auto sell_ok = is->build_book_from_orders(api::SIDE_SELL) == is->book(api::SIDE_SELL); if (!sell_ok) - std::cout << "shit"; + debug_break(); REQUIRE(sell_ok); } } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp index 3424cb5d3..30a8578b2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp @@ -369,12 +369,12 @@ struct ObsWrapper : mdp3::IChannelObserver { last_print = Timer::now(); std::stringstream str; mdp3::cme_ts_to_time_str(ev.transact_time.value(), &str); - std::cout << "\rPacket time: " << str.str() << " " - << (ev.first_packet_sn - previous_packet) << " pkt/sec " - << "(" + std::cout << "Packet time: " << str.str() << " (" + << (ev.first_packet_sn - previous_packet) << " pkt/sec, " << (ev.first_packet_send_time.value() - previous_send_ts) / 1000000000 - << "x realtime)" ; + << "x speedup) \r"; + std::cout.flush(); previous_packet = ev.first_packet_sn; previous_send_ts = ev.first_packet_send_time.value(); } @@ -705,6 +705,17 @@ find_icebergs(std::string const &pcap) { size_t actual_quantity = change.order.quantity.value(); if (expected_quantity != actual_quantity) order_data[ok].is_iceberg = true; + } else { + bts_assert(change.change == OrderChange::removed); + // The order was totally filled, but was the fill size + // consistent with what we thought was the remaining + // order size? If not then the order must have been an + // iceberg. + size_t expected_quantity = + change.previous_order.quantity.value(); + auto actual_quantity = fills; + if (expected_quantity != actual_quantity) + order_data[ok].is_iceberg = true; } } }