From 0f8ae1e3456d875a2b4ff69d6a59f8d4c40ca132 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 7 Sep 2020 08:51:57 -0400 Subject: [PATCH 1/5] Refactored simulator names, structure. --- src/server/sparkd/CMakeDeps.cmake | 2 +- src/server/sparkd/cmds/Spartan.cpp | 259 +++++++++--------- .../mdp3/back_testing/ISimulatorServices.cpp | 8 + .../md/mdp3/back_testing/ISimulatorServices.h | 19 ++ .../spartan/md/mdp3/back_testing/SimAlgo.cpp | 1 - .../spartan/md/mdp3/back_testing/SimAlgo.h | 34 --- .../spartan/md/mdp3/back_testing/SimEvent.h | 21 -- .../md/mdp3/back_testing/gen_sim_input.h | 24 -- .../back_testing/simulator/IAlgoModel.cpp | 7 + .../mdp3/back_testing/simulator/IAlgoModel.h | 21 ++ .../back_testing/simulator/SimulatorEvent.cpp | 1 + .../back_testing/simulator/SimulatorEvent.h | 23 ++ .../SimulatorInput.cpp} | 31 ++- .../back_testing/simulator/SimulatorInput.h | 16 ++ .../simulator/SimulatorInputEvent.h | 15 + .../cmds/spartan/md/mdp3/utils/mdp3_utils.cpp | 8 +- 16 files changed, 264 insertions(+), 226 deletions(-) create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.h delete mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.cpp delete mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.h delete mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimEvent.h delete mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.h rename src/server/sparkd/cmds/spartan/md/mdp3/back_testing/{gen_sim_input.cpp => simulator/SimulatorInput.cpp} (76%) create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInputEvent.h diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index b90703bfb..248dd689f 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/SimAlgo.cpp;cmds/spartan/md/mdp3/back_testing/gen_sim_input.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/ISimulatorServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index 066c4b717..ea2e400fc 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -1,6 +1,6 @@ #include "Spartan.h" #include "server/sparkd/cmds/spartan/algo/RunTAS.h" -#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" @@ -27,148 +27,151 @@ using namespace spartan::mdp3; std::map const &)>> extensions = { - {"validate_fifo", - [](auto const &args) { - struct BackTesterHarness : IChannelObserver { - std::unordered_map orders; - std::unordered_map icebergs; - size_t passed_tests = 0; + {"validate_fifo", + [](auto const &args) { + struct BackTesterHarness : IChannelObserver { + std::unordered_map orders; + std::unordered_map icebergs; + size_t passed_tests = 0; - BackTesterHarness(std::string const &pcap) - : icebergs(find_icebergs(pcap)) {} + BackTesterHarness(std::string const &pcap) + : icebergs(find_icebergs(pcap)) {} - void - on_recovery_complete(InitialSnapshot const &snap) override { - for (auto s : snap) { - auto annotated_orders = s->orders(); - add_iceberg_annotation(annotated_orders); - orders.emplace(s->security_id(), annotated_orders); - } - } + void on_recovery_complete(InitialSnapshot const &snap) override { + for (auto s : snap) { + auto annotated_orders = s->orders(); + add_iceberg_annotation(annotated_orders); + orders.emplace(s->security_id(), annotated_orders); + } + } - void on_real_time_event(MDP3Event const &ev) override { + void on_real_time_event(MDP3Event const &ev) override { + for (auto s : ev.affected_securities) { + auto it = orders.find(s->security_id()); + if (it == orders.end()) { + bts_assert(s->orders().empty()); + it = + orders.emplace(s->security_id(), s->orders()).first; + } + auto &old_orders = it->second; + auto deduced = s->deduced_order_changes(); - for (auto s : ev.affected_securities) { - auto it = orders.find(s->security_id()); - if (it == orders.end()) { - bts_assert(s->orders().empty()); - it = orders.emplace(s->security_id(), s->orders()) - .first; - } - auto &old_orders = it->second; - auto deduced = s->deduced_order_changes(); + // We have a hard time simulating icebergs, + // for now skip them + auto found_ice = false; + for (auto &f : s->fills()) + if (icebergs.count(f.key)) { + found_ice = true; + break; + } + if (found_ice) + continue; - // We have a hard time simulating icebergs, - // for now skip them - auto found_ice = false; - for (auto &f : s->fills()) - if (icebergs.count(f.key)) { - found_ice = true; - break; - } - if (found_ice) - continue; + if (deduced.size()) { - if (deduced.size()) { + // We have deduced orders, lets see what the matcher + // says + add_iceberg_annotation(deduced); - // We have deduced orders, lets see what the matcher - // says - add_iceberg_annotation(deduced); + FifoSecurityMatcher matcher(old_orders); + auto current_orders = s->orders(); + add_iceberg_annotation(current_orders); + for (auto &o : deduced) + matcher.on_order_change(o); + auto error = matcher.has_same_orders(current_orders); + if (error) { + std::cout << "Failing on event:\n " << ev.event_id + << std::endl + << "Error: " << error.value() << std::endl + << "Deduced orders:\n" + << deduced << std::endl + << "Old Orders:\n" + << old_orders << std::endl + << "Order Changes:\n" + << s->order_changes() << std::endl + << "Fills:\n" + << s->fills() << std::endl; + bts_assert(false); + } + old_orders = current_orders; + } else { + it->second = s->orders(); + add_iceberg_annotation(it->second); + } + } + } - FifoSecurityMatcher matcher(old_orders); - auto current_orders = s->orders(); - add_iceberg_annotation(current_orders); - for (auto &o : deduced) - matcher.on_order_change(o); - auto error = matcher.has_same_orders(current_orders); - if (error) { - std::cout << "Failing on event:\n " << ev.event_id - << std::endl - << "Error: " << error.value() << std::endl - << "Deduced orders:\n" - << deduced << std::endl - << "Old Orders:\n" - << old_orders << std::endl - << "Order Changes:\n" - << s->order_changes() << std::endl - << "Fills:\n" - << s->fills() << std::endl; - bts_assert(false); - } - old_orders = current_orders; - } else { - it->second = s->orders(); - add_iceberg_annotation(it->second); - } - } - } - void add_iceberg_annotation(MDP3Order &order) { - auto it = icebergs.find(order.key); - if (it != icebergs.end()) - order.iceberg_quantity = it->second.estimated_quantity; - } - void add_iceberg_annotation(Orders &orders) { - for (auto &o : orders) - add_iceberg_annotation(o.second); - } - void add_iceberg_annotation(OrderChanges &changes) { - for (auto &o : changes) { - add_iceberg_annotation(o.previous_order); - add_iceberg_annotation(o.order); - } - } + void add_iceberg_annotation(MDP3Order &order) { + auto it = icebergs.find(order.key); + if (it != icebergs.end()) + order.iceberg_quantity = it->second.estimated_quantity; + } - void on_real_time_channel_loss() override { bts_assert(false); } - } harness(args[1]); - observe_channel(args[1], harness); - return 0; - }}, + void add_iceberg_annotation(Orders &orders) { + for (auto &o : orders) + add_iceberg_annotation(o.second); + } - {"deduce_orders", - [](auto const &args) { - struct OrderGen : IChannelObserver { - std::ofstream out; - OrderGen(std::string const &out_path) : out(out_path) {} + void add_iceberg_annotation(OrderChanges &changes) { + for (auto &o : changes) { + add_iceberg_annotation(o.previous_order); + add_iceberg_annotation(o.order); + } + } - void on_recovery_complete(InitialSnapshot const &) override {} - void on_real_time_event(MDP3Event const &ev) override { - for (auto s : ev.affected_securities) { - auto deduced = s->deduced_order_changes(); - if (deduced.size()) { - out << "sym: " << s->definition().symbol() - << " eid: " << ev.event_id - << " sn: " << ev.first_packet_sn << " to " - << ev.last_packet_sn - << " ts: " << to_time_string(ev.transact_time) - << std::endl - << "Order changes: " << std::endl - << s->order_changes() << "Fills: " << std::endl - << s->fills() << "Deduced changes: " << std::endl - << s->deduced_order_changes() << std::endl - << std::endl; - } - } - } - void on_real_time_channel_loss() override { bts_assert(false); } - } gen(args[2]); - - observe_channel(args[1], gen); - return 0; - }}, - - {"find_icebergs", [](auto const &args) { - std::ofstream out(args[2]); - for (auto const &[k, ice] : find_icebergs(args[1])) - out << "ok: " << k << " sz: " << ice.estimated_quantity.value() - << std::endl; - return 0; + void on_real_time_channel_loss() override { bts_assert(false); } + } harness(args[1]); + observe_channel(args[1], harness); + return 0; }}, - {"gen_sim_input", [](auto const &args) { - auto input = gen_sim_input(args[1]); + {"deduce_orders", + [](auto const &args) { + struct OrderGen : IChannelObserver { + std::ofstream out; + OrderGen(std::string const &out_path) : out(out_path) {} + + void on_recovery_complete(InitialSnapshot const &) override {} + void on_real_time_event(MDP3Event const &ev) override { + for (auto s : ev.affected_securities) { + auto deduced = s->deduced_order_changes(); + if (deduced.size()) { + out << "sym: " << s->definition().symbol() + << " eid: " << ev.event_id + << " sn: " << ev.first_packet_sn << " to " + << ev.last_packet_sn + << " ts: " << to_time_string(ev.transact_time) + << std::endl + << "Order changes: " << std::endl + << s->order_changes() << "Fills: " << std::endl + << s->fills() << "Deduced changes: " << std::endl + << s->deduced_order_changes() << std::endl + << std::endl; + } + } + } + void on_real_time_channel_loss() override { bts_assert(false); } + } gen(args[2]); + + observe_channel(args[1], gen); + return 0; + }}, + + {"find_icebergs", + [](auto const &args) { + std::ofstream out(args[2]); + for (auto const &[k, ice] : find_icebergs(args[1])) + out << "ok: " << k << " sz: " << ice.estimated_quantity.value() + << std::endl; + return 0; + }}, + + {"gen_sim_input", + [](auto const &args) { + auto input = make_simulator_input(args[1]); std::cout << "Secs: " << input.instruments.size() << " events: " << input.events.size() - << std::endl; + << " max_order_id: " << input.max_order_id << std::endl; return 0; }}, }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.cpp new file mode 100644 index 000000000..cf742d9a6 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.cpp @@ -0,0 +1,8 @@ +#include "ISimulatorServices.h" + + +namespace bts::spartan::mdp3 { + +ISimulatorServices::~ISimulatorServices() {} + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.h new file mode 100644 index 000000000..0754493e7 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.h @@ -0,0 +1,19 @@ +#pragma once + +#include "../common/mdp3_common.h" +#include "../common/InstrumentDefinition.h" + +namespace bts::spartan::mdp3 { + +class ISimulatorServices { + public: + virtual ~ISimulatorServices(); + virtual InstrumentDefinitions const &instruments() const = 0; + virtual MDP3OrderKey create_order(MDP3SecurityID, api::Side, md::IntPrice, + MDP3Quantity, size_t latency_ns) = 0; + virtual void update_order(MDP3OrderKey, md::IntPrice, MDP3Quantity, + size_t latency_ns) = 0; + virtual void cancel_order(MDP3OrderKey, size_t latency_ns) = 0; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.cpp deleted file mode 100644 index b5e661685..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "SimAlgo.h" diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.h deleted file mode 100644 index 01898f2ae..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimAlgo.h +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include "SimEvent.h" -#include "../common/InstrumentDefinition.h" -#include "../common/mdp3_common.h" - -namespace bts::spartan::mdp3 { - -class SimAlgo { - public: - // All securities - InstrumentDefinitions const &instruments() const; - - // Order management - MDP3OrderKey create_order(MDP3SecurityID, api::Side, md::IntPrice, - MDP3Quantity, size_t latency_ns = 0); - void update_order(MDP3OrderKey, md::IntPrice, MDP3Quantity, - size_t latency_ns = 0); - void cancel_order(MDP3OrderKey, size_t latency_ns = 0); - // PnL as of now - int64_t pandl() const; - - // Event handling - virtual void init(){}; - virtual void finit(){}; - virtual void on_time(MDP3TimeStamp) {} - virtual void on_event(SimEvent const &) = 0; - - private: - struct PImpl; - std::unique_ptr me; -}; - -} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimEvent.h deleted file mode 100644 index ea4742047..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/SimEvent.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include "../common/mdp3_common.h" - -namespace bts::spartan::mdp3 { - -class InstrumentEvent { - public: - ~InstrumentEvent(); - MDP3SecurityID security_id() const; - Book const &book(api::Side) const; - Fills const &fills() const; - - private: - struct PImpl; - std::unique_ptr me; -}; - -using SimEvent = std::vector; - -} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.h deleted file mode 100644 index a10c2ba36..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once -#include "../common/mdp3_common.h" -#include "../common/InstrumentDefinition.h" - -namespace bts::spartan::mdp3 { - -struct SimExchangeEvent { - size_t event_id; - MDP3TimeStamp transact_time; - MDP3TimeStamp first_packet_send_time; - MDP3TimeStamp last_packet_send_time; - uint32_t first_packet_sn; - uint32_t last_packet_sn; - std::vector order_changes; -}; - -struct SimInput { - std::vector instruments; - std::vector events; -}; - -SimInput gen_sim_input(std::string const&pcap); - -} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp new file mode 100644 index 000000000..dfbd6557a --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp @@ -0,0 +1,7 @@ +#include "IAlgoModel.h" + +namespace bts::spartan::mdp3 { + +IAlgoModel::~IAlgoModel() {} + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h new file mode 100644 index 000000000..39e512a60 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h @@ -0,0 +1,21 @@ +#pragma once + +#include "../../common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +class ISimulatorServices; +class SimulatorEvent; + +class IAlgoModel { + public: + virtual ~IAlgoModel(); + + virtual void init(ISimulatorServices &) = 0; + virtual void finit(){}; + + virtual void on_time(MDP3TimeStamp) {} + virtual void on_event(SimulatorEvent const &) {}; +}; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.cpp new file mode 100644 index 000000000..d2f1628e4 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.cpp @@ -0,0 +1 @@ +#include "SimulatorEvent.h" diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.h new file mode 100644 index 000000000..fdb672e0d --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.h @@ -0,0 +1,23 @@ +#pragma once + +#include "../../common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +struct SimulatorInstrumentEvent { + MDP3SecurityID security_id; + Book mbp_buy; + Book mbp_sell; + Fills fills; + + Book const &mbp_side(api::Side side) const{ + if (side == api::SIDE_BUY) + return mbp_buy; + else + return mbp_sell; + } +}; + +using SimulatorEvent = std::vector; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp similarity index 76% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.cpp rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp index 4bf5433fe..30547b00a 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp @@ -1,17 +1,17 @@ -#include "gen_sim_input.h" -#include "../common/InstrumentSnapshot.h" -#include "../handler/IChannelObserver.h" -#include "../utils/mdp3_utils.h" +#include "../../common/InstrumentSnapshot.h" +#include "../../handler/IChannelObserver.h" +#include "../../utils/mdp3_utils.h" +#include "SimulatorInput.h" namespace bts::spartan::mdp3 { -SimInput gen_sim_input(std::string const &pcap) { +SimulatorInput make_simulator_input(std::string const &pcap) { struct Obs : IChannelObserver { - SimInput result; + SimulatorInput result; std::unordered_map instruments; - std::vector events; + std::vector events; std::unordered_set ready_secs; void on_recovery_complete( @@ -38,7 +38,7 @@ SimInput gen_sim_input(std::string const &pcap) { if (!sec_ready && s->trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade) { - // Transitioned to ready, boostrap book with existing orders + // Transitioned to ready, boostrap book with existing orders ready_secs.insert(s->security_id()); for (auto const &p : s->orders()) { @@ -47,6 +47,8 @@ SimInput gen_sim_input(std::string const &pcap) { change.order = p.second; change.change = OrderChange::added; this_event.order_changes.push_back(change); + result.max_order_id = + std::max(change.order.key.value(), result.max_order_id); } } else if (sec_ready && s->trading_status() == @@ -55,11 +57,14 @@ SimInput gen_sim_input(std::string const &pcap) { auto changes = s->deduced_order_changes(); if (changes.empty()) changes = s->order_changes(); - for (auto &c : changes) + for (auto &c : changes) { this_event.order_changes.push_back(c); - } - else if (sec_ready && s->trading_status() && s->trading_status() != - ::mdp3::SecurityTradingStatus::ReadyToTrade) { + result.max_order_id = + std::max(c.order.key.value(), result.max_order_id); + } + } else if (sec_ready && s->trading_status() && + s->trading_status() != + ::mdp3::SecurityTradingStatus::ReadyToTrade) { ready_secs.erase(s->security_id()); } } @@ -69,7 +74,7 @@ SimInput gen_sim_input(std::string const &pcap) { for (auto const &p : instruments) result.instruments.push_back(p.second); } - + void on_real_time_channel_loss() override { bts_assert(false); } } obs; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.h new file mode 100644 index 000000000..4bbec766d --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.h @@ -0,0 +1,16 @@ +#pragma once + +#include "../../common/InstrumentDefinition.h" +#include "SimulatorInputEvent.h" + +namespace bts::spartan::mdp3 { + +struct SimulatorInput { + std::vector instruments; + std::vector events; + size_t max_order_id = 0; +}; + +SimulatorInput make_simulator_input(std::string const&pcap); + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInputEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInputEvent.h new file mode 100644 index 000000000..9b5ef9297 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInputEvent.h @@ -0,0 +1,15 @@ +#include "../../common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +struct SimulatorInputEvent { + size_t event_id; + MDP3TimeStamp transact_time; + MDP3TimeStamp first_packet_send_time; + MDP3TimeStamp last_packet_send_time; + uint32_t first_packet_sn; + uint32_t last_packet_sn; + std::vector order_changes; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp index 3424cb5d3..14ea5df2b 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp @@ -369,12 +369,12 @@ struct ObsWrapper : mdp3::IChannelObserver { last_print = Timer::now(); std::stringstream str; mdp3::cme_ts_to_time_str(ev.transact_time.value(), &str); - std::cout << "\rPacket time: " << str.str() << " " - << (ev.first_packet_sn - previous_packet) << " pkt/sec " - << "(" + std::cout << "Packet time: " << str.str() << " (" + << (ev.first_packet_sn - previous_packet) << " pkt/sec, " << (ev.first_packet_send_time.value() - previous_send_ts) / 1000000000 - << "x realtime)" ; + << "x speedup) \r"; + std::cout.flush(); previous_packet = ev.first_packet_sn; previous_send_ts = ev.first_packet_send_time.value(); } From d5bc1a0acb2a0ee5517641d90ec32cd192ac6300 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 7 Sep 2020 14:41:08 -0400 Subject: [PATCH 2/5] Tweaked test. --- .../cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp index 5744e5485..d27a5cae4 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp @@ -5,8 +5,8 @@ #include "lib/utils/TempFile.h" #include "lib/utils/overloaded.h" #include "lib/utils/type_name.h" -#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.h" -#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" @@ -100,12 +100,12 @@ TEST_CASE("spartan_md", "[spartan_md]") { auto buy_ok = is->build_book_from_orders(api::SIDE_BUY) == is->book(api::SIDE_BUY); if (!buy_ok) - std::cout << "shit"; + debug_break(); REQUIRE(buy_ok); auto sell_ok = is->build_book_from_orders(api::SIDE_SELL) == is->book(api::SIDE_SELL); if (!sell_ok) - std::cout << "shit"; + debug_break(); REQUIRE(sell_ok); } } From 5cdc4f5a66043c36f99fab6f8696eb0e09fe853e Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 7 Sep 2020 14:41:23 -0400 Subject: [PATCH 3/5] Flushing out simulator. --- src/server/sparkd/CMakeDeps.cmake | 2 +- src/server/sparkd/cmds/Spartan.cpp | 2 +- .../md/mdp3/back_testing/ISimulatorServices.h | 19 --- .../{ => matcher}/FifoPriceLevelMatcher.cpp | 2 +- .../{ => matcher}/FifoPriceLevelMatcher.h | 2 +- .../{ => matcher}/FifoSecurityMatcher.cpp | 2 +- .../{ => matcher}/FifoSecurityMatcher.h | 2 +- .../md/mdp3/back_testing/matcher/Matcher.h | 32 +++++ .../mdp3/back_testing/simulator/AlgoAction.h | 28 +++++ .../mdp3/back_testing/simulator/AlgoEvent.h | 35 ++++++ .../md/mdp3/back_testing/simulator/IAlgo.h | 17 +++ .../mdp3/back_testing/simulator/IAlgoModel.h | 6 +- .../{ => simulator}/ISimulatorServices.cpp | 0 .../simulator/ISimulatorServices.h | 17 +++ .../mdp3/back_testing/simulator/Simulator.cpp | 110 ++++++++++++++++++ .../mdp3/back_testing/simulator/Simulator.h | 31 +++++ .../back_testing/simulator/SimulatorAction.h | 28 +++++ .../back_testing/simulator/SimulatorEvent.h | 20 +++- 18 files changed, 321 insertions(+), 34 deletions(-) delete mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.h rename src/server/sparkd/cmds/spartan/md/mdp3/back_testing/{ => matcher}/FifoPriceLevelMatcher.cpp (99%) rename src/server/sparkd/cmds/spartan/md/mdp3/back_testing/{ => matcher}/FifoPriceLevelMatcher.h (99%) rename src/server/sparkd/cmds/spartan/md/mdp3/back_testing/{ => matcher}/FifoSecurityMatcher.cpp (99%) rename src/server/sparkd/cmds/spartan/md/mdp3/back_testing/{ => matcher}/FifoSecurityMatcher.h (98%) create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoAction.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgo.h rename src/server/sparkd/cmds/spartan/md/mdp3/back_testing/{ => simulator}/ISimulatorServices.cpp (100%) create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorAction.h diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index 248dd689f..77b6b08f1 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/ISimulatorServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index ea2e400fc..7bd80201c 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -15,7 +15,7 @@ #include #include -#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h" #include "server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h" #include "server/sparkd/cmds/spartan/util/PCapReader.h" diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.h deleted file mode 100644 index 0754493e7..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include "../common/mdp3_common.h" -#include "../common/InstrumentDefinition.h" - -namespace bts::spartan::mdp3 { - -class ISimulatorServices { - public: - virtual ~ISimulatorServices(); - virtual InstrumentDefinitions const &instruments() const = 0; - virtual MDP3OrderKey create_order(MDP3SecurityID, api::Side, md::IntPrice, - MDP3Quantity, size_t latency_ns) = 0; - virtual void update_order(MDP3OrderKey, md::IntPrice, MDP3Quantity, - size_t latency_ns) = 0; - virtual void cancel_order(MDP3OrderKey, size_t latency_ns) = 0; -}; - -} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp similarity index 99% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp index dcc2a5cd1..2e9215a17 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp @@ -1,5 +1,5 @@ #include "FifoPriceLevelMatcher.h" -#include "../common/InstrumentSnapshot.h" +#include "../../common/InstrumentSnapshot.h" namespace bts::spartan::mdp3 { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.h similarity index 99% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.h rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.h index 89dcfbd1e..4e0145c4d 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.h @@ -2,7 +2,7 @@ // did not want to fwd decl everything specially // the templated classes (will fix later) -#include "../common/InstrumentSnapshot.h" +#include "../../common/InstrumentSnapshot.h" namespace bts::spartan::mdp3 { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp similarity index 99% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp index 72b6f8e2e..fcf2203b5 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp @@ -1,5 +1,5 @@ #include "FifoSecurityMatcher.h" -#include "../common/InstrumentSnapshot.h" +#include "../../common/InstrumentSnapshot.h" namespace bts::spartan::mdp3 { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h similarity index 98% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.h rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h index 04521bf9c..f0ede98fa 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h @@ -2,7 +2,7 @@ // did not want to fwd decl everything specially // the templated classes (will fix later) -#include "../common/InstrumentSnapshot.h" +#include "../../common/InstrumentSnapshot.h" #include "FifoPriceLevelMatcher.h" #include diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h new file mode 100644 index 000000000..609ba84fb --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h @@ -0,0 +1,32 @@ +#pragma once + +#include "../simulator/SimulatorEvent.h" +#include "../simulator/AlgoAction.h" + +namespace bts::spartan::mdp3 { + +class InstrumentDefinition; + +class Matcher { +public: + Matcher () {} + Matcher (std::vector const&) {} + Matcher (Matcher&&) {} + Matcher &operator=(Matcher&&) {return *this;} + + using MatchEffect = + std::variant; + + using OrderChange = + std::variant; + + enum ChangeSrc { + ALGO, SIMULATOR_INPUT + }; + + void match(ChangeSrc, + std::vector const&, + std::vector *) {} +}; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoAction.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoAction.h new file mode 100644 index 000000000..75ac8750e --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoAction.h @@ -0,0 +1,28 @@ +#pragma once + +#include "../../common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +struct CreateOrder { + MDP3OrderKey ok; + MDP3SecurityID security; + api::Side side; + MDP3OrderPriority priority; + md::IntPrice price; + MDP3Quantity quantity; +}; + +struct UpdateOrder { + MDP3OrderKey ok; + md::IntPrice price; + MDP3Quantity quantity; +}; + +struct CancelOrder { + MDP3OrderKey ok; +}; + +using AlgoAction = std::variant; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h new file mode 100644 index 000000000..8ebd52eea --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h @@ -0,0 +1,35 @@ +#pragma once + +#include "../../common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +struct MBPUpdate { + MDP3SecurityID security_id; + Book mbp_buy; + Book mbp_sell; + Book const &mbp_side(api::Side side) const{ + if (side == api::SIDE_BUY) + return mbp_buy; + else + return mbp_sell; + } +}; + +struct OrderUpdate { + MDP3OrderKey key; + enum Status { + FILLED, + PARTIALLY_FILLED, + CANCELED + }; + Status status; + MDP3Quantity last_fills; + md::IntPrice last_price; +}; + +struct TimePassed {}; +using AlgoEvent = std::variant; + + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgo.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgo.h new file mode 100644 index 000000000..3ebe2b736 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgo.h @@ -0,0 +1,17 @@ +#pragma once + +#include "../../common/mdp3_common.h" +#include "AlgoEvent.h" + +namespace bts::spartan::mdp3 { + +class ISimulatorServices; + +class IAlgo { + public: + virtual ~IAlgo(); + virtual void init(ISimulatorServices &) = 0; + virtual void on_event(AlgoEvent const&) {} +}; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h index 39e512a60..1b8e6a71d 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h @@ -10,12 +10,8 @@ class SimulatorEvent; class IAlgoModel { public: virtual ~IAlgoModel(); - virtual void init(ISimulatorServices &) = 0; - virtual void finit(){}; - - virtual void on_time(MDP3TimeStamp) {} - virtual void on_event(SimulatorEvent const &) {}; + virtual void on_event(SimulatorEvent const&) {} }; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp similarity index 100% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/ISimulatorServices.cpp rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h new file mode 100644 index 000000000..b897642c3 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h @@ -0,0 +1,17 @@ +#pragma once + +#include "../../common/mdp3_common.h" +#include "AlgoAction.h" + +namespace bts::spartan::mdp3 { +class InstrumentDefinition; + +class ISimulatorServices { + public: + virtual ~ISimulatorServices(); + virtual std::vector const &instruments() const = 0; + virtual MDP3OrderKey generate_order_key() = 0; + virtual void send(AlgoAction const &, size_t latency_ns = 0) = 0; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp new file mode 100644 index 000000000..81a8c9e01 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp @@ -0,0 +1,110 @@ +#include "Simulator.h" +#include "../matcher/Matcher.h" +#include "IAlgoModel.h" +#include "ISimulatorServices.h" +#include "SimulatorEvent.h" +#include "SimulatorInput.h" + + +#include + +namespace bts::spartan::mdp3 { + +struct Simulator::AlgoActionRec { + MDP3TimeStamp transact_time; + AlgoRecord *algo_rec; + AlgoAction action; + bool operator<(AlgoActionRec const &rhs) const { + return rhs.transact_time.value() < transact_time.value(); + } +}; + +struct Simulator::AlgoEventRec { + MDP3TimeStamp send_time; + AlgoEvent event; + bool operator<(AlgoEventRec const &rhs) const { + return rhs.send_time.value() < send_time.value(); + } +}; + +struct Simulator::PImpl { + PImpl(SimulatorInput const &i) : input(i) {} + SimulatorInput const &input; + void reset() { + next_input_event = 0; + last_order_key = input.max_order_id; + algos.clear(); + matcher = Matcher{input.instruments}; + pending_algo_actions = {}; + order_to_algo = {}; + } + size_t next_input_event; + size_t last_order_key; + std::vector> algos; + Matcher matcher; + std::priority_queue pending_algo_actions; + std::unordered_map order_to_algo; +}; + +struct Simulator::AlgoRecord : ISimulatorServices { + Simulator &owner; + IAlgoModel &algo; + uint64_t pandl; + + std::priority_queue pending_events; + + AlgoRecord(Simulator &owner, IAlgoModel &algo) + : owner(owner), algo(algo), pandl(0) { + algo.init(*this); + } + + std::vector const &instruments() const override { + return owner.me->input.instruments; + } + + MDP3OrderKey generate_order_key() override { + return MDP3OrderKey{++owner.me->last_order_key}; + } + + void send(AlgoAction const &action, size_t latency_ns) override { + bts_assert(pending_events.size()); + auto ¤t_event = pending_events.top(); + owner.me->pending_algo_actions.push( + {MDP3TimeStamp{current_event.send_time.value() + latency_ns}, this, + action}); + } +}; + +Simulator::Simulator(SimulatorInput const &input) : me(new PImpl{input}) {} + +Simulator::~Simulator() {} + +int64_t Simulator::run(IAlgoModel &algo) { return run({&algo})[0]; } + +std::vector Simulator::run(std::vector const &algos) { + me->reset(); + + for (auto algo : algos) + me->algos.push_back(std::make_unique(*this, *algo)); + + while (has_work()) + do_work(); + std::vector pandl; + for (auto &ar : me->algos) + pandl.push_back(ar->pandl); + return pandl; +} + +bool Simulator::has_work() const { + if (me->next_input_event != me->input.events.size() || + me->pending_algo_actions.size()) + return true; + else { + for (auto &a : me->algos) + if (a->pending_events.size()) + return true; + return false; + } +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h new file mode 100644 index 000000000..ed857f1e6 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include "../../common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +struct SimulatorInput; +class IAlgoModel; + +class Simulator { + public: + Simulator(SimulatorInput const &); + ~Simulator(); + int64_t run(IAlgoModel &); + std::vector run(std::vector const &); + + private: + struct AlgoRecord; + struct AlgoActionRec; + struct AlgoEventRec; + struct PImpl; + void enqueue_action(AlgoActionRec const&); + bool has_work() const; + void do_work(); + MDP3OrderKey next_order_id(); + std::unique_ptr me; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorAction.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorAction.h new file mode 100644 index 000000000..0e98c51fe --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorAction.h @@ -0,0 +1,28 @@ +#pragma once + +#include "../../common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +struct CreateOrder { + MDP3OrderKey ok; + MDP3SecurityID security; + api::Side side; + MDP3OrderPriority priority; + md::IntPrice price; + MDP3Quantity quantity; +}; + +struct UpdateOrder { + MDP3OrderKey ok; + md::IntPrice price; + MDP3Quantity quantity; +}; + +struct CancelOrder { + MDP3OrderKey ok; +}; + +using SimulatorAction = std::variant; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.h index fdb672e0d..8ebd52eea 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.h @@ -4,12 +4,10 @@ namespace bts::spartan::mdp3 { -struct SimulatorInstrumentEvent { +struct MBPUpdate { MDP3SecurityID security_id; Book mbp_buy; Book mbp_sell; - Fills fills; - Book const &mbp_side(api::Side side) const{ if (side == api::SIDE_BUY) return mbp_buy; @@ -18,6 +16,20 @@ struct SimulatorInstrumentEvent { } }; -using SimulatorEvent = std::vector; +struct OrderUpdate { + MDP3OrderKey key; + enum Status { + FILLED, + PARTIALLY_FILLED, + CANCELED + }; + Status status; + MDP3Quantity last_fills; + md::IntPrice last_price; +}; + +struct TimePassed {}; +using AlgoEvent = std::variant; + } From 26a0508e03d2290f48234df797fadfc37f8a6850 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 8 Sep 2020 11:55:45 -0400 Subject: [PATCH 4/5] Simulator in progress. --- src/server/sparkd/CMakeDeps.cmake | 2 +- .../md/mdp3/back_testing/matcher/Matcher.h | 6 +- .../mdp3/back_testing/simulator/AlgoEvent.h | 22 ++--- .../mdp3/back_testing/simulator/IAlgoModel.h | 3 +- .../simulator/ISimulatorServices.h | 1 + .../mdp3/back_testing/simulator/Simulator.cpp | 91 +++++++++++++++++-- .../mdp3/back_testing/simulator/Simulator.h | 28 +++++- .../back_testing/simulator/SimulatorAction.h | 28 ------ .../back_testing/simulator/SimulatorEvent.cpp | 1 - .../back_testing/simulator/SimulatorEvent.h | 35 ------- 10 files changed, 126 insertions(+), 91 deletions(-) delete mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorAction.h delete mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.cpp delete mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.h diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index 77b6b08f1..c9fdeb846 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h index 609ba84fb..9e903bd2b 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h @@ -1,7 +1,8 @@ #pragma once -#include "../simulator/SimulatorEvent.h" #include "../simulator/AlgoAction.h" +#include "../simulator/AlgoEvent.h" +#include "../../common/mdp3_common.h" namespace bts::spartan::mdp3 { @@ -17,9 +18,6 @@ public: using MatchEffect = std::variant; - using OrderChange = - std::variant; - enum ChangeSrc { ALGO, SIMULATOR_INPUT }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h index 8ebd52eea..816e30fd6 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h @@ -8,9 +8,9 @@ struct MBPUpdate { MDP3SecurityID security_id; Book mbp_buy; Book mbp_sell; - Book const &mbp_side(api::Side side) const{ - if (side == api::SIDE_BUY) - return mbp_buy; + Book const &mbp_side(api::Side side) const { + if (side == api::SIDE_BUY) + return mbp_buy; else return mbp_sell; } @@ -18,18 +18,16 @@ struct MBPUpdate { struct OrderUpdate { MDP3OrderKey key; - enum Status { - FILLED, - PARTIALLY_FILLED, - CANCELED - }; + enum Status { FILLED, PARTIALLY_FILLED, CANCELED }; Status status; MDP3Quantity last_fills; md::IntPrice last_price; }; -struct TimePassed {}; -using AlgoEvent = std::variant; +struct AlgoEvent { + MDP3TimeStamp transact_time; + MDP3TimeStamp send_time; + std::vector> parts; +}; - -} +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h index 1b8e6a71d..54155c7ac 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h @@ -1,6 +1,7 @@ #pragma once #include "../../common/mdp3_common.h" +#include "AlgoEvent.h" namespace bts::spartan::mdp3 { @@ -11,7 +12,7 @@ class IAlgoModel { public: virtual ~IAlgoModel(); virtual void init(ISimulatorServices &) = 0; - virtual void on_event(SimulatorEvent const&) {} + virtual void on_event(AlgoEvent const&) {} }; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h index b897642c3..0f19be3b2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h @@ -10,6 +10,7 @@ class ISimulatorServices { public: virtual ~ISimulatorServices(); virtual std::vector const &instruments() const = 0; + virtual MDP3TimeStamp time() const = 0; virtual MDP3OrderKey generate_order_key() = 0; virtual void send(AlgoAction const &, size_t latency_ns = 0) = 0; }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp index 81a8c9e01..79c5f6033 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp @@ -1,10 +1,10 @@ #include "Simulator.h" #include "../matcher/Matcher.h" +#include "AlgoEvent.h" #include "IAlgoModel.h" #include "ISimulatorServices.h" -#include "SimulatorEvent.h" #include "SimulatorInput.h" - +#include "lib/utils/overloaded.h" #include @@ -49,12 +49,14 @@ struct Simulator::PImpl { struct Simulator::AlgoRecord : ISimulatorServices { Simulator &owner; IAlgoModel &algo; + MDP3TimeStamp current_time; + RunOptions options; uint64_t pandl; std::priority_queue pending_events; - AlgoRecord(Simulator &owner, IAlgoModel &algo) - : owner(owner), algo(algo), pandl(0) { + AlgoRecord(Simulator &owner, IAlgoModel &algo, const RunOptions &options) + : owner(owner), algo(algo), current_time(0), options(options), pandl(0) { algo.init(*this); } @@ -66,26 +68,42 @@ struct Simulator::AlgoRecord : ISimulatorServices { return MDP3OrderKey{++owner.me->last_order_key}; } + MDP3TimeStamp time() const override { return current_time; } + void send(AlgoAction const &action, size_t latency_ns) override { + if (latency_ns == 0) + latency_ns = options.default_algo_latency_ns; bts_assert(pending_events.size()); auto ¤t_event = pending_events.top(); owner.me->pending_algo_actions.push( {MDP3TimeStamp{current_event.send_time.value() + latency_ns}, this, action}); } + + void deliver_events() { + while (pending_events.size()) { + auto &event = pending_events.top().event; + current_time = event.send_time; + algo.on_event(event); + pending_events.pop(); + } + } }; Simulator::Simulator(SimulatorInput const &input) : me(new PImpl{input}) {} Simulator::~Simulator() {} -int64_t Simulator::run(IAlgoModel &algo) { return run({&algo})[0]; } +int64_t Simulator::run(IAlgoModel &algo, const RunOptions &options) { + return run({&algo}, options)[0]; +} -std::vector Simulator::run(std::vector const &algos) { +std::vector Simulator::run(std::vector const &algos, + const RunOptions &options) { me->reset(); for (auto algo : algos) - me->algos.push_back(std::make_unique(*this, *algo)); + me->algos.push_back(std::make_unique(*this, *algo, options)); while (has_work()) do_work(); @@ -107,4 +125,63 @@ bool Simulator::has_work() const { } } +void Simulator::do_work() { + if (me->next_input_event != me->input.events.size() && + me->pending_algo_actions.size()) { + + // Both input events and algo actions are available, pick the + // earliest + auto &top_input_event = me->input.events[me->next_input_event]; + auto top_algo_action = me->pending_algo_actions.top(); + + if (top_input_event.last_packet_send_time.value() <= + top_algo_action.transact_time.value()) { + // Input queue has it + me->next_input_event++; + on_simulator_input_event(top_input_event); + } else { + me->pending_algo_actions.pop(); + on_algo_action(top_algo_action); + } + } else if (me->next_input_event != me->input.events.size()) { + // Only have simulator events + auto &top_input_event = me->input.events[me->next_input_event]; + me->next_input_event++; + on_simulator_input_event(top_input_event); + } else if (me->pending_algo_actions.size()) { + // Only have algo actions + auto top_algo_action = me->pending_algo_actions.top(); + me->pending_algo_actions.pop(); + on_algo_action(top_algo_action); + } + + for (auto &algo : me->algos) + algo->deliver_events(); +} + +void Simulator::on_algo_action(AlgoActionRec const&) { +} + +void Simulator::on_simulator_input_event(SimulatorInputEvent const &) { + // std::vector effects; + // me->matcher.match(Matcher::SIMULATOR_INPUT, ev.order_changes, &effects); + + // for (auto &e : effects) + // for (auto &a : algo) { + // std::visit( + // overloaded{ + + // [&](MBPUpdate const &mbp_update) { + // algo->add_to_pending_mbp_event( + // ev.transact_time, + // ev.last_packet_send_time, + // mbp_update); + // }, + + + // [&](OrderUpdate const &) {}}, + // e); + // } +} } // namespace bts::spartan::mdp3 + diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h index ed857f1e6..e260794aa 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h @@ -8,13 +8,33 @@ namespace bts::spartan::mdp3 { struct SimulatorInput; class IAlgoModel; +struct SimulatorInputEvent; class Simulator { public: Simulator(SimulatorInput const &); ~Simulator(); - int64_t run(IAlgoModel &); - std::vector run(std::vector const &); + + struct RunOptions { + RunOptions() : default_algo_latency_ns(0), + ilink_delta_ns(0), + estimated_exchange_latency_ns(0) + {} + + // The default latency added to all algo actions + size_t default_algo_latency_ns = 0; + + // The delta between the market data and the corresponding + // execution report (can be negative) + int64_t ilink_delta_ns = 0; + + // The latency we use when calculating how long it takes to + // process our own algo events + size_t estimated_exchange_latency_ns = 50000; + }; + + int64_t run(IAlgoModel &, const RunOptions & = RunOptions()); + std::vector run(std::vector const &, const RunOptions & = RunOptions()); private: struct AlgoRecord; @@ -24,6 +44,10 @@ class Simulator { void enqueue_action(AlgoActionRec const&); bool has_work() const; void do_work(); + void on_simulator_input_event(SimulatorInputEvent const&); + void on_algo_action(AlgoActionRec const&); + MDP3TimeStamp next_simulator_event_send_time() const; + MDP3OrderKey next_order_id(); std::unique_ptr me; }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorAction.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorAction.h deleted file mode 100644 index 0e98c51fe..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorAction.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include "../../common/mdp3_common.h" - -namespace bts::spartan::mdp3 { - -struct CreateOrder { - MDP3OrderKey ok; - MDP3SecurityID security; - api::Side side; - MDP3OrderPriority priority; - md::IntPrice price; - MDP3Quantity quantity; -}; - -struct UpdateOrder { - MDP3OrderKey ok; - md::IntPrice price; - MDP3Quantity quantity; -}; - -struct CancelOrder { - MDP3OrderKey ok; -}; - -using SimulatorAction = std::variant; - -} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.cpp deleted file mode 100644 index d2f1628e4..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "SimulatorEvent.h" diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.h deleted file mode 100644 index 8ebd52eea..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorEvent.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include "../../common/mdp3_common.h" - -namespace bts::spartan::mdp3 { - -struct MBPUpdate { - MDP3SecurityID security_id; - Book mbp_buy; - Book mbp_sell; - Book const &mbp_side(api::Side side) const{ - if (side == api::SIDE_BUY) - return mbp_buy; - else - return mbp_sell; - } -}; - -struct OrderUpdate { - MDP3OrderKey key; - enum Status { - FILLED, - PARTIALLY_FILLED, - CANCELED - }; - Status status; - MDP3Quantity last_fills; - md::IntPrice last_price; -}; - -struct TimePassed {}; -using AlgoEvent = std::variant; - - -} From 5758b56296443fdfc1c39ce399a5d0ea48ab863c Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Wed, 9 Sep 2020 10:00:52 -0400 Subject: [PATCH 5/5] Improved iceberg detection --- .../sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp index 14ea5df2b..30a8578b2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp @@ -705,6 +705,17 @@ find_icebergs(std::string const &pcap) { size_t actual_quantity = change.order.quantity.value(); if (expected_quantity != actual_quantity) order_data[ok].is_iceberg = true; + } else { + bts_assert(change.change == OrderChange::removed); + // The order was totally filled, but was the fill size + // consistent with what we thought was the remaining + // order size? If not then the order must have been an + // iceberg. + size_t expected_quantity = + change.previous_order.quantity.value(); + auto actual_quantity = fills; + if (expected_quantity != actual_quantity) + order_data[ok].is_iceberg = true; } } }