From 398397ca7f8667f80cb7f073280a0cf9ea9493ce Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Thu, 26 Nov 2020 09:46:49 -0500 Subject: [PATCH] Towards self-correcting simulator. --- src/data_files/files/spartan/md.sql | 3 +- src/server/sparkd/CMakeDeps.cmake | 2 +- src/server/sparkd/cmds/QuickTest.cpp | 11 +- src/server/sparkd/cmds/Spartan.cpp | 17 +- .../mdp3/back_testing/algos/PolyObserver2.cpp | 154 +++++++++ .../mdp3/back_testing/algos/PolyObserver2.h | 31 ++ .../back_testing/matcher/iceberg_helper.cpp | 23 +- .../back_testing/matcher/iceberg_helper.h | 3 +- .../matcher/validation/iceberg_file_cache.cpp | 10 +- .../back_testing/orchestrator/run_sim.cpp | 60 ++++ .../mdp3/back_testing/orchestrator/run_sim.h | 15 + .../md/mdp3/back_testing/pcap_db/PCapDB.cpp | 3 +- .../mdp3/back_testing/simulator/IAlgoModel.h | 1 + .../simulator/MDP3EventGenerator.cpp | 4 + .../simulator/MDP3EventGenerator.h | 28 ++ .../mdp3/back_testing/simulator/PCapEvent.h | 20 ++ .../simulator/PCapEventSource.cpp | 74 ++++ .../back_testing/simulator/PCapEventSource.h | 33 ++ .../mdp3/back_testing/simulator/SimInput.cpp | 12 +- .../simulator/agents/AlgoModel.cpp | 238 +++++++++++++ .../back_testing/simulator/agents/AlgoModel.h | 71 ++++ .../simulator/agents/ExchangeGateway.cpp | 320 ++++++++++++++++++ .../simulator/agents/ExchangeGateway.h | 48 +++ .../back_testing/simulator/core/IAgent2.h | 17 + .../back_testing/simulator/core/ISimulator2.h | 32 ++ .../simulator/core/Simulator2.cpp | 154 +++++++++ .../back_testing/simulator/core/Simulator2.h | 29 ++ .../back_testing/simulator/events/SimEvents.h | 4 + .../md/mdp3/common/InstrumentDefinition.cpp | 198 ++++++++++- .../md/mdp3/common/InstrumentDefinition.h | 260 ++------------ .../md/mdp3/common/InstrumentSnapshot.cpp | 113 +++++-- .../md/mdp3/common/InstrumentSnapshot.h | 13 +- .../cmds/spartan/md/mdp3/common/mdp3_common.h | 13 +- .../md/mdp3/realtime/RealTimeHandler.cpp | 18 +- .../md/mdp3/recovery/DefinitionSnapshot.cpp | 5 +- .../md/mdp3/recovery/RecoveryHandler.cpp | 4 +- .../cmds/spartan/md/mdp3/utils/mdp3_utils.cpp | 69 ++-- .../cmds/spartan/md/mdp3/utils/mdp3_utils.h | 15 + .../md/mdp3/utils/queries/IcebergFinder.cpp | 6 +- .../md/mdp3/utils/queries/IcebergFinder.h | 17 +- .../mdp3/utils/queries/TransactionDeducer.cpp | 5 +- src/server/sparkd/main.cpp | 2 +- 42 files changed, 1783 insertions(+), 372 deletions(-) create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver2.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver2.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/run_sim.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/run_sim.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/MDP3EventGenerator.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/MDP3EventGenerator.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEvent.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEventSource.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEventSource.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoModel.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoModel.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/ExchangeGateway.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/ExchangeGateway.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/IAgent2.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/ISimulator2.h create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator2.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator2.h diff --git a/src/data_files/files/spartan/md.sql b/src/data_files/files/spartan/md.sql index aea57fdb9..219f99e82 100644 --- a/src/data_files/files/spartan/md.sql +++ b/src/data_files/files/spartan/md.sql @@ -82,7 +82,8 @@ CREATE INDEX book_sid_idx ON books(sid); drop table if exists icebergs; CREATE TABLE icebergs( ok numeric, - estimated_qty numeric, + total_qty numeric, + show_qty numeric, filled numeric ); diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index 8d4e7e492..a6a9e0d54 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/tas/RunTAS.cpp;cmds/spartan/algo/tas/TASOrderReactor.cpp;cmds/spartan/algo/tas/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp;cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/SimOrderGateway.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoImpliedSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoMatcherContext.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoMergedSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/MatchingCommon.cpp;cmds/spartan/md/mdp3/back_testing/matcher/fill_depot.cpp;cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp;cmds/spartan/md/mdp3/back_testing/matcher/tests/matcher_tests.cpp;cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.cpp;cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.cpp;cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp;cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto;cmds/spartan/md/mdp3/back_testing/pcap_db/PCapDB.cpp;cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp;cmds/spartan/md/mdp3/back_testing/simulator/Bridge/AlgoServicesBridge.cpp;cmds/spartan/md/mdp3/back_testing/simulator/Bridge/BeforeLaunchBridge.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IExchangeServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/PandL.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.proto;cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/TimerAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/AgentBase.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/tests/test_simulator_core.cpp;cmds/spartan/md/mdp3/back_testing/simulator/events/SimEvents.cpp;cmds/spartan/md/mdp3/common/InstrumentDefinition.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/common/mdp3_common.cpp;cmds/spartan/md/mdp3/common/tests/book_merging_tests.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/implied/implied_market.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/md/mdp3/utils/order_utils.cpp;cmds/spartan/md/mdp3/utils/polygon_utils.cpp;cmds/spartan/md/mdp3/utils/queries/DefinitionCollector.cpp;cmds/spartan/md/mdp3/utils/queries/FillCounter.cpp;cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp;cmds/spartan/md/mdp3/utils/queries/PolygonFinder.cpp;cmds/spartan/md/mdp3/utils/queries/PolygonOpportunityFinder.cpp;cmds/spartan/md/mdp3/utils/queries/TransactionDeducer.cpp;cmds/spartan/md/mdp3/utils/security_comp.cpp;cmds/spartan/md/mdp3/utils/security_parser.cpp;cmds/spartan/md/mdp3/utils/tests/test_polygon_finder.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;cmds/spartan/util/action_summarizer.cpp;cmds/spartan/util/args_parser.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/tas/RunTAS.cpp;cmds/spartan/algo/tas/TASOrderReactor.cpp;cmds/spartan/algo/tas/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp;cmds/spartan/md/mdp3/back_testing/algos/PolyObserver2.cpp;cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/SimOrderGateway.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoImpliedSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoMatcherContext.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoMergedSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/MatchingCommon.cpp;cmds/spartan/md/mdp3/back_testing/matcher/fill_depot.cpp;cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp;cmds/spartan/md/mdp3/back_testing/matcher/tests/matcher_tests.cpp;cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.cpp;cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.cpp;cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp;cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto;cmds/spartan/md/mdp3/back_testing/orchestrator/run_sim.cpp;cmds/spartan/md/mdp3/back_testing/pcap_db/PCapDB.cpp;cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp;cmds/spartan/md/mdp3/back_testing/simulator/Bridge/AlgoServicesBridge.cpp;cmds/spartan/md/mdp3/back_testing/simulator/Bridge/BeforeLaunchBridge.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IExchangeServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/MDP3EventGenerator.cpp;cmds/spartan/md/mdp3/back_testing/simulator/PCapEventSource.cpp;cmds/spartan/md/mdp3/back_testing/simulator/PandL.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.proto;cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/ExchangeGateway.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/TimerAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/AgentBase.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator2.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/tests/test_simulator_core.cpp;cmds/spartan/md/mdp3/back_testing/simulator/events/SimEvents.cpp;cmds/spartan/md/mdp3/common/InstrumentDefinition.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/common/mdp3_common.cpp;cmds/spartan/md/mdp3/common/tests/book_merging_tests.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/implied/implied_market.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/md/mdp3/utils/order_utils.cpp;cmds/spartan/md/mdp3/utils/polygon_utils.cpp;cmds/spartan/md/mdp3/utils/queries/DefinitionCollector.cpp;cmds/spartan/md/mdp3/utils/queries/FillCounter.cpp;cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp;cmds/spartan/md/mdp3/utils/queries/PolygonFinder.cpp;cmds/spartan/md/mdp3/utils/queries/PolygonOpportunityFinder.cpp;cmds/spartan/md/mdp3/utils/queries/TransactionDeducer.cpp;cmds/spartan/md/mdp3/utils/security_comp.cpp;cmds/spartan/md/mdp3/utils/security_parser.cpp;cmds/spartan/md/mdp3/utils/tests/test_polygon_finder.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;cmds/spartan/util/action_summarizer.cpp;cmds/spartan/util/args_parser.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index 33442403b..aa635c158 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -1,11 +1,16 @@ #include "QuickTest.h" #include "lib/utils/PBUtils.h" +#include "lib/utils/bts_chrono.h" #include "lib/utils/datetime.h" #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h" #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/pcap_db/PCapDB.h" #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEventSource.h" + #include #include #include @@ -15,8 +20,10 @@ namespace cmd { using namespace spartan::mdp3; int QuickTest::execute(std::vector const &) { - PCapDB db("310", datetime::make_ptime(2020, 9, 10)); - std::cout << db.get_orders().size() << std::endl; + + auto start = Timer::now(); + PCapEventSource src("/home/ppinto/2020-09-10_310_24h.pcap"); + std::cout << ms_diff(Timer::now(), start) << "ms to go through all\n"; return 0; } diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index ccb2d8dfa..c5df20d53 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -26,6 +26,7 @@ #include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/FillCounter.h" #include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/PolygonFinder.h" #include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/PolygonOpportunityFinder.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/run_sim.h" #include #include @@ -189,27 +190,23 @@ void sim_books(std::string const &sim_file) { } } +size_t total_orders = 0; void stats(std::string const &pcap) { - - std::cout << "MDP3order: " << sizeof(MDP3Order) << std::endl; - return; - auto out_path = drop_extensions(drop_extensions(pcap)) + ".order_counts.csv"; auto out_file = std::ofstream{out_path}; out_file << "sid,n_orders" << std::endl; - struct StatPrinter : IChannelObserver { StatPrinter(std::ostream &out) : out{out} {} std::ostream &out; void on_real_time_event(MDP3Event const &ev) override { for (auto s : ev.affected_securities) { - out << s->security_id().value() << "," << s->orders().size() - << std::endl; + total_orders += s->orders().size(); } } } stats{out_file}; observe_channel(pcap, stats); + std::cout << total_orders << std::endl; } std::map const &)>> @@ -233,6 +230,12 @@ std::map const &)>> return 0; }}, + {"sim2", + [](auto const &args) { + run_sim(args[1]); + return 0; + }}, + {"sim_books", [](auto const &args) { sim_books(args[1]); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver2.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver2.cpp new file mode 100644 index 000000000..7c0740f2b --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver2.cpp @@ -0,0 +1,154 @@ +#include "PolyObserver2.h" + +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h" + +namespace bts::spartan::mdp3 { + +static auto constexpr Worthless = -1.0; + +struct PolyObserver2::Target { + Polygon const polygon; + std::string const descriptor; + std::string const product; + double value = Worthless; + MDP3Quantity quantity = MDP3Quantity{0}; + void mark_as_worthless() { value = Worthless; } + bool has_value() const { return value > securities::market_epsilon(); } +}; + +struct PolyObserver2::Opportunity { + Target *target; + AlgoEventID event_id; + double value; + MDP3Quantity quantity; + MDP3TimeStamp transact_time; + MDP3TimeStamp observable_time; +}; + +struct PolyObserver2::PImpl { + std::vector> targets; + std::optional current_opportunity; + AlgoEventID event_id; + MDP3TimeStamp transact_time; + MDP3TimeStamp observable_time; +}; + +PolyObserver2::PolyObserver2() : me() {} +PolyObserver2::~PolyObserver2() {} + +void PolyObserver2::on_simulation_reset() { + me.reset(new PImpl()); + + for (auto &p : polygons()) { + auto buy_t = Target{p.second, polygon_descriptor(p.second, instruments()), + polygon_product(p.second, instruments())}; + auto sell_t = Target{flip(p.second), + polygon_descriptor(flip(p.second), instruments()), + polygon_product(p.second, instruments())}; + + me->targets.emplace_back(std::make_unique(buy_t)); + me->targets.emplace_back(std::make_unique(sell_t)); + } +} + +void PolyObserver2::on_notification(AlgoNotification const &n) { + me->event_id = n.event_id; + me->transact_time = n.transact_time; + me->observable_time = n.observable_time; + std::visit(*this, n.event); +} + +void PolyObserver2::eval() { + for (auto &t : me->targets) { + t->value = 0; + int qty = std::numeric_limits::max(); + for (auto &l : t->polygon.parts) { + auto mkt_side = securities::other_side(side_from_ratio(l.ratio)); + auto &mkt = books(l.sec_id).mbp_side(mkt_side); + auto this_qty = mkt[0].quantity.value(); + auto this_ratio = this_qty / std::abs(l.ratio); + if (this_ratio < qty) + qty = this_ratio; + if (qty == 0) { + t->mark_as_worthless(); + break; + } else { + auto cash = get_sec(l.sec_id).to_cash(mkt[0].price); + t->quantity.value() = qty; + t->value += -l.ratio * cash; + } + } + } +} + +void PolyObserver2::rank() { + std::sort(me->targets.begin(), me->targets.end(), + [](std::unique_ptr const &ptr1, + std::unique_ptr const &ptr2) { + auto val1 = ptr1->value * ptr1->quantity.value(); + auto val2 = ptr2->value * ptr2->quantity.value(); + if (std::abs(val1 - val2) > securities::market_epsilon()) + return val1 > val2; + else + return ptr1->polygon.parts.size() < + ptr2->polygon.parts.size(); + }); +} + +void PolyObserver2::try_start_new_opportunity() { + bts_assert(!me->current_opportunity); + auto best = me->targets.begin()->get(); + if (best->has_value()) { + // We have an opportunity lets record it + me->current_opportunity = + Opportunity{best, me->event_id, best->value, + best->quantity, me->transact_time, me->observable_time}; + log("opportunity_begin", best->descriptor, best->value, + best->quantity.value(), 0, 0, + us_diff(me->observable_time, me->transact_time), 1, + me->event_id.first, me->event_id.first); + } +} + +void PolyObserver2::analyze() { + if (me->current_opportunity) { + auto &opportunity = *me->current_opportunity; + auto &target = *opportunity.target; + if (target.has_value()) { + // Add value and still has value, opportunity is still ongoing + log( + "opportunity_continue", target.descriptor, target.value, + target.quantity.value(), + us_diff(me->transact_time,opportunity.observable_time), + us_diff(me->transact_time,opportunity.transact_time), + us_diff(opportunity.observable_time,opportunity.transact_time), + me->event_id.first - opportunity.event_id.first, + opportunity.event_id.first, me->event_id.first); + } else { + // Had value but not anymore, opportunity is concluded + log( + "opportunity_end", target.descriptor, opportunity.value, + opportunity.quantity.value(), + us_diff(me->transact_time,opportunity.observable_time), + us_diff(me->transact_time,opportunity.transact_time), + us_diff(opportunity.observable_time,opportunity.transact_time), + me->event_id.first - opportunity.event_id.first, + opportunity.event_id.first, me->event_id.first-1); + me->current_opportunity.reset(); + try_start_new_opportunity(); + } + } else { + // We did not have an opportunity before. Maybe we do now? + try_start_new_opportunity(); + } +} + +void PolyObserver2::operator()(const MBPUpdate &) { + eval(); + rank(); + analyze(); +} + +void PolyObserver2::operator()(const OrderUpdate &) {} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver2.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver2.h new file mode 100644 index 000000000..c7fa8d456 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver2.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include + +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoModel.h" + +namespace bts::spartan::mdp3 { +struct Polygon; + +class PolyObserver2 : public AlgoModel { +public: + PolyObserver2(); + ~PolyObserver2(); + void on_simulation_reset() override; + void on_notification(AlgoNotification const &) override; + void operator()(const MBPUpdate &); + void operator()(const OrderUpdate &); +private: + struct Target; + struct Opportunity; + struct PImpl; + void eval(); + void rank(); + void analyze(); + void try_start_new_opportunity(); + std::unique_ptr me; +}; + +} + diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp index ef79c4077..17e3032e8 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp @@ -4,15 +4,6 @@ namespace bts::spartan::mdp3::iceberg_helper { template using flat_map = boost::container::flat_map; -IcebergQuantity to_iceberg_qty(const DeducedIceberg& dib) { - // initializer lists with diff fields of the same type seem dangerous - // ...what if the declaration order changes? accidents happen. - IcebergQuantity ib_qty; - ib_qty.show = dib.show_quantity; - ib_qty.total = dib.estimated_quantity; - return ib_qty; -} - void add_iceberg_annotation( MDP3Order& order, const flat_map& icebergs, @@ -20,9 +11,9 @@ void add_iceberg_annotation( auto it = icebergs.find(order.key); if (it != icebergs.end()) { auto over_it = overrides.find(order.key); - auto ibq = to_iceberg_qty(it->second); + auto ibq = it->second.quantity; if (over_it != overrides.end()) - ibq = to_iceberg_qty(over_it->second); + ibq = over_it->second.quantity; if (order.iceberg_quantity != ibq) order.iceberg_quantity = ibq; // adjust order's actual quantity in accordance @@ -68,8 +59,8 @@ void detect_show_qty_changes( if (ibs_with_fills.find(ch.order.key) != ibs_with_fills.end()) continue; // subtract the difference from the estimated (total) qty - ib_it->second.estimated_quantity -= (ib_it->second.show_quantity - ch.order.quantity); - ib_it->second.show_quantity = ch.order.quantity; + ib_it->second.quantity.total -= (ib_it->second.quantity.show - ch.order.quantity); + ib_it->second.quantity.show = ch.order.quantity; show_qty_changes.insert_or_assign(ib_it->first, ib_it->second); break; } @@ -110,8 +101,8 @@ void subtract_iceberg_fills( auto ib_it = icebergs.find(f.key); if (ib_it != icebergs.end()) { // watch out for overfills - ib_it->second.estimated_quantity -= - std::min(f.quantity, ib_it->second.estimated_quantity); + ib_it->second.quantity.total -= + std::min(f.quantity, ib_it->second.quantity.total); } } } @@ -133,4 +124,4 @@ void add_iceberg_annotation( } } } -} // namespace bts::spartan::mdp3 \ No newline at end of file +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h index 3bb68d238..bb8ef261f 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h @@ -19,7 +19,6 @@ #include namespace bts::spartan::mdp3::iceberg_helper { -IcebergQuantity to_iceberg_qty(const DeducedIceberg&); void add_iceberg_annotation( MDP3Order& order, @@ -55,4 +54,4 @@ inline void subtract_iceberg_fills( first++; } } -} // namespace bts::spartan::mdp3::iceberg_helper \ No newline at end of file +} // namespace bts::spartan::mdp3::iceberg_helper diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.cpp index 06d84c59a..ea9121611 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.cpp @@ -39,8 +39,8 @@ flat_map find_icebergs(const std::string& pcap) { // agree with one another, wrt the sequence of fields auto read_op = [](std::istream& in, DeducedIceberg& d) -> std::istream& { d.key.value() = file::read_from_bin(in); - d.show_quantity.value() = file::read_from_bin(in); - d.estimated_quantity.value() = file::read_from_bin(in); + d.quantity.show.value() = file::read_from_bin(in); + d.quantity.total.value() = file::read_from_bin(in); d.filled = file::read_from_bin(in); return in; }; @@ -51,8 +51,8 @@ flat_map find_icebergs(const std::string& pcap) { const std::pair& p) -> std::ostream& { auto& d = p.second; file::write_to_bin(out, d.key.value()); - file::write_to_bin(out, d.show_quantity.value()); - file::write_to_bin(out, d.estimated_quantity.value()); + file::write_to_bin(out, d.quantity.show.value()); + file::write_to_bin(out, d.quantity.total.value()); file::write_to_bin(out, d.filled); return out; }; @@ -66,4 +66,4 @@ flat_map find_icebergs(const std::string& pcap) { } return rv; } -} // namespace bts::spartan::mdp3::iceberg_file_cache \ No newline at end of file +} // namespace bts::spartan::mdp3::iceberg_file_cache diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/run_sim.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/run_sim.cpp new file mode 100644 index 000000000..a3c293a8f --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/run_sim.cpp @@ -0,0 +1,60 @@ +#include "run_sim.h" + +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEventSource.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoModel.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/ExchangeGateway.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator2.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver2.h" + +#include "lib/utils/PBUtils.h" +#include + +namespace bts::spartan::mdp3 { + +std::unique_ptr make_algo_model(protos::SimRunOptions const &options, + PolygonMap const &pm, + InstrumentMap const &im, + SimLogger const &l) { + std::unique_ptr result = std::make_unique (); + result->init_algo_model(options, pm, im, l); + return result; +} + +std::string pcap_location(std::string const &channel, std::string const &date) { + auto base_path = os::bts_data_path(); + auto pcap_path = fmt::format("{}pcaps/{}/{}_{}_24h.pcap.xz", base_path, + channel, date, channel); + return pcap_path; +} + +void run_sim(protos::SimRunOptions const &options) { + auto pcap_loc = pcap_location(options.channels()[0], options.market_date()); + PCapEventSource source(pcap_loc); + + SimDB db; + auto sid = db.create_session(options.algo()); + auto rid = db.begin_run(sid, options); + auto logger = [rid, &db](auto ts, auto const &entry_type, auto const &args) { + db.log(rid, ts, entry_type, args); + }; + + auto algo = make_algo_model(options, source.polygons(), source.instruments(), + logger); + auto event_gen = source.get_event_generator(); + ExchangeGateway gw(options,source.icebergs(),event_gen,logger); + + sim::Simulator2 sim {algo.get(),&gw}; + sim.run(); +} + +void run_sim(std::string const&path_to_run_options) { + spartan::protos::SimRunOptions options; + std::ifstream proto_in(path_to_run_options); + std::string str((std::istreambuf_iterator(proto_in)), + std::istreambuf_iterator()); + bts_assert(utils::pb::from_string(str, &options)); + run_sim(options); +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/run_sim.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/run_sim.h new file mode 100644 index 000000000..4f9f2c076 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/run_sim.h @@ -0,0 +1,15 @@ +#pragma once + +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimLogger.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h" + +namespace bts::spartan::mdp3 { +class MDP3EventGenerator; +class AlgoModel; + +void run_sim(protos::SimRunOptions const&); +void run_sim(std::string const&path_to_run_options); + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/pcap_db/PCapDB.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/pcap_db/PCapDB.cpp index 66adb2bef..be98ee5a6 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/pcap_db/PCapDB.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/pcap_db/PCapDB.cpp @@ -47,8 +47,7 @@ std::vector PCapDB::get_orders() const { SQLite::Statement query_orders( me->db, "select eid, t_time, sid, status,impl,change, ok, side, " - "pri, price, qty from event_items where type='order_change' " - "order by eid"); + "pri, price, qty from event_items where type='order_change'"); std::lock_guard l(me->db_lock); while (query_orders.executeStep()) { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h index 6312509d0..fcc9e0374 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h @@ -26,6 +26,7 @@ class IAlgoModel { virtual void finit() {} virtual void on_time(MDP3TimeStamp) {} virtual void on_notification(AlgoNotification const&) = 0; + virtual void on_simulation_reset() {}; }; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/MDP3EventGenerator.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/MDP3EventGenerator.cpp new file mode 100644 index 000000000..3c16aceda --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/MDP3EventGenerator.cpp @@ -0,0 +1,4 @@ +#include "MDP3EventGenerator.h" + +namespace bts::spartan::mdp3 { +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/MDP3EventGenerator.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/MDP3EventGenerator.h new file mode 100644 index 000000000..a03975ab6 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/MDP3EventGenerator.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEvent.h" + +namespace bts::spartan::mdp3 { + +class MDP3EventGenerator { +public: + MDP3EventGenerator(std::vector const&all_events) : all_events(&all_events), idx(0) {} + PCapEvent const* next_event() { + if (idx < all_events->size()) { + auto &result = (*all_events)[idx]; + idx++; + return &result; + } + else + return nullptr; + } + +private: + std::vector const*all_events; + size_t idx; +}; + + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEvent.h new file mode 100644 index 000000000..1bcac11f4 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEvent.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +class PCapEvent { + public: + MDP3TimeStamp transact_time = MDP3TimeStamp{0}; + size_t event_id = 0; + MDP3TimeStamp first_packet_send_time; + MDP3TimeStamp last_packet_send_time; + uint32_t first_packet_sn; + uint32_t last_packet_sn; + boost::container::small_vector affected_securities; +}; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEventSource.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEventSource.cpp new file mode 100644 index 000000000..d29c2046b --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEventSource.cpp @@ -0,0 +1,74 @@ +#include "PCapEventSource.h" + +#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/DefinitionCollector.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/PolygonFinder.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEvent.h" + +namespace bts::spartan::mdp3 { + +struct EventCollector: mdp3::IChannelObserver { + std::vector all_events; + + void init() override { + all_events.reserve(1000*1000*100); + } + + void on_recovery_complete( + std::vector const &snaps) override { + auto &ev = all_events.emplace_back(); + for (auto snap_ptr : snaps) { + auto &snap_cpy = ev.affected_securities.emplace_back(); + snap_cpy.light_copy(*snap_ptr); + } + } + + void on_real_time_event(MDP3Event const &ev_in) override { + auto &ev = all_events.emplace_back(); + ev.transact_time = ev_in.transact_time; + ev.event_id = ev_in.event_id; + ev.first_packet_send_time = ev_in.first_packet_send_time; + ev.last_packet_send_time = ev_in.last_packet_send_time; + ev.first_packet_sn = ev.first_packet_sn; + ev.last_packet_sn = ev.last_packet_sn; + + for (auto snap_ptr : ev_in.affected_securities) { + auto &snap_cpy = ev.affected_securities.emplace_back(); + snap_cpy.light_copy(*snap_ptr); + } + } +}; + +struct PCapEventSource::PImpl { + std::string pcap_file; + DefinitionCollector dc = {}; + PolygonFinder pf = {}; + IcebergFinder ice_finder = {}; + EventCollector ev_collector = {}; +}; + +PCapEventSource::PCapEventSource(std::string const &pcap) + : me(new PImpl{pcap}) { + observe_channel(pcap, {&me->dc, &me->pf, &me->ice_finder,&me->ev_collector}); +} + +PCapEventSource::~PCapEventSource() {} + +InstrumentMap const& PCapEventSource::instruments() const { + return me->dc.ptr_result; +} + +std::unordered_map const& PCapEventSource::polygons() const { + return me->pf.result; +} + +IcebergMap const&PCapEventSource::icebergs() const { + return me->ice_finder.result; +} + + MDP3EventGenerator PCapEventSource::get_event_generator() const { + return MDP3EventGenerator {me->ev_collector.all_events}; +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEventSource.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEventSource.h new file mode 100644 index 000000000..4eb789c50 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEventSource.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include +#include + +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/MDP3EventGenerator.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentMap.h" + +namespace bts::spartan::mdp3 { + +class InstrumentDefinition; +class MDP3Event; + +class PCapEventSource { +public: + PCapEventSource(std::string const &pcap); + ~PCapEventSource(); + InstrumentMap const &instruments() const ; + std::unordered_map const &polygons() const; + IcebergMap const&icebergs() const; + + MDP3EventGenerator get_event_generator() const; +private: + struct PImpl; + std::unique_ptr me; +}; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp index af7ee3e22..ba327c1cb 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp @@ -37,8 +37,8 @@ protos::SimInput to_proto(SimInput const &in) { out.set_max_order_id(in.max_order_id.value()); // instruments - for (auto &i : in.instruments) - out.add_instruments(std::string{i.second.data()}); + // for (auto &i : in.instruments) + // out.add_instruments(std::string{i.second.data()}); // polygons for (auto &[id, p] : in.polygons) { @@ -84,10 +84,10 @@ SimInput make_sim_input_from_proto(protos::SimInput const &proto) { SimInput result; result.max_order_id = MDP3OrderKey{proto.max_order_id()}; - for (auto &ibytes : proto.instruments()) { - auto instrument = InstrumentDefinition::make_from_bytes(ibytes); - result.instruments.insert({instrument.security_id(), instrument}); - } + // for (auto &ibytes : proto.instruments()) { + // auto instrument = InstrumentDefinition::make_from_bytes(ibytes); + // result.instruments.insert({instrument.security_id(), instrument}); + // } for (auto &spoly : proto.polygons()) { auto &poly = result.polygons[spoly.id()]; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoModel.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoModel.cpp new file mode 100644 index 000000000..8d6aa8aa4 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoModel.cpp @@ -0,0 +1,238 @@ +#include "AlgoModel.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PandL.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PandLSummary.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/ISimulator2.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/events/SimEvents.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" + +#include + +namespace bts::spartan::mdp3 { + +enum class AlgoModel::Mode { REPLAY, SIMULATION, RESETTING }; + +struct AlgoModel::PImpl { + PolygonMap const polygons; + InstrumentMap const instruments; + SimLogger logger; + protos::SimRunOptions const options; + Mode mode = Mode::REPLAY; + sim::ISimulator2 *sim = nullptr; + MDP3OrderKey last_order_key = + MDP3OrderKey{std::numeric_limits::max()}; + std::unordered_set my_orders = {}; + std::vector my_fills = {}; + std::unordered_map books = {}; +}; + +AlgoModel::AlgoModel() : me() {} +AlgoModel::~AlgoModel() {} + +void AlgoModel::init_algo_model(protos::SimRunOptions const &options, + PolygonMap const &pm, InstrumentMap const &im, + SimLogger const &logger) { + bts_assert(!me); + me.reset(new PImpl{pm, im, logger, options}); +} + +std::string AlgoModel::param(std::string const &name, + std::string const &default_value) const { + bts_assert(me); + bts_assert(me->sim); + auto it = me->options.parameters().find(name); + if (it != me->options.parameters().end()) + return it->second; + else + return default_value; +} + +int AlgoModel::param(std::string const &name, int default_value) const { + bts_assert(me); + bts_assert(me->sim); + auto it = me->options.parameters().find(name); + if (it != me->options.parameters().end()) + return std::stoi(it->second); + else + return default_value; +} + +PolygonMap const &AlgoModel::polygons() const { + bts_assert(me); + bts_assert(me->sim); + return me->polygons; +} + +InstrumentMap const &AlgoModel::instruments() const { + bts_assert(me); + bts_assert(me->sim); + return me->instruments; +} + +MDP3TimeStamp AlgoModel::time() const { + bts_assert(me); + bts_assert(me->sim); + return MDP3TimeStamp(me->sim->time()); +} + +InstrumentDefinition const &AlgoModel::get_sec(MDP3SecurityID id) const { + bts_assert(me); + bts_assert(me->sim); + + auto it = me->instruments.find(id); + bts_assert(it != me->instruments.end()); + return *it->second; +} + +MBPUpdate const& AlgoModel::books(MDP3SecurityID id) const { + bts_assert(me); + bts_assert(me->sim); + bts_assert(me->mode != Mode::RESETTING); + + auto it = me->books.find(id); + bts_assert(it != me->books.end()); + return it->second; +} + +MDP3OrderKey AlgoModel::send(SmallOrderChange const &change, + size_t propagation_delay) { + bts_assert(me); + bts_assert(me->sim); + bts_assert(me->mode != Mode::RESETTING); + + if (me->mode == Mode::REPLAY) + me->mode = Mode::SIMULATION; + + auto total_delay = me->options.base_algo_latency_ns() + propagation_delay; + auto t_time = me->sim->time() + total_delay; + + if (change.change == OrderChange::added) { + auto cpy = change; + cpy.order.key = generate_order_key(); + me->my_orders.insert(cpy.order.key); + me->sim->emit_delayed( + total_delay, AlgoAction{MDP3TimeStamp{t_time}, cpy}); + return cpy.order.key; + } else { + bts_assert(change.order.key != MDP3OrderKey{}); + me->sim->emit_delayed( + total_delay, AlgoAction{MDP3TimeStamp{t_time}, change}); + return change.order.key; + } +} + +void AlgoModel::log_pandl() { + bts_assert(me); + bts_assert(me->sim); + + auto mkt_pricer = [this](MDP3SecurityID id, MDP3Quantity qty, api::Side s) { + return price_at_market(id, qty, s); + }; + + PandL pl{me->instruments, mkt_pricer}; + auto outright_pos = pl.outright_position(me->my_fills); + + log("pnl", pl.net_cash(me->my_fills), + pl.position_liquidation_value(outright_pos)); +} + +void AlgoModel::sleep(size_t ns) { + bts_assert(me); + bts_assert(me->sim); + me->sim->sleep(ns); +} + +void AlgoModel::reset_simulation() { + bts_assert(me); + bts_assert(me->sim); + bts_assert(me->mode == Mode::SIMULATION); + me->sim->emit(); + me->mode = Mode::RESETTING; +} + +void AlgoModel::init(sim::ISimulator2 &sim) { + bts_assert(me); + bts_assert(!me->sim); + me->sim = ∼ + on_simulation_reset(); +} + +void AlgoModel::handle(sim::Event const &ev) { + bts_assert(me); + bts_assert(me->sim); + if (dynamic_cast(&ev)) { + if (me->mode != Mode::RESETTING) + on_time(MDP3TimeStamp{time()}); + } else if (auto n = dynamic_cast(&ev); n) { + std::visit(overloaded{[&](const MBPUpdate &mbp) { + me->books[mbp.security_id] = mbp; + if (me->mode != Mode::RESETTING) + on_notification(n->ev); + }, + [&](const OrderUpdate &update) { + if (me->my_orders.count(update.key)) { + if (update.last_fills.value()) { + me->my_fills.push_back(update); + } + if (me->mode != Mode::RESETTING) + on_notification(n->ev); + } + }}, + n->ev.event); + } else if (dynamic_cast(&ev)) { + bts_assert(me->mode == Mode::RESETTING); + me->mode = Mode::REPLAY; + me->books.clear(); + on_simulation_reset(); + } +} + +void AlgoModel::finit() { log_pandl(); } + +MDP3OrderKey AlgoModel::generate_order_key() { + return MDP3OrderKey{me->last_order_key.value()--}; +} + +double AlgoModel::price_at_market(MDP3SecurityID id, MDP3Quantity qty, + api::Side s) const { + bts_assert(me); + bts_assert(me->sim); + + auto it = me->books.find(id); + if (it != me->books.end()) + return os::nan(); + else { + auto &book = it->second.mbp_side(s); + double cummulative_flow = 0; + auto remaining_qty = qty.value(); + for (auto &l : book) { + if (remaining_qty == 0) + return cummulative_flow; + else { + auto from_this_level = std::min(remaining_qty, l.quantity.value()); + cummulative_flow += + (from_this_level * get_sec(id).to_cash(l.price.price())); + remaining_qty -= from_this_level; + } + } + if (remaining_qty) + return os::nan(); + else + return cummulative_flow; + } +} + +void AlgoModel::_log(std::string const &log_entry_type, + std::initializer_list const &log_args) { + bts_assert(me); + bts_assert(me->sim); + + std::ostringstream str; + str << to_time_string(time()) << " " << log_entry_type << " "; + for (auto const &a : log_args) + str << a << " "; + LOG(Spartan, Info, "{}", str.str()); + me->logger(time(), log_entry_type, log_args); +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoModel.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoModel.h new file mode 100644 index 000000000..b7c5eaee0 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoModel.h @@ -0,0 +1,71 @@ +#pragma once + +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimLogger.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/IAgent2.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h" + +namespace bts::spartan::mdp3 { + +// Base class for modeling algos. Derive from this to implement your +// algo model. + +class AlgoModel : public sim::IAgent2 { + + public: + ~AlgoModel(); + // Override these + virtual void on_time(MDP3TimeStamp){}; + virtual void on_simulation_reset() = 0; + virtual void on_notification(AlgoNotification const &) = 0; + + // Call these + std::string param(std::string const ¶m_name, + std::string const &default_value = "") const; + int param(std::string const ¶m_name, int default_value = 0) const; + + PolygonMap const &polygons() const; + InstrumentMap const &instruments() const; + MDP3TimeStamp time() const; + InstrumentDefinition const &get_sec(MDP3SecurityID id) const; + MBPUpdate const& books(MDP3SecurityID) const; + MDP3OrderKey send(SmallOrderChange const &, size_t propagation_delay = 0); + + void log_pandl(); + void sleep(size_t ns); + void reset_simulation(); + template + void log(std::string const &log_entry_type, T &&... args) { + auto lst = + std::initializer_list{to_str(std::forward(args))...}; + _log(log_entry_type, lst); + } + + protected: + AlgoModel(); + + friend std::unique_ptr + make_algo_model(protos::SimRunOptions const &options, PolygonMap const &pm, + InstrumentMap const &im, SimLogger const &l); + void init_algo_model(protos::SimRunOptions const &,PolygonMap const &, InstrumentMap const &, + SimLogger const &); + + // From IAgent + void init(sim::ISimulator2 &) override; + void handle(sim::Event const &) override; + void finit() override; + + private: + enum class Mode; + struct PImpl; + std::unique_ptr me; + + MDP3OrderKey generate_order_key(); + double price_at_market(MDP3SecurityID, MDP3Quantity, api::Side s) const; + void _log(std::string const &log_entry_type, + std::initializer_list const &log_args); +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/ExchangeGateway.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/ExchangeGateway.cpp new file mode 100644 index 000000000..357b93718 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/ExchangeGateway.cpp @@ -0,0 +1,320 @@ +#include "ExchangeGateway.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/MDP3EventGenerator.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEventSource.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/ISimulator2.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentMap.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" + +namespace bts::spartan::mdp3 { + +enum class ExchangeGateway::Mode { REPLAY, SIMULATION }; + +struct ExchangeGateway::SimulationState { + MergedMatcher matcher = {}; + + // CME does not guarantee uniqueness of order keys during a + // session, though of course, they must guarantee uniqueness of + // *alive* order keys. During a simulation its possible that the + // life cycle of a CME order changes from what happened in reality, + // which could lead to key colision if we used the cme key. For + // this reason we must remap the CME keys to ones we know will be + // unique during the entire simulation. + std::unordered_map cme_key_map; + + // The matcher does not currently support non-trading security + // states. For this reason we have to use this map to detect + // changes of trading state in securities so that when a security + // starts trading we can seed the matcher with existing orders and + // when it stops trading we can remove its orders + std::unordered_set ready_secs; +}; + +struct ExchangeGateway::PImpl { + MDP3EventGenerator &event_generator; + IcebergMap const &ib_map; + SimLogger logger; + protos::SimRunOptions const options; + AlgoEventID current_event = {0, 0}; + sim::ISimulator2 *sim = nullptr; + std::unordered_map snapshots = {}; + Mode mode = Mode::REPLAY; + std::optional simulation = {}; +}; + +ExchangeGateway::ExchangeGateway(protos::SimRunOptions const &options, + IcebergMap const &ib_map, + MDP3EventGenerator &ev_gen, + SimLogger const &logger) + : me(new PImpl{ev_gen, ib_map, logger, options}) {} + +ExchangeGateway::~ExchangeGateway() {} + +void ExchangeGateway::init(sim::ISimulator2 &sim) { + me->sim = ∼ + enqueue_next_pcap_event(); +} + +void ExchangeGateway::handle(sim::Event const &ev) { + if (auto aa = dynamic_cast(&ev); aa) + handle(*aa); + else if (auto pcap_ev = dynamic_cast(&ev); pcap_ev) { + handle(*pcap_ev); + } else if (auto r = dynamic_cast(&ev); r) { + handle(*r); + } +} + +void ExchangeGateway::handle(AlgoActionEvent const &e) { + if (me->mode == Mode::REPLAY) + start_simulation(); + me->current_event.second++; + auto const &algo_action = e.ev; + auto transaction_delay = me->options.transaction_latency_ns(); + match_and_send_effects(MDP3TimeStamp{e.observable_time}, transaction_delay, + {algo_action.change}); + me->sim->sleep(transaction_delay); +} + +void ExchangeGateway::handle(ResetToPCapRequestEvent const &) { + bts_assert(me->mode == Mode::SIMULATION); + bts_assert(me->simulation); + + me->simulation.reset(); + me->mode = Mode::REPLAY; + me->current_event.second = 0; + + auto now = me->sim->time(); + // We want to guarantee the ResetToPCapReplyEvent event is not + // visible until any already scheduled AlgoNotificationEvent events + // are observed. This transaction delay will ensure that. + auto transaction_delay = std::max(me->options.mbp_update_delay_ns(), + me->options.order_update_delay_ns()); + auto md_delay = me->options.mbp_update_delay_ns(); + + me->sim->emit_delayed(transaction_delay); + + for (auto &[id, snap] : me->snapshots) + if (snap.ready_to_trade()) { + me->sim->emit_delayed( + md_delay, AlgoNotification{me->current_event, MDP3TimeStamp{now}, + MDP3TimeStamp(now + transaction_delay), + MBPUpdate{ + id, + snap.book(api::SIDE_BUY), + snap.book(api::SIDE_SELL), + snap.implied_book(api::SIDE_BUY), + snap.implied_book(api::SIDE_SELL), + }}); + } +} + +void ExchangeGateway::handle(PCapSimEvent const &ev) { + auto now = me->sim->time(); + auto event = &ev.ev; + auto send_time = event->last_packet_send_time.value(); + auto transaction_delay = + now > send_time ? me->options.transaction_latency_ns() : send_time - now; + + me->current_event.first = event->event_id; + + switch (me->mode) { + case Mode::REPLAY: { + auto md_delay = transaction_delay + me->options.mbp_update_delay_ns(); + for (auto &snap : event->affected_securities) { + me->snapshots[snap.security_id()].update_with(snap); + if (snap.ready_to_trade()) { + me->sim->emit_delayed( + md_delay, + AlgoNotification{me->current_event, event->transact_time, + MDP3TimeStamp(now + transaction_delay), + MBPUpdate{ + snap.security_id(), + snap.book(api::SIDE_BUY), + snap.book(api::SIDE_SELL), + snap.implied_book(api::SIDE_BUY), + snap.implied_book(api::SIDE_SELL), + }}); + } + } + } break; + case Mode::SIMULATION: + for (auto &snap : event->affected_securities) { + auto this_sid = snap.security_id(); + me->snapshots[this_sid].update_with(snap); + + bool was_ready = me->simulation->ready_secs.count(this_sid); + bool is_ready = snap.ready_to_trade(); + + // The security was not ready for trading but it is now. We + // need to boostrap the matcher with its current order book. + if (!was_ready && is_ready) { + bts_assert(snap.order_changes().empty()); + bootstrap_simulated_order_book(event->transact_time, + transaction_delay, this_sid); + } + + // The security was trading but is no longer. We need to + // clear it from the matcher. + else if (was_ready && !is_ready) { + bts_assert(snap.order_changes().empty()); + clear_simulated_order_book(event->transact_time, transaction_delay, + this_sid); + } + + // The security was and remains ready. We need to incorporate + // the latest deduced orders in the matcher and send out any + // matcher effects. + else if (was_ready && is_ready) { + auto changes = snap.deduced_order_changes(); + SmallOrderChangeVec matcher_input; + for (auto &c : changes) + matcher_input.push_back(correct_pcap_order(c)); + match_and_send_effects(event->transact_time, transaction_delay, + matcher_input); + } + } + break; + } + enqueue_next_pcap_event(); + me->sim->sleep(transaction_delay); +} + +void ExchangeGateway::match_and_send_effects( + MDP3TimeStamp transact_time, size_t transaction_delay, + SmallOrderChangeVec const &matcher_input) { + + bts_assert(me->mode == Mode::SIMULATION); + bts_assert(me->simulation); + + auto &simulation = *me->simulation; + auto now = me->sim->time(); + auto md_delay = transaction_delay + me->options.mbp_update_delay_ns(); + auto fill_delay = transaction_delay + me->options.order_update_delay_ns(); + + std::vector effects; + simulation.matcher.match(MergedMatcher::SIMULATOR_INPUT, matcher_input, + &effects); + + for (auto &ef : effects) + std::visit( + overloaded{ + [&](MBPUpdate &mbp) { + me->sim->emit_delayed( + md_delay, + AlgoNotification{me->current_event, transact_time, + MDP3TimeStamp(now + transaction_delay), + mbp}); + }, + [&](OrderUpdate const &order_update) { + me->sim->emit_delayed( + fill_delay, + AlgoNotification{me->current_event, transact_time, + MDP3TimeStamp(now + transaction_delay), + order_update}); + }}, + ef); +} + +void ExchangeGateway::start_simulation() { + bts_assert(me->mode == Mode::REPLAY); + bts_assert(!me->simulation); + + auto &simulation = me->simulation.emplace(); + auto &matcher = simulation.matcher; + for (auto &[sec_id, snap] : me->snapshots) { + matcher.add_def(snap.definition()); + if (snap.ready_to_trade()) { + simulation.ready_secs.insert(sec_id); + SmallOrderChangeVec matcher_input; + for (auto &p : snap.orders()) + matcher_input.push_back(correct_pcap_order( + OrderChange{p.second, p.second, OrderChange::added})); + } + } + me->mode = Mode::SIMULATION; +} + +void ExchangeGateway::bootstrap_simulated_order_book( + MDP3TimeStamp transact_time, size_t transaction_delay, MDP3SecurityID sid) { + bts_assert(me->mode == Mode::SIMULATION); + bts_assert(me->simulation); + auto &simulation = *me->simulation; + + simulation.ready_secs.insert(sid); + auto &snap = me->snapshots[sid]; + SmallOrderChangeVec matcher_input; + for (auto const &p : snap.orders()) + matcher_input.push_back(correct_pcap_order( + OrderChange{p.second, p.second, OrderChange::added})); + match_and_send_effects(transact_time, transaction_delay, matcher_input); +} + +void ExchangeGateway::clear_simulated_order_book(MDP3TimeStamp transact_time, + size_t transaction_delay, + MDP3SecurityID sid) { + bts_assert(me->mode == Mode::SIMULATION); + bts_assert(me->simulation); + auto &simulation = *me->simulation; + + simulation.ready_secs.erase(sid); + auto &snap = me->snapshots[sid]; + SmallOrderChangeVec matcher_input; + for (auto const &p : snap.orders()) + matcher_input.push_back(correct_pcap_order( + OrderChange{p.second, p.second, OrderChange::removed})); + match_and_send_effects(transact_time, transaction_delay, matcher_input); +} + +void ExchangeGateway::enqueue_next_pcap_event() { + auto event = me->event_generator.next_event(); + if (!event) + return; + auto now = me->sim->time(); + auto delay = now < event->transact_time.value() + ? event->transact_time.value() - now + : 0ul; + me->sim->emit_delayed(delay, PCapEvent{*event}); +} + +SmallOrderChange ExchangeGateway::correct_pcap_order(OrderChange const &o) { + bts_assert(me->mode == Mode::SIMULATION); + bts_assert(me->simulation); + auto &key_map = me->simulation->cme_key_map; + + bts_assert(o.order.key.value()); + MDP3OrderKey &our_key = key_map[o.order.key]; + + SmallOrderChange result(o); + result.original_key = o.order.key; + + switch (o.change) { + case OrderChange::added: { + our_key = MDP3OrderKey{key_map.size() + 1}; + result.order.key = our_key; + break; + } + case OrderChange::removed: { + bts_assert(our_key != MDP3OrderKey{}); + result.order.key = our_key; + our_key = MDP3OrderKey{}; + break; + } + case OrderChange::updated: { + bts_assert(our_key != MDP3OrderKey{}); + result.order.key = our_key; + } + } + + // This isn't exactly right, as CME can re-use an order key, so an + // order that used to be an iceberg can stop being one. For now, we + // ignore that possibility, since both being an iceberg and getting + // your key re-used are low probability. + auto it = me->ib_map.find(result.original_key); + if (it != me->ib_map.end()) + result.order.iceberg_quantity.emplace() = it->second.quantity; + return result; +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/ExchangeGateway.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/ExchangeGateway.h new file mode 100644 index 000000000..c867bb359 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/ExchangeGateway.h @@ -0,0 +1,48 @@ +#pragma once + +#include "../../simulator/SimLogger.h" +#include "../events/SimEvents.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/IAgent2.h" + +namespace bts::spartan::mdp3 { + +class MDP3EventGenerator; + +// Simulates a single CME exchange gateway using a single day's worth +// of market activity. +class ExchangeGateway : public sim::IAgent2 { + public: + ExchangeGateway(protos::SimRunOptions const &, IcebergMap const &, + MDP3EventGenerator &, SimLogger const &); + ~ExchangeGateway(); + void init(sim::ISimulator2 &) override; + void handle(sim::Event const &) override; + + private: + enum class Mode; + struct PImpl; + struct SimulationState; + + void match_and_send_effects(MDP3TimeStamp transact_time, + size_t transaction_delay, + SmallOrderChangeVec const &); + void bootstrap_simulated_order_book(MDP3TimeStamp transact_time, + size_t transaction_delay, + MDP3SecurityID); + void clear_simulated_order_book(MDP3TimeStamp transact_time, + size_t transaction_delay, MDP3SecurityID); + SmallOrderChange correct_pcap_order(OrderChange const &pcap_order); + void enqueue_next_pcap_event(); + + + void emit_next_pcap_event(); + void emit_next_simulation_event(); + void handle(AlgoActionEvent const &); + void handle(ResetToPCapRequestEvent const &); + void handle(PCapSimEvent const&); + void start_simulation(); + std::unique_ptr me; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/IAgent2.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/IAgent2.h new file mode 100644 index 000000000..eaef0abb4 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/IAgent2.h @@ -0,0 +1,17 @@ +#pragma once + +#include "Event.h" + +namespace bts::spartan::sim { + +class ISimulator2; + +class IAgent2 { +public: + virtual ~IAgent2() {}; + virtual void init(ISimulator2&) = 0; + virtual void handle(Event const&) = 0; + virtual void finit() {}; +}; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/ISimulator2.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/ISimulator2.h new file mode 100644 index 000000000..47f21f92b --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/ISimulator2.h @@ -0,0 +1,32 @@ +#pragma once + +#include "Event.h" + +namespace bts::spartan::sim { + +class Simulator2; + +class ISimulator2 { +public: + virtual ~ISimulator2() {} + + template + void emit(Args &&... args) { + _emit(0,std::make_shared (std::forward(args)...)); + } + + template + void emit_delayed(size_t delay_ns,Args &&... args) { + _emit(delay_ns,std::make_shared (std::forward(args)...)); + } + + virtual TimeStamp time() = 0; + virtual void sleep(size_t ns) = 0; + +protected: + friend class Simulator2; + virtual void _emit(size_t delay_ns,EventPtr const&) = 0; +}; + +} + diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator2.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator2.cpp new file mode 100644 index 000000000..6b5f0ed45 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator2.cpp @@ -0,0 +1,154 @@ +#include "Simulator2.h" +#include "Event.h" +#include "IAgent2.h" +#include "ISimulator2.h" +#include "lib/utils/bts_assert.h" + +#include + +namespace bts::spartan::sim { + +struct EventSort { + bool operator()(EventPtr const &l, EventPtr const &r) const { + if (l->observable_time != r->observable_time) + return l->observable_time > r->observable_time; + else + return l->event_idx > r->event_idx; + } +}; + +using FutureEventQueue = + std::priority_queue, EventSort>; + +using PastEventQueue = std::vector; + +struct Simulator2::AgentRec : ISimulator2 { + AgentRec(IAgent2 *agent, Simulator2 &owner) + : owner(&owner), agent(agent), wakeup_ts(0), next_event_idx(0) {} + + void init() { agent->init(*this); } + void finit() { agent->finit(); } + + // From ISimulator2 + TimeStamp time() override { return owner->now(); } + void sleep(size_t ns) override { + if (is_ready()) + wakeup_ts = time() + ns; + else { + bts_assert(wakeup_ts > owner->now()); + wakeup_ts += ns; + } + } + void _emit(size_t delay_ns, EventPtr const &ev) override { + owner->emit(delay_ns, ev); + } + // End ISimulator2 + + bool is_ready() const { return owner->now() >= wakeup_ts; } + + void consume_unseen_events() { + bts_assert(is_ready()); + while (is_ready()) { + auto next_ev = owner->get_past_event(next_event_idx); + if (next_ev) { + next_event_idx++; + agent->handle(*next_ev); + } else + break; + } + } + + TimeStamp wakeup_time() const { + bts_assert(!is_ready()); + return wakeup_ts; + } + + Simulator2 *owner; + IAgent2 *agent; + TimeStamp wakeup_ts; + size_t next_event_idx; +}; + +struct Simulator2::PImpl { + PImpl(std::initializer_list const &ag, Simulator2 &owner) + : agents(), future(), past(), present() { + for (auto a : ag) + agents.emplace_back(a, owner); + } + std::vector agents; + FutureEventQueue future; + PastEventQueue past; + TimeStamp present; + size_t event_count = 0; +}; + +Simulator2::Simulator2(std::initializer_list agents) + : me(new PImpl{agents, *this}) {} + +Simulator2::~Simulator2() {} + +TimeStamp Simulator2::now() const { return me->present; } + +void Simulator2::advance_time(TimeStamp new_time) { + bts_assert(new_time >= me->present); + me->present = new_time; +} + +void Simulator2::emit(size_t delay_ns, EventPtr const &ev) { + me->event_count++; + ev->event_idx = me->event_count; + ev->observable_time = now() + delay_ns; + me->future.push(ev); +} + +EventPtr Simulator2::get_past_event(size_t eidx) { + if (eidx < me->past.size()) + return me->past[eidx]; + else + return {}; +} + +void Simulator2::run() { + // Prepare agents for simulation run + for (auto &a : me->agents) + a.init(); + + do { + // Move events that are now observable from the future to the + // past + while (me->future.size()) { + auto &next_event = me->future.top(); + if (next_event->observable_time <= now()) { + me->past.push_back(next_event); + me->future.pop(); + } else + break; + } + + // Give each agent that is not sleeping a chance to process any + // past events it has not seen yet + for (auto &a : me->agents) { + if (a.is_ready()) + a.consume_unseen_events(); + } + + // Move time to the next "interesting" point, which is either + // when a currently sleeping agent will wake up or when the + // earliest future event occurs. + auto next_interesting_time = EndOfTime; + if (me->future.size()) + next_interesting_time = me->future.top()->observable_time; + for (auto &a : me->agents) { + if (!a.is_ready()) + next_interesting_time = + std::min(next_interesting_time, a.wakeup_time()); + } + advance_time(next_interesting_time); + } while (now() != EndOfTime); + + // Simulation is complete + for (auto &a : me->agents) + a.finit(); +} + +} // namespace bts::spartan::sim diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator2.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator2.h new file mode 100644 index 000000000..b18a05910 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator2.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include "simulator_common.h" +#include "Event.h" + +namespace bts::spartan::sim { + +class IAgent2; + +class Simulator2 { +public: + Simulator2(std::initializer_list agents); + ~Simulator2(); + void run(); + +private: + TimeStamp now() const; + void advance_time(TimeStamp new_time); + void emit(size_t delay_ns,EventPtr const&); + EventPtr get_past_event(size_t eidx); + struct AgentRec; + struct PImpl; + std::unique_ptr me; +}; + +} + diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/events/SimEvents.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/events/SimEvents.h index aff585f45..12f1310b2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/events/SimEvents.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/events/SimEvents.h @@ -5,6 +5,7 @@ #include "../AlgoEvent.h" #include "../ExchangeTransaction.h" #include "../core/Event.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/PCapEvent.h" namespace bts::spartan::mdp3 { @@ -18,8 +19,11 @@ struct ExchangeTransactionEvent : sim::Event { ExchangeTransaction const *ev; }; +using PCapSimEvent = EventWrapper; using AlgoActionEvent = EventWrapper; using AlgoNotificationEvent = EventWrapper; struct TimerEvent : sim::Event {}; +struct ResetToPCapRequestEvent : sim::Event {}; +struct ResetToPCapReplyEvent : sim::Event {}; } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.cpp index a9bcf1387..6ea7b5480 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.cpp @@ -7,20 +7,194 @@ namespace bts::spartan::mdp3 { -double InstrumentDefinition::get_cash_per_point() const { - auto &cpp_map = config::get(); - auto asset_str = asset(); - if (asset_str[0] == '0') { - return 1.0; // test products - } else { - auto it = cpp_map.cpp_entry().find(asset_str); - if (it == cpp_map.cpp_entry().end()) { - LOG(Spartan,Error,"Cannot find CPP for {}. Returning 0", asset_str); - return 0; +struct InstrumentDefinition::PImpl { + + template PImpl(M m) { + symbol = str_from_fixed_array(m.symbol(), m.symbolEncodingLength()); + security_group = str_from_fixed_array(m.securityGroup(), + m.securityGroupEncodingLength()); + asset = str_from_fixed_array(m.asset(), m.assetEncodingLength()); + display_factor = md::IntPrice(m.displayFactor().mantissa(), + m.displayFactor().exponent()) + .price(); + + auto &cpp_map = config::get(); + if (asset[0] == '0') { + cash_per_point = 1.0; // test products + } else { + auto it = cpp_map.cpp_entry().find(asset); + if (it == cpp_map.cpp_entry().end()) { + LOG(Spartan, Error, "Cannot find CPP for {}. Returning 0", asset); + cash_per_point = 0.0; + } else + cash_per_point = it->second; } - else - return it->second; + + security_id.value() = m.securityID(); + match_algorithm = std::string(1, m.matchAlgorithm()); + min_price_increment = md::IntPrice(m.minPriceIncrement().mantissa(), + m.minPriceIncrement().exponent()); + + cme_security_type = str_from_fixed_array(m.securityType(), + m.securityTypeEncodingLength()); + + // As in the constructor we have to skip over all the arrays + // that preceed ours + m.noEvents().forEach([](auto const &) {}); + m.noMDFeedTypes().forEach([](auto const &) {}); + m.noInstAttrib().forEach([this](auto &attr) { + implied_matching = attr.instAttribValue().impliedMatchingEligibility(); + }); } + + PImpl() {} + + std::vector legs; + double display_factor; + double cash_per_point; + std::string symbol; + std::string security_group; + std::string asset; + MDP3TimeStamp last_update_time; + MDP3SecurityID security_id; + std::string match_algorithm; + md::IntPrice min_price_increment; + double min_price_increment_ammount = 0; + bool implied_matching = false; + api::SecurityType security_type; + std::string cme_security_type; +}; + +InstrumentDefinition::InstrumentDefinition() : me (new PImpl()) {} + +InstrumentDefinition::InstrumentDefinition( + ::mdp3::MDInstrumentDefinitionOption55 option) + : me(new PImpl(option)) { + me->security_type = api::SECURITYTYPE_OPTION; + me->min_price_increment_ammount = + md::IntPrice(option.minPriceIncrement().mantissa(), + option.minPriceIncrement().exponent()) + .price(); +} + +InstrumentDefinition::InstrumentDefinition( + ::mdp3::MDInstrumentDefinitionFuture54 future) + : me(new PImpl(future)) { + me->security_type = api::SECURITYTYPE_FUTURE; + me->min_price_increment_ammount = + md::IntPrice(future.minPriceIncrement().mantissa(), + future.minPriceIncrement().exponent()) + .price(); +} + +InstrumentDefinition::InstrumentDefinition( + ::mdp3::MDInstrumentDefinitionSpread56 spread) + : me(new PImpl(spread)) { + me->security_type = api::SECURITYTYPE_SPREAD; + // The generated code offers a streaming + // interface. Though you can randomly access + // the fixed size fields, the arrays have to + // be transversed in order, even if we do + // nothing to them. + spread.noEvents().forEach([](auto const &) {}); + spread.noMDFeedTypes().forEach([](auto const &) {}); + spread.noInstAttrib().forEach([](auto const &) {}); + spread.noLotTypeRules().forEach([](auto const &) {}); + spread.noLegs().forEach([this](auto const &l) { + LegDef ll{MDP3SecurityID{l.legSecurityID()}, + l.legSide() == ::mdp3::LegSide::BuySide ? api::SIDE_BUY + : api::SIDE_SELL, + MDP3Quantity{l.legRatioQty()}}; + me->legs.push_back(ll); + }); +} + +InstrumentDefinition::InstrumentDefinition(InstrumentDefinition &&d) + : me(std::move(d.me)) {} + +InstrumentDefinition::InstrumentDefinition(InstrumentDefinition const &d) + : me(new PImpl()) { + *me = *d.me; +} + +InstrumentDefinition & +InstrumentDefinition::operator=(InstrumentDefinition &&other) { + me = std::move(other.me); + return *this; +} + +InstrumentDefinition & +InstrumentDefinition::operator=(InstrumentDefinition const &other) { + *me = *other.me; + return *this; +} + +InstrumentDefinition::~InstrumentDefinition() {} + +MDP3SecurityID InstrumentDefinition::security_id() const { + return me->security_id; +} + +std::string const &InstrumentDefinition::symbol() const { return me->symbol; } +std::string const &InstrumentDefinition::security_group() const { + return me->security_group; +} +std::string const &InstrumentDefinition::asset() const { return me->asset; } + +std::string InstrumentDefinition::match_algorithm() const { + return me->match_algorithm; +} + +double InstrumentDefinition::display_factor() const { + return me->display_factor; +} + +double InstrumentDefinition::min_price_increment() const { + return me->min_price_increment.price(); +} + +double InstrumentDefinition::min_price_increment_ammount() const { + return me->min_price_increment_ammount; +} + +md::IntPrice InstrumentDefinition::min_price_increment_int_price() const { + return me->min_price_increment; +} + +bool InstrumentDefinition::implied_matching() const { + return me->implied_matching; +} + +api::SecurityType InstrumentDefinition::security_type() const { + return me->security_type; +} + +double InstrumentDefinition::to_cash(md::IntPrice const &p) const { + return p.price() * display_factor() * cash_per_point(); +} + +double InstrumentDefinition::to_cash(double p) const { + return p * display_factor() * cash_per_point(); +} + +double InstrumentDefinition::to_display(md::IntPrice const &p) const { + return p.price() * display_factor(); +} + +double InstrumentDefinition::to_display(double p) const { + return p * display_factor(); +} + +double InstrumentDefinition::cash_per_point() const { + return me->cash_per_point; +} + +std::string const &InstrumentDefinition::cme_security_type() const { + return me->cme_security_type; +} + +std::vector const &InstrumentDefinition::legs() const { + return me->legs; } } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h index 13bc5f193..a8c36ad0b 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h @@ -10,249 +10,49 @@ namespace bts::spartan::mdp3 { -// A simple abstraction for a instrument definition, which can be one of: -// ::mdp3::MDInstrumentDefinitionFuture54 -// ::mdp3::MDInstrumentDefinitionOption55 -// ::mdp3::MDInstrumentDefinitionSpread56 -// To get the actual definition type use the visit() method. - struct LegDef { MDP3SecurityID security_id; api::Side side; MDP3Quantity ratio; }; -inline std::ostream &operator<<(std::ostream &out, const LegDef &l) { - out << "sec: " << l.security_id << ", side: " << l.side - << ", ratio: " << l.ratio; - return out; -} - class InstrumentDefinition { public: - InstrumentDefinition(InstrumentDefinition &&d) = default; - InstrumentDefinition(InstrumentDefinition const &d) = default; - InstrumentDefinition &operator=(InstrumentDefinition &&other) = default; - InstrumentDefinition &operator=(InstrumentDefinition const &other) = default; - - template - explicit InstrumentDefinition(M const &d) : _def(MDP3MsgBox(d)), _legs() { - auto visitor = - overloaded{[](::mdp3::MDInstrumentDefinitionFuture54 const &) {}, - [](::mdp3::MDInstrumentDefinitionOption55 const &) {}, - [this](::mdp3::MDInstrumentDefinitionSpread56 spread) { - // The generated code offers a streaming - // interface. Though you can randomly access - // the fixed size fields, the arrays have to - // be transversed in order, even if we do - // nothing to them. - spread.noEvents().forEach([](auto const &) {}); - spread.noMDFeedTypes().forEach([](auto const &) {}); - spread.noInstAttrib().forEach([](auto const &) {}); - spread.noLotTypeRules().forEach([](auto const &) {}); - spread.noLegs().forEach([this](auto const &l) { - LegDef ll{MDP3SecurityID{l.legSecurityID()}, - l.legSide() == ::mdp3::LegSide::BuySide - ? api::SIDE_BUY - : api::SIDE_SELL, - MDP3Quantity{l.legRatioQty()}}; - _legs.push_back(ll); - }); - }}; - visit(visitor); - - _symbol = get([&](auto const &m) { - return str_from_fixed_array(m.symbol(), m.symbolEncodingLength()); - }); - - _security_group = get([&](auto const &m) { - return str_from_fixed_array(m.securityGroup(), - m.securityGroupEncodingLength()); - }); - - _asset = get([&](auto const &m) { - return str_from_fixed_array(m.asset(), m.assetEncodingLength()); - }); - - _display_factor = get([](auto m) { - return md::IntPrice(m.displayFactor().mantissa(), - m.displayFactor().exponent()) - .price(); - }); - _cash_per_point = get_cash_per_point(); - } - - // Visit specific type in this definition - template void visit(Visitor v) const { - std::visit([&v](auto &m) { v(*m); }, _def); - } - - // Query the specific type contained in this definition and return - // something about it - template auto get(Getter g) const { - return std::visit([g](auto &m) { return g(*m); }, _def); - } - - // Helpers - MDP3TimeStamp last_update_time() const { - return MDP3TimeStamp{ - get([](auto const &m) { return m.lastUpdateTime(); })}; - } - size_t tot_num_reports() const { - return get([](auto const &m) { return m.totNumReports(); }); - } - MDP3SecurityID security_id() const { - return MDP3SecurityID{get([](auto const &m) { return m.securityID(); })}; - } - - std::string const &symbol() const { return _symbol; } - - ::mdp3::MatchEventIndicator match_event_indicator() const { - return get([](auto m) { return m.matchEventIndicator(); }); - } - - std::string const &security_group() const { return _security_group; } - - std::string const &asset() const { return _asset; } - - std::string match_algorithm() const { - return get( - [](auto const &m) { return std::string(1, m.matchAlgorithm()); }); - } - - double display_factor() const { return _display_factor; } - - double min_price_increment() const { - return get([](auto m) { - return md::IntPrice(m.minPriceIncrement().mantissa(), - m.minPriceIncrement().exponent()) - .price(); - }); - } - - md::IntPrice min_price_increment_int_price() const { - return get([](auto m) { - return md::IntPrice(m.minPriceIncrement().mantissa(), - m.minPriceIncrement().exponent()); - }); - } + InstrumentDefinition(); + InstrumentDefinition(::mdp3::MDInstrumentDefinitionFuture54); + InstrumentDefinition(::mdp3::MDInstrumentDefinitionOption55); + InstrumentDefinition(::mdp3::MDInstrumentDefinitionSpread56); + InstrumentDefinition(InstrumentDefinition &&d); + InstrumentDefinition(InstrumentDefinition const &d); + ~InstrumentDefinition(); - double min_price_increment_ammount() const { - auto visitor = overloaded{ - [](::mdp3::MDInstrumentDefinitionFuture54 m) { - return md::IntPrice(m.minPriceIncrement().mantissa(), - m.minPriceIncrement().exponent()) - .price(); - }, - [](::mdp3::MDInstrumentDefinitionOption55 m) { - return md::IntPrice(m.minPriceIncrement().mantissa(), - m.minPriceIncrement().exponent()) - .price(); - }, - [](::mdp3::MDInstrumentDefinitionSpread56 const &) { return 0.0; }}; - return get(visitor); - } - - bool implied_matching() const { - return get([](auto m) { - // As in the constructor we have to skip over all the arrays - // that preceed ours - m.noEvents().forEach([](auto const &) {}); - m.noMDFeedTypes().forEach([](auto const &) {}); - bool imatch = false; - m.noInstAttrib().forEach([&imatch](auto &attr) { - imatch = attr.instAttribValue().impliedMatchingEligibility(); - }); - return imatch; - }); - } - - api::SecurityType security_type() const { - auto visitor = - overloaded{[](::mdp3::MDInstrumentDefinitionFuture54 const &) { - return api::SECURITYTYPE_FUTURE; - }, - [](::mdp3::MDInstrumentDefinitionOption55 const &) { - return api::SECURITYTYPE_OPTION; - }, - [](::mdp3::MDInstrumentDefinitionSpread56 const &) { - return api::SECURITYTYPE_SPREAD; - }}; - return get(visitor); - } - - double to_cash(md::IntPrice const &p) const { - return p.price() * display_factor() * cash_per_point(); - } - - double to_cash(double p) const { - return p * display_factor() * cash_per_point(); - } - double to_display(md::IntPrice const &p) const { - return p.price() * display_factor(); - } - double to_display(double p) const { return p * display_factor(); } - - double cash_per_point() const { return _cash_per_point; } - - const char *cme_security_type() const { - return get([](auto const &m) { return m.securityType(); }); - } - - std::vector const &legs() const { return _legs; } - - ::mdp3::SecurityTradingStatus::Value trading_status() const { - return get([](auto const &m) { return m.mDSecurityTradingStatus(); }); - } - - std::string_view data() const { - return std::visit([](auto &box) { return box.data(); }, _def); - } - - static InstrumentDefinition make_from_bytes(std::string_view const &data) { - bts_assert(data.size() > sizeof(MDP3MsgHeader)); - auto msg = (MDP3Msg const *)data.data(); - if (msg->header().template_id == 54) { - auto m = ::mdp3::MDInstrumentDefinitionFuture54( - (char *)msg->payload().data(), msg->payload().size()); - return InstrumentDefinition{m}; - } else if (msg->header().template_id == 55) { - auto m = ::mdp3::MDInstrumentDefinitionOption55( - (char *)msg->payload().data(), msg->payload().size()); - return InstrumentDefinition{m}; - } else if (msg->header().template_id == 56) { - auto m = ::mdp3::MDInstrumentDefinitionSpread56( - (char *)msg->payload().data(), msg->payload().size()); - return InstrumentDefinition{m}; - } else { - bts_assert(false); - // Compiler doesn't know we will never reach this - auto m = ::mdp3::MDInstrumentDefinitionSpread56((char *)nullptr, 0); - return InstrumentDefinition{m}; - } - } + InstrumentDefinition &operator=(InstrumentDefinition &&other) ; + InstrumentDefinition &operator=(InstrumentDefinition const &other) ; + MDP3SecurityID security_id() const; + std::string const &symbol() const ; + std::string const &security_group() const ; + std::string const &asset() const ; + std::string match_algorithm() const ; + double display_factor() const ; + double min_price_increment() const; + double min_price_increment_ammount() const; + md::IntPrice min_price_increment_int_price() const ; + bool implied_matching() const ; + api::SecurityType security_type() const ; + double to_cash(md::IntPrice const &p) const ; + double to_cash(double p) const ; + double to_display(md::IntPrice const &p) const ; + double to_display(double p) const ; + double cash_per_point() const ; + std::string const&cme_security_type() const ; + std::vector const &legs() const; + private: - using DefVar = - std::variant, - MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55>, - MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56>>; - mutable DefVar _def; - std::vector _legs; - double get_cash_per_point() const; - double _display_factor; - double _cash_per_point; - std::string _symbol; - std::string _security_group; - std::string _asset; + struct PImpl; + std::unique_ptr me; }; -inline std::ostream &operator<<(std::ostream &out, - InstrumentDefinition const &d) { - d.visit([&out](auto const &m) { out << m; }); - return out; -} - using MaybeInstrumentDefinition = std::optional; using InstrumentDefinitions = std::unordered_map; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp index f2b57004b..ff24b0c5b 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp @@ -4,28 +4,19 @@ namespace bts::spartan::mdp3 { struct InstrumentSnapshot::PImpl { + PImpl() {} PImpl(InstrumentDefinition const &def) : security_id(def.security_id()), books(), implied_books(), orders(), order_changes(), last_processed_sn(0), fills(), high_limit_price(), low_limit_price(), max_price_variation(), cleared_volume(), electronic_volume(), fixing_price(), open_interest(), open_price(), high_bid(), low_offer(), settlement_price(), high_trade_price(), - low_trade_price(), definition(def), - trading_status(definition.trading_status()), - implied_matching_status(false) {} - - PImpl(InstrumentDefinition &&def) - : security_id(def.security_id()), books(), implied_books(), orders(), - order_changes(), last_processed_sn(0), fills(), high_limit_price(), - low_limit_price(), max_price_variation(), cleared_volume(), - electronic_volume(), fixing_price(), open_interest(), open_price(), - high_bid(), low_offer(), settlement_price(), high_trade_price(), - low_trade_price(), definition(std::move(def)), - trading_status(definition.trading_status()), + low_trade_price(), last_update_time(), definition(def), + trading_status(::mdp3::SecurityTradingStatus::UnknownorInvalid), implied_matching_status(false) {} // Books - const MDP3SecurityID security_id; + MDP3SecurityID security_id; std::array books; std::array implied_books; // Orders @@ -52,12 +43,21 @@ struct InstrumentSnapshot::PImpl { md::IntPrice settlement_price; md::IntPrice high_trade_price; md::IntPrice low_trade_price; + MDP3TimeStamp last_update_time; InstrumentDefinition definition; ::mdp3::SecurityTradingStatus::Value trading_status; bool implied_matching_status; }; +InstrumentSnapshot::InstrumentSnapshot() : me (new PImpl()) {} + +InstrumentSnapshot::InstrumentSnapshot(InstrumentSnapshot const&src) : me (new PImpl()) { + operator=(src); +} +InstrumentSnapshot::InstrumentSnapshot(InstrumentSnapshot &&src) { + operator=(std::move(src)); +} InstrumentSnapshot::InstrumentSnapshot(InstrumentDefinition const &def) : me(new PImpl(def)) {} @@ -183,6 +183,72 @@ void InstrumentSnapshot::set_low_trade_price(md::IntPrice v) { me->low_trade_price = v; } +MDP3TimeStamp InstrumentSnapshot::last_update_time() const { + return me->last_update_time; +} + +bool InstrumentSnapshot::ready_to_trade() const { + return trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade; +} + +void InstrumentSnapshot::set_last_update_time(MDP3TimeStamp ts) { + me->last_update_time = ts; +} + +void InstrumentSnapshot::light_copy(InstrumentSnapshot const&src) { + me->security_id = src.me->security_id; + me->books = src.me->books; + me->implied_books = src.me->implied_books; + me->order_changes = src.me->order_changes; + me->last_processed_sn = src.me->last_processed_sn; + me->fills = src.me->fills; + me->high_limit_price = src.me->high_limit_price; + me->low_limit_price = src.me->low_limit_price; + me->max_price_variation = src.me->max_price_variation; + me->cleared_volume = src.me->cleared_volume; + me->electronic_volume = src.me->electronic_volume; + me->fixing_price = src.me->fixing_price; + me->open_interest = src.me->open_interest; + me->open_price = src.me->open_price; + me->high_bid = src.me->high_bid; + me->low_offer = src.me->low_offer; + me->settlement_price = src.me->settlement_price; + me->high_trade_price = src.me->high_trade_price; + me->low_trade_price = src.me->low_trade_price; + me->last_update_time = src.me->last_update_time; + me->definition = src.me->definition; + me->trading_status = src.me->trading_status; + me->implied_matching_status = src.me->implied_matching_status; +} + +void InstrumentSnapshot::update_with(InstrumentSnapshot const&src) { + light_copy(src); + me->order_changes.clear(); + for (auto &ch : src.order_changes()) + switch (ch.change) { + case OrderChange::ChangeType::added: + insert_order(ch.order); + break; + case OrderChange::ChangeType::updated: + update_order(ch.order); + break; + case OrderChange::ChangeType::removed: + remove_order(ch.order.key); + break; + } +} + +InstrumentSnapshot& InstrumentSnapshot::operator=(InstrumentSnapshot const&src) { + light_copy(src); + me->orders = src.me->orders; + return *this; +} + +InstrumentSnapshot& InstrumentSnapshot::operator=(InstrumentSnapshot &&src) { + me = std::move(src.me); + return *this; +} + Book const &InstrumentSnapshot::book(api::Side s) const { return me->books[s - api::SIDE_BUY]; } @@ -193,7 +259,7 @@ ImpliedBook const &InstrumentSnapshot::implied_book(api::Side s) const { Book InstrumentSnapshot::merged_book(api::Side s) const { Book result; - merge_books(s,book(s), implied_book(s),&result); + merge_books(s, book(s), implied_book(s), &result); return result; } @@ -364,12 +430,14 @@ OrderChanges InstrumentSnapshot::deduced_order_changes() const { // generated by the matcher. OrderChanges rv; // removes - std::copy_if( - me->order_changes.begin(), me->order_changes.end(), std::back_inserter(rv), - [](const auto& ch) { return ch.change == OrderChange::ChangeType::removed; }); - std::copy_if( - me->order_changes.begin(), me->order_changes.end(), std::back_inserter(rv), - [](const auto& ch) { return ch.change != OrderChange::ChangeType::removed; }); + std::copy_if(me->order_changes.begin(), me->order_changes.end(), + std::back_inserter(rv), [](const auto &ch) { + return ch.change == OrderChange::ChangeType::removed; + }); + std::copy_if(me->order_changes.begin(), me->order_changes.end(), + std::back_inserter(rv), [](const auto &ch) { + return ch.change != OrderChange::ChangeType::removed; + }); return rv; } else { struct AggOrderRec { @@ -405,9 +473,8 @@ OrderChanges InstrumentSnapshot::deduced_order_changes() const { // orders. for (auto &agg_order : agg_orders) { for (auto const &f : fills()) - if (agg_order.ok == f.key && - f.aggressor != MDP3Fill::THIS_BUY && - f.aggressor != MDP3Fill::THIS_SELL ) + if (agg_order.ok == f.key && f.aggressor != MDP3Fill::THIS_BUY && + f.aggressor != MDP3Fill::THIS_SELL) agg_order.fills.push_back(f); } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h index 9798e4964..112dc54bf 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h @@ -25,8 +25,15 @@ namespace bts::spartan::mdp3 { class InstrumentSnapshot { public: + InstrumentSnapshot(); + InstrumentSnapshot(InstrumentSnapshot const&); + InstrumentSnapshot(InstrumentSnapshot &&); InstrumentSnapshot(InstrumentDefinition const &); InstrumentSnapshot(InstrumentDefinition &&); + InstrumentSnapshot& operator=(InstrumentSnapshot const&src); + InstrumentSnapshot& operator=(InstrumentSnapshot &&src); + void light_copy(InstrumentSnapshot const&src); // skips orders + void update_with(InstrumentSnapshot const&src); // use order_changes to update orders ~InstrumentSnapshot(); // Queries ------------------------------------------------------------ @@ -56,7 +63,9 @@ class InstrumentSnapshot { md::IntPrice settlement_price() const; md::IntPrice high_trade_price() const; md::IntPrice low_trade_price() const; - + MDP3TimeStamp last_update_time() const; + bool ready_to_trade() const; + // Mutators ------------------------------------------------------------ void set_definition(InstrumentDefinition const &); void set_last_msg_processed_sn(size_t); @@ -92,6 +101,7 @@ class InstrumentSnapshot { void set_settlement_price(md::IntPrice); void set_high_trade_price(md::IntPrice); void set_low_trade_price(md::IntPrice); + void set_last_update_time(MDP3TimeStamp); private: template @@ -103,6 +113,7 @@ class InstrumentSnapshot { std::unique_ptr me; }; +using SnapshotMap = std::unordered_map ; std::ostream &operator<<(std::ostream &, InstrumentSnapshot const &); } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h index 4626206ff..80d357578 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h @@ -97,6 +97,15 @@ struct IcebergQuantity { MDP3Quantity total; }; +struct DeducedIceberg { + MDP3OrderKey key; + IcebergQuantity quantity; + bool filled; +}; + +using IcebergMap = + std::unordered_map ; + inline std::ostream &operator<<(std::ostream &out, const IcebergQuantity &ibq) { out << ibq.show << "/" << ibq.total; return out; @@ -165,7 +174,7 @@ inline std::ostream &operator<<(std::ostream &out, return out; } -using OrderChanges = std::vector; +using OrderChanges = boost::container::small_vector; inline std::ostream &operator<<(std::ostream &s, MDP3Order const &order) { s << " k: " << order.key.value() << " sk: " << order.security.value() @@ -275,7 +284,7 @@ inline std::ostream &operator<<(std::ostream &s, MDP3Fill const &fill) { return s; } -using Fills = std::vector; +using Fills = boost::container::small_vector; inline std::ostream &operator<<(std::ostream &s, Fills const &fills) { for (auto f : fills) s << " " << f << "\n"; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp index 8c5b55fd5..569273bb2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp @@ -65,20 +65,21 @@ struct RealTimeHandler::EventGuard { } } - EventGuard(RealTimeHandler &owner, InstrumentDefinition &def) + EventGuard(RealTimeHandler &owner, MDP3TimeStamp last_update_time, + bool end_of_event) : owner(owner) { bts_assert(owner.me->current_packet_sn); auto &info = owner.me->current_event; if (info.transact_time == MDP3TimeStamp{0}) { // Starting a new event - info.transact_time = MDP3TimeStamp{def.last_update_time()}; + info.transact_time = MDP3TimeStamp{last_update_time}; info.first_packet_send_time = MDP3TimeStamp{owner.me->current_packet_send_time}; info.first_packet_sn = owner.me->current_packet_sn; info.last_packet_send_time = MDP3TimeStamp{0}; info.last_packet_sn = 0; } - if (def.match_event_indicator().endOfEvent()) { + if (end_of_event) { // last event in bts_assert(info.transact_time != MDP3TimeStamp{0}); info.last_packet_send_time = @@ -465,9 +466,10 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshVolume37 &m) { }); } -template void RealTimeHandler::on_instrument_definition(Def &d) { +template void RealTimeHandler::on_instrument_definition(DefM &d) { InstrumentDefinition def{d}; - EventGuard ev(*this, def); + EventGuard ev(*this, MDP3TimeStamp{d.lastUpdateTime()}, + d.matchEventIndicator().endOfEvent()); auto sid = MDP3SecurityID{d.securityID()}; if (auto iptr = find_instrument(sid); !iptr) { @@ -542,7 +544,8 @@ void RealTimeHandler::operator()(::mdp3::SecurityStatus30 &ss) { for (auto &p : me->instruments) { auto &iptr = p.second; auto def = iptr->definition(); - auto sgroup = str_from_fixed_array(ss.securityGroup(),ss.securityGroupEncodingLength()); + auto sgroup = str_from_fixed_array(ss.securityGroup(), + ss.securityGroupEncodingLength()); if (def.security_group() == sgroup) { ev.record_security_change(iptr.get()); update_trading_status(*iptr); @@ -557,7 +560,8 @@ void RealTimeHandler::operator()(::mdp3::SecurityStatus30 &ss) { for (auto &p : me->instruments) { auto &iptr = p.second; auto def = iptr->definition(); - auto sasset = str_from_fixed_array(ss.asset(), ss.assetEncodingLength()) ; + auto sasset = + str_from_fixed_array(ss.asset(), ss.assetEncodingLength()); if (def.asset() == sasset) { ev.record_security_change(iptr.get()); update_trading_status(*iptr); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp index 9c54f1893..e7d97226a 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp @@ -51,9 +51,8 @@ template void DefinitionSnapshot::on_definition(D &d) { break; case State::ACCUMULATING_DEFS: me->definitions.emplace_back(d); - if (auto nr = me->definitions[0].tot_num_reports(); - nr == me->definitions.size()) { - bts_assert(nr); + if (d.totNumReports() == me->definitions.size()) { + bts_assert(d.totNumReports()); transition(State::COMPLETE); } break; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp index d7e2da3a9..298c87822 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp @@ -168,7 +168,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { LOG(Spartan, Diag, "Added snapshot with last update time {} for security {} with " "last processed sn {}", - this_snapshot.snapshot->definition().last_update_time().value(), + this_snapshot.snapshot->last_update_time().value(), this_snapshot.snapshot->security_id().value(), this_snapshot.last_processed_sn); rt_handler.add_snapshot(std::move(this_snapshot.snapshot)); @@ -190,7 +190,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { LOG(Spartan, Diag, "Added snapshot with last update time {} for security {} with " "last processed sn {}", - this_snapshot.snapshot->definition().last_update_time().value(), + this_snapshot.snapshot->last_update_time().value(), this_snapshot.snapshot->security_id().value(), this_snapshot.last_processed_sn); rt_handler.add_snapshot(std::move(this_snapshot.snapshot)); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp index e7e99d962..d7202bde6 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp @@ -411,15 +411,21 @@ struct Ctrl : mdp3::IFeedController { void stop_incremental_feed() override{}; }; -void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) { - ObsWrapper obs_wrapper(obs); +struct PCapStreamer::PImpl { + PImpl(std::string const &file, mdp3::IChannelObserver &obs) + : obs_wrapper(obs), ctrl(), reader(file), handler(obs_wrapper, ctrl) {} + + ObsWrapper obs_wrapper; Ctrl ctrl; - + PCapReader reader; + mdp3::ChannelHandler handler; std::array buffers; - Guesser guesser; +}; - auto reader = PCapReader{file}; - reader.on_packet() = [&](auto const &info, auto const &payload) { +PCapStreamer::PCapStreamer(std::string const &file, mdp3::IChannelObserver &obs) + : me(new PImpl(file, obs)) { + Guesser guesser; + me->reader.on_packet() = [&](auto const &info, auto const &payload) { guesser.set_current_source(info); parse_packet(MDP3PacketView{payload}, guesser); if (guesser.result) @@ -428,7 +434,7 @@ void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) { return true; }; - while (reader.read_next()) + while (me->reader.read_next()) ; auto feed_map = guesser.result; @@ -437,30 +443,38 @@ void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) { file); bts_assert(false); } - mdp3::ChannelHandler handler(obs_wrapper, ctrl); - obs_wrapper.init(); - reader.on_packet() = [&](auto const &info, auto const &payload) { + me->obs_wrapper.init(); + me->reader.on_packet() = [this, feed_map](auto const &info, + auto const &payload) { if ((info.dst_ip == feed_map.incremental[0].ip && info.dst_port == feed_map.incremental[0].port)) { - buffers[0].i.set_bytes(payload); - handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view()); + me->buffers[0].i.set_bytes(payload); + me->handler.on_incremental_feed(me->buffers[0].i.view(), + me->buffers[1].i.view()); } else if ((info.dst_ip == feed_map.incremental[1].ip && info.dst_port == feed_map.incremental[1].port)) { - buffers[1].i.set_bytes(payload); - handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view()); + me->buffers[1].i.set_bytes(payload); + me->handler.on_incremental_feed(me->buffers[0].i.view(), + me->buffers[1].i.view()); } else if ((info.dst_ip == feed_map.instrument.ip && info.dst_port == feed_map.instrument.port)) { - handler.on_instrument_feed(mdp3::MDP3PacketView{payload}); + me->handler.on_instrument_feed(mdp3::MDP3PacketView{payload}); } else if ((info.dst_ip == feed_map.snapshot.ip && info.dst_port == feed_map.snapshot.port)) { - handler.on_snapshot_feed(mdp3::MDP3PacketView{payload}); + me->handler.on_snapshot_feed(mdp3::MDP3PacketView{payload}); } return true; }; +} - while (reader.read_next()) +PCapStreamer::~PCapStreamer() { me->obs_wrapper.finit(); } + +bool PCapStreamer::next() { return me->reader.read_next(); } + +void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) { + PCapStreamer streamer(file, obs); + while (streamer.next()) ; - obs_wrapper.finit(); } void observe_channel(std::string const &pcap, @@ -532,7 +546,7 @@ struct CSVObs : IChannelObserver { << s->definition().min_price_increment() << "," << s->definition().min_price_increment_ammount() << "," << s->definition().legs().size() << "," - << s->definition().last_update_time() << std::endl; + << s->last_update_time() << std::endl; for (auto const &l : s->definition().legs()) spread_legs << s->security_id().value() << "," @@ -606,7 +620,7 @@ struct CSVObs : IChannelObserver { << s->definition().min_price_increment() << "," << s->definition().min_price_increment_ammount() << "," << s->definition().legs().size() << "," - << s->definition().last_update_time() // last update time + << s->last_update_time() // last update time << std::endl; } @@ -703,7 +717,8 @@ int to_csv(std::string const &file) { observe_channel(file, {&cvs, &ice_finder, &def_collector, &poly_finder}); auto icebergs = std::ofstream(drop_extensions(file) + ".icebergs.csv"); for (auto &i : ice_finder.result) - icebergs << i.first.value() << "," << i.second.estimated_quantity.value() + icebergs << i.first.value() << "," << i.second.quantity.total.value() + << "," << i.second.quantity.show.value() << "," << i.second.filled << std::endl; auto polys = std::ofstream(drop_extensions(file) + ".polygons.csv"); @@ -846,13 +861,13 @@ find_icebergs(std::string const &pcap) { if (od.is_iceberg) { auto &di = result[k]; di.key = k; - di.estimated_quantity.value() = od.accumulated_fills; + di.quantity.total.value() = od.accumulated_fills; di.filled = od.filled; // If we have not seen an iceberg order fill completely then // all we can say is that its hidden size is at least one // contract bigger than the observed fills. if (!od.filled) - di.estimated_quantity.value() += 1; + di.quantity.total.value() += 1; } return result; } @@ -883,7 +898,8 @@ int to_sql(std::string const &pcap) { std::string import = ".mode csv\n"; for (auto const &t : tables) - import += fmt::format(".import {0}.{1}.csv {1}\n", drop_extensions(pcap), t,t); + import += fmt::format(".import {0}.{1}.csv {1}\n", + drop_extensions(pcap), t, t); utils::TempFile f; std::ofstream{f.path()} << import; @@ -896,8 +912,9 @@ int to_sql(std::string const &pcap) { std::cerr << "Error importing CSVs" << std::endl; return result; } else - for (auto const &t: tables) - boost::filesystem::remove(fmt::format("{0}.{1}.csv",drop_extensions(pcap),t)); + for (auto const &t : tables) + boost::filesystem::remove( + fmt::format("{0}.{1}.csv", drop_extensions(pcap), t)); return 0; } else diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h index ae9f45184..1e9fd49a9 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h @@ -64,6 +64,21 @@ int to_csv(std::string const &pcap); // generated by to_csv int to_sql(std::string const &pcap); +// Stream through a pcap file, invoking the passed observer on each +// call to next() +class PCapStreamer { +public: + PCapStreamer(std::string const &file, mdp3::IChannelObserver &obs); + ~PCapStreamer(); + // Returns false when done + bool next() ; + +private: + struct PImpl; + std::unique_ptr me; +}; + +// Stream through an entire pcap. void observe_channel(std::string const &pcap, IChannelObserver &obs); void observe_channel(std::string const &pcap, std::initializer_list observers); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp index 6eceb26a0..a15bbfdc2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp @@ -37,14 +37,14 @@ void IcebergFinder::finit() { if (od.is_iceberg) { auto &di = result[k]; di.key = k; - di.estimated_quantity.value() = od.accumulated_fills + od.accumulated_qty_changes + od.last_observed_qty; - di.show_quantity = MDP3Quantity(od.show_qty); + di.quantity.total.value() = od.accumulated_fills + od.accumulated_qty_changes + od.last_observed_qty; + di.quantity.show = MDP3Quantity(od.show_qty); di.filled = od.filled; // If we have not seen an iceberg order fill completely then // all we can say is that its hidden size is at least one // contract bigger than the observed fills. if (!od.filled && od.last_observed_qty == 0) - di.estimated_quantity.value() += 1; + di.quantity.total.value() += 1; } } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h index 0692d60c4..7d64e3d3c 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h @@ -5,17 +5,8 @@ namespace bts::spartan::mdp3 { -// Find all the icebegers in the pcap, estimate their size. - -struct DeducedIceberg { - MDP3OrderKey key; - MDP3Quantity show_quantity; - MDP3Quantity estimated_quantity; - bool filled; -}; - struct IcebergFinder : IChannelObserver { - std::unordered_map result; + IcebergMap result; IcebergFinder(); ~IcebergFinder(); @@ -29,10 +20,4 @@ struct IcebergFinder : IChannelObserver { std::unique_ptr me; }; -inline std::ostream& operator <<(std::ostream& out, const DeducedIceberg& ib) { - out << "key " << ib.key << ", show_qty " << ib.show_quantity << ", est_qty " - << ib.estimated_quantity << ", filled " << (ib.filled ? "Y" : "N"); - return out; -} - } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/TransactionDeducer.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/TransactionDeducer.cpp index f68489c84..1d3bd0501 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/TransactionDeducer.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/TransactionDeducer.cpp @@ -91,10 +91,7 @@ void TransactionDeducer::finit() { o_index[o.order.key].push_back(&o); for (auto &[ok, ib] : me->ice_finder.result) { for (auto &o : o_index[ok]) { - IcebergQuantity ibq; - ibq.show = o->order.quantity; - ibq.total = ib.estimated_quantity; - o->order.iceberg_quantity = ibq; + o->order.iceberg_quantity = ib.quantity; } } diff --git a/src/server/sparkd/main.cpp b/src/server/sparkd/main.cpp index a85ed1421..2fa898a06 100644 --- a/src/server/sparkd/main.cpp +++ b/src/server/sparkd/main.cpp @@ -17,9 +17,9 @@ #include "server/sparkd/cmds/QuickTest.h" #include "server/sparkd/cmds/Run.h" #include "server/sparkd/cmds/Span.h" +#include "server/sparkd/cmds/Spartan.h" #include "server/sparkd/cmds/Test.h" #include "server/sparkd/cmds/Web.h" -#include "server/sparkd/cmds/Spartan.h" #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Bridge/BeforeLaunchBridge.h" using namespace bts;