diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index d64cf4a5d..9fee855df 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimEventSource.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp;cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/Sim.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimEventSource.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index a679b2815..83c5bd6c9 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -30,7 +30,8 @@ std::map const &)>> {"validate_fifo", [](auto const &args) { struct BackTesterHarness : IChannelObserver { - std::unordered_map orders; + std::unordered_map previous_orders; + std::unordered_map current_orders; std::unordered_map icebergs; std::unordered_set ready_securities; size_t passed_tests = 0; @@ -42,7 +43,9 @@ std::map const &)>> for (auto s : snap) { auto annotated_orders = s->orders(); add_iceberg_annotation(annotated_orders); - orders.emplace(s->security_id(), annotated_orders); + previous_orders.emplace(s->security_id(), + annotated_orders); + current_orders.emplace(s->security_id(), annotated_orders); if (s->trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade) ready_securities.insert(s->security_id()); @@ -50,14 +53,42 @@ std::map const &)>> } void on_real_time_event(MDP3Event const &ev) override { - if (ev.event_id < 40000) - return; - + + // Testing loop. We may skip the work for some events for (auto s : ev.affected_securities) { + auto it = previous_orders.find(s->security_id()); + if (it == previous_orders.end()) { + bts_assert(s->orders().empty()); + auto with_iceberg = s->orders(); + add_iceberg_annotation(with_iceberg); + it = previous_orders + .emplace(s->security_id(), with_iceberg) + .first; + } + + auto &new_orders = current_orders[s->security_id()]; + for (auto ch : s->order_changes()) { + switch (ch.change) { + case OrderChange::added: + case OrderChange::updated: + add_iceberg_annotation(ch.order); + new_orders[ch.order.key] = ch.order; + break; + case OrderChange::removed: + new_orders.erase(ch.order.key); + break; + } + } + + auto &old_orders = it->second; + + if (ev.event_id < 3342000) + continue; + // We do not want to process the first event at - // the open since the matching at the open it - // works differently. + // the open since matching at the open works + // differently. if (ready_securities.count(s->security_id()) == 0) { if (s->trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade) { @@ -66,13 +97,6 @@ std::map const &)>> continue; } - auto it = orders.find(s->security_id()); - if (it == orders.end()) { - bts_assert(s->orders().empty()); - it = - orders.emplace(s->security_id(), s->orders()).first; - } - auto &old_orders = it->second; auto deduced = s->deduced_order_changes(); // We have a hard time simulating icebergs, @@ -87,34 +111,35 @@ std::map const &)>> continue; if (deduced.size()) { - // We have deduced orders, lets see what the matcher - // says - add_iceberg_annotation(deduced); - - std::cout << "Event id: " << ev.event_id << std::endl; FifoSecurityMatcher matcher(ev.event_id, old_orders); - auto current_orders = s->orders(); - add_iceberg_annotation(current_orders); for (auto &o : deduced) matcher.on_order_change(o); - auto error = matcher.has_same_orders(current_orders); + auto error = matcher.has_same_orders(new_orders); if (error) { std::cout << "Failing on event:\n " << ev.event_id << std::endl - << "Error: " << error.value() << std::endl; - // << "Deduced orders:\n" - // << deduced << std::endl - // << "Old Orders:\n" - // << old_orders << std::endl - // << "Order Changes:\n" - // << s->order_changes() << std::endl - // << "Fills:\n" - // << s->fills() << std::endl; + << "Error: " << error.value() << std::endl + << "Old Orders:\n" + << old_orders << std::endl; + } + } + } // For each security + + // Update our order cache so we can use it to + // populate the matcher on the next event + for (auto s : ev.affected_securities) { + auto &old_orders = previous_orders[s->security_id()]; + for (auto ch : s->order_changes()) { + switch (ch.change) { + case OrderChange::added: + case OrderChange::updated: + add_iceberg_annotation(ch.order); + old_orders[ch.order.key] = ch.order; + break; + case OrderChange::removed: + old_orders.erase(ch.order.key); + break; } - old_orders = current_orders; - } else { - it->second = s->orders(); - add_iceberg_annotation(it->second); } } } @@ -143,47 +168,6 @@ std::map const &)>> return 0; }}, - {"deduce_orders", - [](auto const &args) { - struct OrderGen : IChannelObserver { - std::ofstream out; - OrderGen(std::string const &out_path) : out(out_path) {} - - void on_recovery_complete(InitialSnapshot const &) override {} - void on_real_time_event(MDP3Event const &ev) override { - for (auto s : ev.affected_securities) { - auto deduced = s->deduced_order_changes(); - if (deduced.size()) { - out << "sym: " << s->definition().symbol() - << " eid: " << ev.event_id - << " sn: " << ev.first_packet_sn << " to " - << ev.last_packet_sn - << " ts: " << to_time_string(ev.transact_time) - << std::endl - << "Order changes: " << std::endl - << s->order_changes() << "Fills: " << std::endl - << s->fills() << "Deduced changes: " << std::endl - << s->deduced_order_changes() << std::endl - << std::endl; - } - } - } - void on_real_time_channel_loss() override { bts_assert(false); } - } gen(args[2]); - - observe_channel(args[1], gen); - return 0; - }}, - - {"find_icebergs", - [](auto const &args) { - std::ofstream out(args[2]); - for (auto const &[k, ice] : find_icebergs(args[1])) - out << "ok: " << k << " sz: " << ice.estimated_quantity.value() - << std::endl; - return 0; - }}, - {"gen_sim_input", [](auto const &args) { auto input = make_sim_input(args[1]); @@ -227,6 +211,5 @@ std::string Spartan::help() { str += fmt::format("\t{} [.xz] ...\n", p.first); return str; } - } // namespace cmd } // namespace bts diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoAction.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoAction.h index 75ac8750e..3a4751695 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoAction.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoAction.h @@ -4,25 +4,12 @@ namespace bts::spartan::mdp3 { -struct CreateOrder { - MDP3OrderKey ok; - MDP3SecurityID security; - api::Side side; - MDP3OrderPriority priority; - md::IntPrice price; - MDP3Quantity quantity; -}; +class IAlgoModel; -struct UpdateOrder { - MDP3OrderKey ok; - md::IntPrice price; - MDP3Quantity quantity; +struct AlgoAction { + MDP3TimeStamp transact_time; + IAlgoModel *algo; + OrderChange change; }; -struct CancelOrder { - MDP3OrderKey ok; -}; - -using AlgoAction = std::variant; - } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.cpp new file mode 100644 index 000000000..11ff3600e --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.cpp @@ -0,0 +1,5 @@ +#include "ISimServices.h" + +namespace bts::spartan::mdp3 { +ISimServices::~ISimServices() {} +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.h similarity index 87% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.h index 0f19be3b2..cb1282643 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.h @@ -6,9 +6,9 @@ namespace bts::spartan::mdp3 { class InstrumentDefinition; -class ISimulatorServices { +class ISimServices { public: - virtual ~ISimulatorServices(); + virtual ~ISimServices(); virtual std::vector const &instruments() const = 0; virtual MDP3TimeStamp time() const = 0; virtual MDP3OrderKey generate_order_key() = 0; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp deleted file mode 100644 index cf742d9a6..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp +++ /dev/null @@ -1,8 +0,0 @@ -#include "ISimulatorServices.h" - - -namespace bts::spartan::mdp3 { - -ISimulatorServices::~ISimulatorServices() {} - -} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/RecordedInputEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/RecordedInputEvent.h index 38488c565..eab5d3752 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/RecordedInputEvent.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/RecordedInputEvent.h @@ -1,3 +1,5 @@ +#pragma once + #include "../../common/mdp3_common.h" namespace bts::spartan::mdp3 { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Sim.cpp similarity index 72% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Sim.cpp index 79968488a..ecc1e4cd0 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Sim.cpp @@ -1,62 +1,36 @@ -// #include "Simulator.h" +// #include "Sim.h" // #include "../matcher/Matcher.h" // #include "AlgoEvent.h" // #include "IAlgoModel.h" // #include "ISimulatorServices.h" -// #include "SimulatorInput.h" // #include "lib/utils/overloaded.h" // #include // namespace bts::spartan::mdp3 { -// struct Simulator::AlgoActionRec { -// MDP3TimeStamp transact_time; -// AlgoRecord *algo_rec; -// AlgoAction action; -// bool operator<(AlgoActionRec const &rhs) const { -// return rhs.transact_time.value() < transact_time.value(); -// } -// }; - -// struct Simulator::AlgoEventRec { -// MDP3TimeStamp send_time; -// AlgoEvent event; -// bool operator<(AlgoEventRec const &rhs) const { -// return rhs.send_time.value() < send_time.value(); -// } -// }; - // struct Simulator::PImpl { // PImpl(SimulatorRecordedInput const &i) : input(i) {} // SimulatorRecordedInput const &input; // void reset() { -// next_input_event = 0; // last_order_key = input.max_order_id; // algos.clear(); // matcher = Matcher{input.instruments}; -// pending_algo_actions = {}; -// order_to_algo = {}; +// order_owners = {}; // } -// size_t next_input_event; // size_t last_order_key; // std::vector> algos; // Matcher matcher; -// std::priority_queue pending_algo_actions; -// std::unordered_map order_to_algo; +// std::unordered_map order_owners; // }; -// struct Simulator::AlgoRecord : ISimulatorServices { -// Simulator &owner; +// struct Sim::AlgoRecord : ISimServices { +// Sim &owner; // IAlgoModel &algo; -// MDP3TimeStamp current_time; -// RunOptions options; // uint64_t pandl; -// std::priority_queue pending_events; - -// AlgoRecord(Simulator &owner, IAlgoModel &algo, const RunOptions &options) -// : owner(owner), algo(algo), current_time(0), options(options), pandl(0) { +// AlgoRecord(Sim &owner, IAlgoModel &algo, const RunOptions &options) +// : owner(owner), algo(algo), pandl(0) { // algo.init(*this); // } @@ -90,15 +64,16 @@ // } // }; -// Simulator::Simulator(SimulatorRecordedInput const &input) : me(new PImpl{input}) {} +// Sim::Sim(SimRecordedInput const &input) +// : me(new PImpl{input}) {} -// Simulator::~Simulator() {} +// Sim::~Sim() {} -// int64_t Simulator::run(IAlgoModel &algo, const RunOptions &options) { +// int64_t Sim::run(IAlgoModel &algo, const RunOptions &options) { // return run({&algo}, options)[0]; // } -// std::vector Simulator::run(std::vector const &algos, +// std::vector Sim::run(std::vector const &algos, // const RunOptions &options) { // me->reset(); @@ -113,7 +88,7 @@ // return pandl; // } -// bool Simulator::has_work() const { +// bool Sim::has_work() const { // if (me->next_input_event != me->input.events.size() || // me->pending_algo_actions.size()) // return true; @@ -125,7 +100,7 @@ // } // } -// void Simulator::do_work() { +// void Sim::do_work() { // if (me->next_input_event != me->input.events.size() && // me->pending_algo_actions.size()) { @@ -159,10 +134,9 @@ // algo->deliver_events(); // } -// void Simulator::on_algo_action(AlgoActionRec const&) { -// } +// void Sim::on_algo_action(AlgoActionRec const &) {} -// void Simulator::on_simulator_input_event(AlgoInputEvent const &) { +// void Sim::on_simulator_input_event(AlgoInputEvent const &) { // // std::vector effects; // // me->matcher.match(Matcher::SIMULATOR_INPUT, ev.order_changes, &effects); @@ -178,10 +152,8 @@ // // mbp_update); // // }, - // // [&](OrderUpdate const &) {}}, // // e); // // } // } // } // namespace bts::spartan::mdp3 - diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Sim.h similarity index 53% rename from src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h rename to src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Sim.h index e260794aa..1ca4d3dae 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Simulator.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/Sim.h @@ -1,8 +1,8 @@ #pragma once +#include "../../common/mdp3_common.h" #include #include -#include "../../common/mdp3_common.h" namespace bts::spartan::mdp3 { @@ -10,17 +10,16 @@ struct SimulatorInput; class IAlgoModel; struct SimulatorInputEvent; -class Simulator { +class Sim { public: - Simulator(SimulatorInput const &); - ~Simulator(); + Sim(SimulatorInput const &); + ~Sim(); struct RunOptions { - RunOptions() : default_algo_latency_ns(0), - ilink_delta_ns(0), - estimated_exchange_latency_ns(0) - {} - + RunOptions() + : default_algo_latency_ns(0), ilink_delta_ns(0), + estimated_exchange_latency_ns(0) {} + // The default latency added to all algo actions size_t default_algo_latency_ns = 0; @@ -30,24 +29,16 @@ class Simulator { // The latency we use when calculating how long it takes to // process our own algo events - size_t estimated_exchange_latency_ns = 50000; + size_t estimated_exchange_latency_ns = 50000; }; int64_t run(IAlgoModel &, const RunOptions & = RunOptions()); - std::vector run(std::vector const &, const RunOptions & = RunOptions()); + std::vector run(std::vector const &, + const RunOptions & = RunOptions()); private: struct AlgoRecord; - struct AlgoActionRec; - struct AlgoEventRec; struct PImpl; - void enqueue_action(AlgoActionRec const&); - bool has_work() const; - void do_work(); - void on_simulator_input_event(SimulatorInputEvent const&); - void on_algo_action(AlgoActionRec const&); - MDP3TimeStamp next_simulator_event_send_time() const; - MDP3OrderKey next_order_id(); std::unique_ptr me; }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimEvent.h index 3261d5f7a..cecc3c1e2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimEvent.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimEvent.h @@ -1,9 +1,11 @@ #pragma once +#include "RecordedInputEvent.h" +#include "AlgoAction.h" + namespace bts::spartan::mdp3 { -struct SimEvent { -}; +using SimEvent = std::variant; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimEventSource.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimEventSource.cpp index 11ec574d6..d3d43d04c 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimEventSource.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimEventSource.cpp @@ -1,11 +1,61 @@ #include "SimEventSource.h" +#include "AlgoAction.h" +#include "SimInput.h" +#include "SimEvent.h" + +#include namespace bts::spartan::mdp3 { struct SimEventSource::PImpl { + SimInput const &input; + size_t next_input_event; + struct ActionSort { + bool operator()(const AlgoAction &l, const AlgoAction &r) const { + return l.transact_time.value() > r.transact_time.value(); + } + }; + std::priority_queue, ActionSort> + pending_algo_actions; }; -SimEventSource::SimEventSource(SimInput const&) : me (new PImpl()) {} +SimEventSource::SimEventSource(SimInput const &input) + : me(new PImpl{input, 0, {}}) {} + SimEventSource::~SimEventSource() {} +void SimEventSource::push(AlgoAction const &action) { + me->pending_algo_actions.push(action); } + +std::optional SimEventSource::pop() { + if (me->next_input_event != me->input.events.size() && + me->pending_algo_actions.size()) { + + // Both input events and algo actions are available, pick the + // earliest + auto &top_input_event = me->input.events[me->next_input_event]; + auto &top_algo_action = me->pending_algo_actions.top(); + + if (top_input_event.last_packet_send_time.value() <= + top_algo_action.transact_time.value()) { + // Input queue has it + return SimEvent{me->input.events[me->next_input_event++]}; + } else { + auto result = SimEvent {top_algo_action}; + me->pending_algo_actions.pop(); + return result; + } + } else if (me->next_input_event != me->input.events.size()) { + // Only have simulator events + return SimEvent{me->input.events[me->next_input_event++]}; + } else if (me->pending_algo_actions.size()) { + // Only have algo actions + auto result = me->pending_algo_actions.top(); + me->pending_algo_actions.pop(); + return result; + } else + return {}; +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimEventSource.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimEventSource.h index 749376aaa..4e3312be6 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimEventSource.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimEventSource.h @@ -1,21 +1,23 @@ #pragma once #include "../../common/mdp3_common.h" +#include "SimEvent.h" #include namespace bts::spartan::mdp3 { -class SimInput; -struct SimEvent; +struct SimInput; +struct AlgoAction; class SimEventSource { public: SimEventSource(SimInput const&); ~SimEventSource(); - void push_algo_action(MDP3TimeStamp,OrderChange const&); - std::optional pop_event(); + void push(AlgoAction const&); + std::optional pop(); private: struct PImpl; + struct OrderChangeRec; std::unique_ptr me; }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp index 30a8578b2..30e2ff777 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp @@ -369,12 +369,14 @@ struct ObsWrapper : mdp3::IChannelObserver { last_print = Timer::now(); std::stringstream str; mdp3::cme_ts_to_time_str(ev.transact_time.value(), &str); - std::cout << "Packet time: " << str.str() << " (" + std::cout << "Packet time: " << str.str() + << " Event ID: " << ev.event_id + << " (" << (ev.first_packet_sn - previous_packet) << " pkt/sec, " << (ev.first_packet_send_time.value() - previous_send_ts) / 1000000000 - << "x speedup) \r"; - std::cout.flush(); + << "x speedup)" + << std::endl; previous_packet = ev.first_packet_sn; previous_send_ts = ev.first_packet_send_time.value(); }