From 1cd1a23c800da66c8bb52faf65c2258d746303bd Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Wed, 5 Aug 2020 11:22:09 -0400 Subject: [PATCH] Added a unit test and fixed some bugs. --- src/lib/testing/catch_test_runner.cpp | 6 +- src/server/sparkd/CMakeDeps.cmake | 2 +- .../cmds/spartan/analysis/RunAnalysis.cpp | 63 +++++-- .../md/mdp3/handler/ChannelHandler.cpp | 42 +---- .../spartan/md/mdp3/handler/ChannelHandler.h | 4 +- .../cmds/spartan/md/mdp3/parsing/MDP3Parser.h | 18 +- .../spartan/md/mdp3/recovery/BookSnapshot.cpp | 58 +++++- .../spartan/md/mdp3/recovery/BookSnapshot.h | 6 +- .../md/mdp3/recovery/DefinitionSnapshot.cpp | 41 ++++- .../md/mdp3/recovery/DefinitionSnapshot.h | 2 + .../md/mdp3/recovery/IncrementalQueue.cpp | 29 ++- .../md/mdp3/recovery/IncrementalQueue.h | 2 + .../md/mdp3/recovery/RecoveryHandler.cpp | 32 ++-- .../md/mdp3/tests/snapshot_builder_tests.cpp | 91 ---------- .../md/mdp3/tests/spartan_md_tests.cpp | 169 ++++++++++++++++++ 15 files changed, 379 insertions(+), 186 deletions(-) delete mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/tests/snapshot_builder_tests.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp diff --git a/src/lib/testing/catch_test_runner.cpp b/src/lib/testing/catch_test_runner.cpp index 5ba3a5abb..59f9d16ef 100644 --- a/src/lib/testing/catch_test_runner.cpp +++ b/src/lib/testing/catch_test_runner.cpp @@ -16,9 +16,9 @@ int run_tests (std::vector const&args) { utils::internal::force_unittest_environment(); - // auto cfg = config::get (); - // cfg.mutable_console()->set_default_level("Fatal"); - // log::configure_logging(cfg); + auto cfg = config::get (); + cfg.mutable_console()->set_default_level("Fatal"); + log::configure_logging(cfg); bts_assert(args.size() < 100); char const *argv[100]; diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index 51f84da5b..ac3242ea6 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/analysis/ActionableEventAnalyzer.cpp;cmds/spartan/analysis/RunAnalysis.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/snapshot_builder_tests.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/analysis/ActionableEventAnalyzer.cpp;cmds/spartan/analysis/RunAnalysis.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp index 7260412f0..20fc6bd17 100644 --- a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp +++ b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp @@ -90,6 +90,16 @@ int convert_pcap_to_json(std::vector const &args) { return 0; } +template transactTime()) = 0> +uint64_t get_transact_time(M &m) { + return m.transactTime(); +} + +template lastUpdateTime()) = 0> +uint64_t get_transact_time(M &m) { + return m.lastUpdateTime(); +} + struct CSVPrinter { std::stringstream stime_buffer; @@ -107,28 +117,57 @@ struct CSVPrinter { } void on_begin_packet(mdp3::MDP3PacketView const &v) { packet = v; } - void on_end_packet() {} - template void operator()(M const &m) { + + template void print(M& m) { stime_buffer.str(""); stime_buffer.clear(); cme_ts_to_str(packet.sending_time(), &stime_buffer); ttime_buffer.str(""); ttime_buffer.clear(); - cme_ts_to_str(m.transactTime(), &ttime_buffer); + cme_ts_to_str(get_transact_time(m), &ttime_buffer); + auto t = get_transact_time(m); std::cout << ttime_buffer.str() << "," << stime_buffer.str() << "," - << (packet.sending_time() - m.transactTime()) / 1000 << "," - << m.transactTime() << "," << packet.sending_time() << "," - << packet.seq_num() << "," << utils::type_name() << "," - << (m.transactTime() < previous_s_time - ? ((previous_s_time - m.transactTime()) / 1000) - : 0) - << "," << const_cast(m).matchEventIndicator().endOfEvent() - << std::endl; + << (packet.sending_time() - t) / 1000 << "," << t << "," + << packet.sending_time() << "," << packet.seq_num() << "," + << utils::type_name() << "," + << (t < previous_s_time ? ((previous_s_time - t) / 1000) : 0) + << "," << m.matchEventIndicator().endOfEvent() << std::endl; previous_s_time = packet.sending_time(); - previous_t_time = m.transactTime(); + previous_t_time = t; } + + void operator()(::mdp3::MDIncrementalRefreshOrderBook43 &m) { print(m); } + void operator()(::mdp3::MDIncrementalRefreshOrderBook47 &m) { print(m); } + void operator()(::mdp3::MDIncrementalRefreshTradeSummary42 &m) { print(m); } + void operator()(::mdp3::MDIncrementalRefreshTradeSummary48 &m) { print(m); } + void operator()(::mdp3::MDIncrementalRefreshVolume37 &m) { print(m); } + void operator()(::mdp3::MDIncrementalRefreshDailyStatistics33 &m) { + print(m); + } + void operator()(::mdp3::MDIncrementalRefreshDailyStatistics49 &m) { + print(m); + } + void operator()(::mdp3::MDIncrementalRefreshLimitsBanding34 &m) { print(m); } + void operator()(::mdp3::MDIncrementalRefreshLimitsBanding50 &m) { print(m); } + void operator()(::mdp3::MDIncrementalRefreshBook46 &m) { print(m); } + void operator()(::mdp3::MDInstrumentDefinitionFuture54 &m) { print(m); } + void operator()(::mdp3::MDIncrementalRefreshSessionStatistics35 &m) { + print(m); + } + void operator()(::mdp3::MDIncrementalRefreshSessionStatistics51 &m) { + print(m); + } + void operator()(::mdp3::MDInstrumentDefinitionFuture27 &m) { print(m); } + void operator()(::mdp3::ChannelReset4 &m) { print(m); } + void operator()(::mdp3::MDIncrementalRefreshBook32 &m) { print(m); } + void operator()(::mdp3::MDInstrumentDefinitionOption41 &m) { print(m); } + void operator()(::mdp3::MDInstrumentDefinitionOption55 &m) { print(m); } + void operator()(::mdp3::MDInstrumentDefinitionSpread29 &m) { print(m); } + void operator()(::mdp3::MDInstrumentDefinitionSpread56 &m) { print(m); } + void operator()(::mdp3::QuoteRequest39 &m) { print(m); } + void operator()(::mdp3::SecurityStatus30 &m) { print(m); } }; int convert_pcap_to_csv_summary(std::vector const &args) { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp index 12c4c36ed..830038a5d 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp @@ -20,8 +20,6 @@ struct ChannelHandler::PImpl { AB_Arbitrageur real_time_ab; RealTimeHandler real_time_handler; - AB_Arbitrageur instrument_ab; - AB_Arbitrageur snapshot_ab; RecoveryHandler recovery_handler; IChannelObserver &observer; @@ -49,45 +47,21 @@ ChannelHandler::ChannelHandler(IChannelObserver &o, IFeedController &c) ChannelHandler::~ChannelHandler() {} -void ChannelHandler::on_instrument_feed(MDP3PacketView const &a, - MDP3PacketView const &b) { +void ChannelHandler::on_instrument_feed(MDP3PacketView const &pv) { switch (me->state) { - case State::RECOVERY: { - auto action = me->instrument_ab(a, b); - switch (action.action) { - case AB_Arbitrageur::APPLY: - parse_packet(action.packet, me->recovery_handler.on_instrument_feed); - break; - case AB_Arbitrageur::IGNORE: - break; - case AB_Arbitrageur::RECOVER: - start_recovery(); - break; - } + case State::RECOVERY: + parse_packet(pv, me->recovery_handler.on_instrument_feed); break; - } case State::REALTIME: break; } } -void ChannelHandler::on_snapshot_feed(MDP3PacketView const &a, - MDP3PacketView const &b) { +void ChannelHandler::on_snapshot_feed(MDP3PacketView const &pv) { switch (me->state) { - case State::RECOVERY: { - auto action = me->snapshot_ab(a, b); - switch (action.action) { - case AB_Arbitrageur::APPLY: - parse_packet(action.packet, me->recovery_handler.on_snapshot_feed); - break; - case AB_Arbitrageur::IGNORE: - break; - case AB_Arbitrageur::RECOVER: - start_recovery(); - break; - } + case State::RECOVERY: + parse_packet(pv, me->recovery_handler.on_snapshot_feed); break; - } case State::REALTIME: break; } @@ -131,15 +105,11 @@ void ChannelHandler::start_recovery() { switch (me->state) { case State::RECOVERY: me->real_time_ab = {}; - me->instrument_ab = {}; - me->snapshot_ab = {}; me->recovery_handler.reset(); break; case State::REALTIME: me->real_time_ab = {}; me->real_time_handler.reset(); - me->instrument_ab = {}; - me->snapshot_ab = {}; me->feed_controller.start_instrument_feed(); me->feed_controller.start_snapshot_feed(); me->state = State::RECOVERY; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h index 148757b39..31d2d8e75 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h @@ -15,8 +15,8 @@ public: ChannelHandler(IChannelObserver&, IFeedController &); ~ChannelHandler(); - void on_instrument_feed(MDP3PacketView const&a, MDP3PacketView const&b = {} ); - void on_snapshot_feed(MDP3PacketView const&a, MDP3PacketView const&b = {} ); + void on_instrument_feed(MDP3PacketView const&); + void on_snapshot_feed(MDP3PacketView const&); void on_incremental_feed(MDP3PacketView const&a, MDP3PacketView const&b = {} ); private: diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h b/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h index a019e2b79..e6524cba2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h @@ -183,9 +183,10 @@ using AllMDTypes = meta::list< template class ParserImpl { public: + static_assert (sizeof...(Msg) > 1); ParserImpl(Handler &h) : handler(h) {} - void on_packet(MDP3PacketView const &packet) { + void on_packet(MDP3PacketView const &packet) { handler.on_begin_packet(packet); for (auto const &msg : packet) (void)(try_handle(msg) || ...); @@ -197,7 +198,8 @@ template class ParserImpl { template bool try_handle(MDP3Msg const &msg) { if (msg.header().template_id == M::sbeTemplateId()) { - handler(M{(char *)msg.payload().data(), msg.payload().size()}); + M m{(char *)msg.payload().data(), msg.payload().size()}; + handler(m); return true; } return false; @@ -207,12 +209,11 @@ template class ParserImpl { template class Parser { template using is_handled = - std::is_invocable; + std::is_invocable; using HandledMsgs = typename meta::filter::type; using TypeListArgs = typename meta::cons::type; using RealParser = typename meta::with_type_list::type; - RealParser real_parser; public: @@ -287,16 +288,11 @@ class MDP3MsgWrapper { template class MDP3MsgBox { public: MDP3MsgBox(M const &m) - : _store(m.buffer(), m.sbePosition()), - _msg(_store.data(), _store.size()) {} - M *operator->() { return &_msg; } - M const *operator->() const { return &_msg; } - M const &operator*() const { return _msg; } - M &operator*() { return _msg; } + : _store(m.buffer(), m.sbePosition()) {} + M unbox() const {return M((char*)_store.data(),_store.size());} private: std::string _store; - M _msg; }; } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp index 94d3efa82..8deb6d858 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp @@ -1,6 +1,7 @@ #include "BookSnapshot.h" #include "../gen/SnapshotFullRefresh52.h" #include "../gen/SnapshotFullRefreshOrderBook53.h" +#include "lib/logging/logging.h" namespace bts::spartan::mdp3 { @@ -15,19 +16,29 @@ struct BookSnapshot::PImpl { }; BookSnapshot::BookSnapshot() : me(new PImpl()) {} - BookSnapshot::~BookSnapshot() {} void BookSnapshot::on_begin_packet(MDP3PacketView const &pv) { + LOG(Spartan, Diag, "Received packet: {} in snapshot feed", pv.seq_num()); switch (me->state) { case State::WFIRST_PACKET: - if (pv.seq_num () == 1) { - me->state = State::ACCUMULATING_BOOKS; + if (pv.seq_num() == 1) { + LOG(Spartan, Info, + "Observed packet 1 in snapshot feed. Starting to accumulate books"); + transition(State::ACCUMULATING_BOOKS); me->last_sn = 1; } break; case State::ACCUMULATING_BOOKS: - bts_assert(me->last_sn + 1 == pv.seq_num()); + if (me->last_sn + 1 != pv.seq_num()) { + LOG(Spartan, Warn, + "Detected loss in snapshot feed. Expected {} but received {}. " + "Restarting snapshot recovery", + me->last_sn + 1, pv.seq_num()); + reset(); + } + else + me->last_sn = pv.seq_num(); break; case State::COMPLETE: break; @@ -45,6 +56,8 @@ void BookSnapshot::operator()(::mdp3::SnapshotFullRefresh52 &book) { me->newest_snapshot_sn = std::max(book.lastMsgSeqNumProcessed(), me->newest_snapshot_sn); me->books.emplace_back(ib); + LOG(Spartan, Info, "Recovered book {}/{} from snapshot feed", + me->books.size(), book.totNumReports()); check_completion(); break; } @@ -59,7 +72,12 @@ void BookSnapshot::operator()(::mdp3::SnapshotFullRefreshOrderBook53 &chunk) { break; case State::ACCUMULATING_BOOKS: { bts_assert(me->books.size()); - me->books.back().book_chunks.push_back(chunk); + auto &ibs = me->books.back(); + auto book_msg = ibs.book.unbox(); + ibs.book_chunks.push_back(chunk); + LOG(Spartan, Diag, "Recovered chunk {}/{} of book {}/{}", + ibs.book_chunks.size(), chunk.noChunks(), + me->books.size(),book_msg.totNumReports()); check_completion(); break; } @@ -70,19 +88,22 @@ void BookSnapshot::operator()(::mdp3::SnapshotFullRefreshOrderBook53 &chunk) { void BookSnapshot::on_end_packet() {} -void BookSnapshot::reset() { *me = {}; } +void BookSnapshot::reset() { + transition(State::WFIRST_PACKET); + *me = {}; +} bool BookSnapshot::is_complete() const { if (me->books.empty()) return false; - if (me->books[0].book->totNumReports() != me->books.size()) + if (me->books[0].book.unbox().totNumReports() != me->books.size()) return false; for (auto &book : me->books) { if (book.book_chunks.empty()) return false; - if (book.book_chunks.size() != book.book_chunks[0]->noChunks()) + if (book.book_chunks.size() != book.book_chunks[0].unbox().noChunks()) return false; } return true; @@ -98,7 +119,7 @@ uint32_t BookSnapshot::newest_snapshot_seq_num() const { void BookSnapshot::check_completion() { if (is_complete()) - me->state = State::COMPLETE; + transition(State::COMPLETE); } std::vector const &BookSnapshot::get_books() const { @@ -111,4 +132,23 @@ std::vector BookSnapshot::move_books_and_reset() { return result; } +std::string BookSnapshot::state_name(State st) { + switch (st) { + case State::WFIRST_PACKET: + return "WFIRST_PACKET"; + case State::ACCUMULATING_BOOKS: + return "ACCUMULATING_BOOKS"; + case State::COMPLETE: + return "COMPLETE"; + } + bts_assert(false); + return ""; +} + +void BookSnapshot::transition(State st) { + LOG(Spartan,Info,"Book snapshot transitioning from {} to {}", + state_name(me->state), state_name(st)); + me->state = st; +} + } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.h b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.h index 9fe06f3a1..1edf850bf 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.h @@ -33,16 +33,16 @@ public: void on_begin_packet(MDP3PacketView const&); void on_end_packet(); - void operator()(::mdp3::SnapshotFullRefresh52 &); void operator()(::mdp3::SnapshotFullRefreshOrderBook53 &); - private: - void check_completion(); enum class State; struct PImpl; std::unique_ptr me; + void check_completion(); + std::string state_name(State); + void transition(State); }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp index 3e03c67e8..7ff308db4 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp @@ -19,15 +19,26 @@ DefinitionSnapshot::DefinitionSnapshot() : me(new PImpl()) {} DefinitionSnapshot::~DefinitionSnapshot() {} void DefinitionSnapshot::on_begin_packet(MDP3PacketView const &pv) { + LOG(Spartan, Diag, "Received packet: {} in instrument feed", pv.seq_num()); switch (me->state) { case State::WFIRST_PACKET: if (pv.seq_num() == 1) { - me->state = State::ACCUMULATING_DEFS; + LOG(Spartan, Info, + "Observed packet 1 in instrument feed. Starting to accumulate instruments"); + transition(State::ACCUMULATING_DEFS); me->last_sn = 1; } break; case State::ACCUMULATING_DEFS: - bts_assert(me->last_sn + 1 == pv.seq_num()); + if (me->last_sn + 1 != pv.seq_num()) { + LOG(Spartan, Warn, + "Detected loss in instrument feed. Expected {} but received {}. " + "Restarting snapshot recovery", + me->last_sn + 1, pv.seq_num()); + reset(); + } + else + me->last_sn = pv.seq_num(); break; case State::COMPLETE: break; @@ -43,7 +54,7 @@ template void DefinitionSnapshot::on_definition(D &d) { if (auto nr = me->definitions[0].tot_num_reports(); nr == me->definitions.size()) { bts_assert(nr); - me->state = State::COMPLETE; + transition(State::COMPLETE); } break; case State::COMPLETE: @@ -65,7 +76,10 @@ void DefinitionSnapshot::operator()(::mdp3::MDInstrumentDefinitionSpread56 &d) { void DefinitionSnapshot::on_end_packet() {} -void DefinitionSnapshot::reset() { *me = {}; } +void DefinitionSnapshot::reset() { + transition(State::WFIRST_PACKET); + *me = {}; +} bool DefinitionSnapshot::is_complete() const { return me->state == State::COMPLETE; @@ -82,4 +96,23 @@ DefinitionSnapshot::move_definitions_and_reset() { return result; } +std::string DefinitionSnapshot::state_name(State st) { + switch (st) { + case State::WFIRST_PACKET: + return "WFIRST_PACKET"; + case State::ACCUMULATING_DEFS: + return "ACCUMULATING_DEFS"; + case State::COMPLETE: + return "COMPLETE"; + } + bts_assert(false); + return ""; +} + +void DefinitionSnapshot::transition(State st) { + LOG(Spartan,Info,"Instrument snapshot transitioning from {} to {}", + state_name(me->state), state_name(st)); + me->state = st; +} + } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.h b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.h index cf89569cf..8c929a70e 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.h @@ -36,6 +36,8 @@ class DefinitionSnapshot { struct PImpl; std::unique_ptr me; template void on_definition(D &); + void transition(State); + std::string state_name(State); }; } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp index 283a05c9d..4aa029467 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp @@ -44,7 +44,10 @@ uint32_t IncrementalQueue::newest_packet_seq_num() const { return 0; } -void IncrementalQueue::reset() { *me = {}; } +void IncrementalQueue::reset() { + transition(State::W_PRELUDE); + *me = {}; +} std::vector IncrementalQueue::get_queued_packets_and_reset() { bts_assert(is_consistent()); @@ -69,11 +72,11 @@ void IncrementalQueue::on_end_packet() { break; case State::W_EPILOGUE: if (me->last_message_was_event_end) - me->state = State::CONSISTENT; + transition(State::CONSISTENT); break; case State::CONSISTENT: if (!me->last_message_was_event_end) - me->state = State::W_EPILOGUE; + transition(State::W_EPILOGUE); break; } } @@ -106,7 +109,7 @@ template void IncrementalQueue::on_message(M &m) { previous_last_observed_event != me->last_observed_event && was_first) { // A new event has started with the first message of a new packet - me->state = State::W_EPILOGUE; + transition(State::W_EPILOGUE); } break; case State::CONSISTENT: @@ -167,4 +170,22 @@ void IncrementalQueue::operator()(::mdp3::SecurityStatus30 &m) { on_message(m); } +std::string IncrementalQueue::state_name(State st) { + switch (st) { + case State::W_PRELUDE: + return "W_PRELUDE"; + case State::W_EPILOGUE: + return "W_EPILOGUE"; + case State::CONSISTENT: + return "CONSISTENT"; + } + bts_assert(false); + return ""; +} + +void IncrementalQueue::transition(State st) { + LOG(Spartan,Info,"Incremental queue transitioning from {} to {}", + state_name(me->state), state_name(st)); + me->state = st; +} } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.h b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.h index 4fc49705f..4535f3b63 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.h @@ -61,6 +61,8 @@ class IncrementalQueue { std::unique_ptr me; template void on_message(M &); template void on_definition(D &); + std::string state_name(State); + void transition(State); }; } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp index 7e3d92b67..97cfa2327 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp @@ -10,9 +10,16 @@ namespace bts::spartan::mdp3 { struct RecoveryHandler::PImpl { PImpl(RecoveryHandler &owner) - : owner(owner), book_snapshot(), definition_snapshot(), - incremental_queue(), instrument_feed_handler(*this), - snapshot_feed_handler(*this), incremental_feed_handler(*this) {} + : owner(owner), + + book_snapshot(), + definition_snapshot(), + incremental_queue(), + + instrument_feed_handler(*this), + snapshot_feed_handler(*this), + incremental_feed_handler(*this) + {} RecoveryHandler &owner; @@ -33,7 +40,9 @@ RecoveryHandler::RecoveryHandler() RecoveryHandler::~RecoveryHandler() {} void RecoveryHandler::reset() { - me = std::make_unique (*this); + me->book_snapshot.reset(); + me->definition_snapshot.reset(); + me->incremental_queue.reset(); } RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot() { @@ -51,7 +60,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot() { for (auto &d : definitions_ref) definition_ids.insert(d.security_id()); for (auto &b : books) - if (definition_ids.count(MDP3SecurityID{b.book->securityID()}) == 0) { + if (definition_ids.count(MDP3SecurityID{b.book.unbox().securityID()}) == 0) { // We are missing a definition for one of the books so we // need to start over and get a new round of definitions me->definition_snapshot.reset(); @@ -99,7 +108,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot() { // Index books so we can join them with the definitions std::unordered_map book_snaps_idx; for (auto &bs : book_snaps) - book_snaps_idx[MDP3SecurityID{bs.book->securityID()}] = &bs; + book_snaps_idx[MDP3SecurityID{bs.book.unbox().securityID()}] = &bs; // Now iterate over all definitions, locate the corresponding book, // if any, and use both to build an InstrumentSnapshot instance. @@ -109,10 +118,13 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot() { auto this_book_snap = book_snaps_idx[d.security_id()]; uint32_t last_processed_sn = 0; if (this_book_snap) { - last_processed_sn = this_book_snap->book->lastMsgSeqNumProcessed(); - update_book_in_snapshot(*instrument_snapshot, *(this_book_snap->book)); - for (auto &bchunk : this_book_snap->book_chunks) - update_book_chunk_in_snapshot(*instrument_snapshot, *bchunk); + last_processed_sn = this_book_snap->book.unbox().lastMsgSeqNumProcessed(); + auto book_snap_msg = this_book_snap->book.unbox(); + update_book_in_snapshot(*instrument_snapshot,book_snap_msg); + for (auto &bchunk : this_book_snap->book_chunks) { + auto chunk_msg = bchunk.unbox(); + update_book_chunk_in_snapshot(*instrument_snapshot,chunk_msg); + } } sortable_snapshots.push_back( {std::move(instrument_snapshot), last_processed_sn}); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/tests/snapshot_builder_tests.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/tests/snapshot_builder_tests.cpp deleted file mode 100644 index 116d45b6f..000000000 --- a/src/server/sparkd/cmds/spartan/md/mdp3/tests/snapshot_builder_tests.cpp +++ /dev/null @@ -1,91 +0,0 @@ -#include - -#include "server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h" - -#include "lib/utils/overloaded.h" -#include "lib/utils/type_name.h" -#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h" -#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h" -#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" -#include "server/sparkd/cmds/spartan/util/PCapReader.h" - -#include -#include - -namespace bts::spartan::mdp3::tests { - -auto IA_ip = "224.0.31.1"; -auto IA_port = 14310; -auto IB_ip = "224.0.32.1"; -auto IB_port = 15310; -auto NA_ip = "224.0.31.43"; -auto NA_port = 14310; -auto NB_ip = "224.0.32.43"; -auto NB_port = 15310; -auto SA_ip = "224.0.31.22"; -auto SA_port = 14310; -auto SB_ip = "224.0.32.22"; -auto SB_port = 15310; -auto SAMBO_ip = "233.72.75.1"; -auto SAMBO_port = 23310; -auto SBMBO_ip = "233.72.75.64"; -auto SBMBO_port = 22310; - -TEST_CASE("MDP3 Tests", "[snapshot_builder]") { - - struct PacketBuffers { - MDP3Packet i; - MDP3Packet n; - MDP3Packet smbo; - }; - - auto reader = PCapReader{"/home/ppinto/es_all.pcap"}; - std::array buffers; - struct Obs : IChannelObserver { - void on_recovery_complete( - std::vector const &) override {} - void on_real_time_event(MDP3EventInfo const &) override {} - void on_real_time_channel_loss() override {} - } obs; - struct Ctrl : IFeedController { - void start_instrument_feed() override {}; - void stop_instrument_feed() override {}; - void start_snapshot_feed() override {}; - void stop_snapshot_feed() override {}; - void start_incremental_feed() override {}; - void stop_incremental_feed() override {}; - } ctrl; - - ChannelHandler handler (obs,ctrl); - - SECTION("smoke_test") { - bool recovered = false; - reader.on_packet() = [&](auto const &info, auto const &payload) { - if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) { - buffers[0].i = MDP3Packet{payload}; - handler.on_incremental_feed(buffers[0].i.view(),buffers[1].i.view()); - } else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) { - buffers[1].i = MDP3Packet{payload}; - handler.on_incremental_feed(buffers[0].i.view(),buffers[1].i.view()); - } else if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) { - buffers[0].n = MDP3Packet{payload}; - handler.on_instrument_feed(buffers[0].n.view(),buffers[1].n.view()); - } else if ((info.dst_ip == NB_ip && info.dst_port == NB_port)) { - buffers[1].n = MDP3Packet{payload}; - handler.on_instrument_feed(buffers[0].n.view(),buffers[1].n.view()); - } else if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) { - buffers[0].smbo = MDP3Packet{payload}; - handler.on_snapshot_feed(buffers[0].smbo.view(),buffers[1].smbo.view()); - } else if ((info.dst_ip == SBMBO_ip && info.dst_port == SBMBO_port)) { - buffers[1].smbo = MDP3Packet{payload}; - handler.on_snapshot_feed(buffers[0].smbo.view(),buffers[1].smbo.view()); - } - return !recovered; - }; - while (reader.read_next()) - ; - REQUIRE(recovered); - } -} - -} // namespace bts::spartan::mdp3::tests diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp new file mode 100644 index 000000000..47911bcba --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp @@ -0,0 +1,169 @@ +#include + +#include "server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h" + +#include "lib/utils/overloaded.h" +#include "lib/utils/type_name.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" +#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" +#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h" +#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h" +#include "server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.h" + +#include "server/sparkd/cmds/spartan/util/PCapReader.h" + +#include +#include + +namespace bts::spartan::mdp3::tests { + +auto IA_ip = "224.0.31.1"; +auto IA_port = 14310; +auto IB_ip = "224.0.32.1"; +auto IB_port = 15310; +auto NA_ip = "224.0.31.43"; +auto NA_port = 14310; +auto NB_ip = "224.0.32.43"; +auto NB_port = 15310; +auto SA_ip = "224.0.31.22"; +auto SA_port = 14310; +auto SB_ip = "224.0.32.22"; +auto SB_port = 15310; +auto SAMBO_ip = "233.72.75.1"; +auto SAMBO_port = 23310; +auto SBMBO_ip = "233.72.75.64"; +auto SBMBO_port = 22310; + +struct PacketBuffers { + MDP3Packet i; +}; + +TEST_CASE("spartan_smoke", "[spartan_smoke]") { + + auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"}; + std::array buffers; + struct Obs : IChannelObserver { + std::unordered_map + instruments; + + void on_recovery_complete( + std::vector const &snaps) override { + for (auto &i : snaps) + instruments[i->security_id()] = i; + } + void on_real_time_event(MDP3EventInfo const &ev) override { + for (auto i : ev.affected_securities) + instruments[i->security_id()] = i; + } + + void on_real_time_channel_loss() override {} + } obs; + struct Ctrl : IFeedController { + void start_instrument_feed() override{}; + void stop_instrument_feed() override{}; + void start_snapshot_feed() override{}; + void stop_snapshot_feed() override{}; + void start_incremental_feed() override{}; + void stop_incremental_feed() override{}; + } ctrl; + + ChannelHandler handler(obs, ctrl); + + SECTION("can_generate_snapshot") { + reader.on_packet() = [&](auto const &info, auto const &payload) { + if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) { + buffers[0].i = MDP3Packet{payload}; + handler.on_incremental_feed(buffers[0].i.view(), + buffers[1].i.view()); + } else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) { + buffers[1].i = MDP3Packet{payload}; + handler.on_incremental_feed(buffers[0].i.view(), + buffers[1].i.view()); + } else if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) { + handler.on_instrument_feed(MDP3PacketView{payload}); + } else if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) { + handler.on_snapshot_feed(MDP3PacketView{payload}); + } + return true; + }; + while (reader.read_next()) + ; + REQUIRE(obs.instruments.size() == 12); + } +} + +TEST_CASE("spartan_book_snap", "[spartan_book_snap]") { + + auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"}; + + BookSnapshot book_snapshot; + SECTION("can_generate_book_snapshot") { + reader.on_packet() = [&](auto const &info, auto const &payload) { + if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) + parse_packet(MDP3PacketView{payload}, book_snapshot); + return true; + }; + while (reader.read_next()) + ; + REQUIRE(book_snapshot.is_complete()); + } +} + +TEST_CASE("spartan_def_snap", "[spartan_def_snap]") { + + auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"}; + + DefinitionSnapshot def_snapshot; + SECTION("can_generate_definition_snapshot") { + reader.on_packet() = [&](auto const &info, auto const &payload) { + if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) + parse_packet(MDP3PacketView{payload}, def_snapshot); + return true; + }; + while (reader.read_next()) + ; + REQUIRE(def_snapshot.is_complete()); + } +} + +TEST_CASE("spartan_inc_queue", "[spartan_inc_queue]") { + + auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"}; + std::array buffers; + IncrementalQueue q; + AB_Arbitrageur ab; + SECTION("can_get_consistent_queue") { + + reader.on_packet() = [&](auto const &info, auto const &payload) { + + if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) { + buffers[0].i = MDP3Packet{payload}; + } else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) { + buffers[1].i = MDP3Packet{payload}; + } else + return true; + + auto action = ab(buffers[0].i.view(), buffers[1].i.view()); + switch (action.action) { + case AB_Arbitrageur::APPLY: + parse_packet(action.packet,q); + return !q.is_consistent(); + case AB_Arbitrageur::IGNORE: + break; + case AB_Arbitrageur::RECOVER: + REQUIRE(false); + break; + } + return true; + }; + + while (reader.read_next()) + ; + REQUIRE(q.is_consistent()); + } +} + +} // namespace bts::spartan::mdp3::tests