From 5af89e0124efb893e257bf3afa40996cbe262690 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Thu, 6 Aug 2020 12:11:58 -0400 Subject: [PATCH] Fixed a few more bugs. --- src/server/sparkd/CMakeDeps.cmake | 2 +- .../md/mdp3/common/InstrumentDefinition.h | 163 ++++++------------ .../md/mdp3/common/InstrumentSnapshot.cpp | 13 +- .../spartan/md/mdp3/parsing/MDP3MsgBox.cpp | 1 + .../cmds/spartan/md/mdp3/parsing/MDP3MsgBox.h | 44 +++++ .../cmds/spartan/md/mdp3/parsing/MDP3Parser.h | 35 +--- .../md/mdp3/realtime/RealTimeHandler.cpp | 66 +++---- .../spartan/md/mdp3/recovery/BookSnapshot.cpp | 6 +- .../spartan/md/mdp3/recovery/BookSnapshot.h | 1 + .../md/mdp3/recovery/RecoveryHandler.cpp | 48 +++--- .../md/mdp3/tests/spartan_md_tests.cpp | 8 +- 11 files changed, 181 insertions(+), 206 deletions(-) create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp create mode 100644 src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3MsgBox.h diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index ac3242ea6..710304743 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/analysis/ActionableEventAnalyzer.cpp;cmds/spartan/analysis/RunAnalysis.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/analysis/ActionableEventAnalyzer.cpp;cmds/spartan/analysis/RunAnalysis.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h index fd46422a7..96d0b2ce4 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h @@ -1,6 +1,7 @@ #pragma once #include "mdp3_common.h" +#include "../parsing/MDP3MsgBox.h" #include "../gen/MDInstrumentDefinitionFuture54.h" #include "../gen/MDInstrumentDefinitionOption55.h" #include "../gen/MDInstrumentDefinitionSpread56.h" @@ -12,174 +13,118 @@ namespace bts::spartan::mdp3 { class InstrumentDefinition { public: - InstrumentDefinition(InstrumentDefinition &&d) { operator=(std::move(d)); } + InstrumentDefinition(InstrumentDefinition &&d) = default; + InstrumentDefinition(InstrumentDefinition const &d) = default; + InstrumentDefinition &operator=(InstrumentDefinition &&other) = default; + InstrumentDefinition &operator=(InstrumentDefinition const &other) = default; - InstrumentDefinition(InstrumentDefinition const &d) { operator=(d); } - - InstrumentDefinition &operator=(InstrumentDefinition &&other) { - _store = std::move(other._store); - init_def_from_store(other._def); - other._def = {}; - return *this; - } - - InstrumentDefinition &operator=(InstrumentDefinition const &other) { - _store = other._store; - init_def_from_store(other._def); - return *this; - } - - InstrumentDefinition(::mdp3::MDInstrumentDefinitionFuture54 const &d) - : _store(d.buffer(), d.sbePosition()), - _def(::mdp3::MDInstrumentDefinitionFuture54{_store.data(), - _store.size()}) {} - - InstrumentDefinition(::mdp3::MDInstrumentDefinitionOption55 const &d) - : _store(d.buffer(), d.sbePosition()), - _def(::mdp3::MDInstrumentDefinitionOption55{_store.data(), - _store.size()}) {} - - InstrumentDefinition(::mdp3::MDInstrumentDefinitionSpread56 const &d) - : _store(d.buffer(), d.sbePosition()), - _def(::mdp3::MDInstrumentDefinitionSpread56{_store.data(), - _store.size()}) {} + template + InstrumentDefinition(M const &d) + : _def (MDP3MsgBox (d)) + {} MDP3TimeStamp last_update_time() const { return std::visit( - overloaded{[](::mdp3::MDInstrumentDefinitionFuture54 const &m) { - return MDP3TimeStamp{m.lastUpdateTime()}; + overloaded{[](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> const &m) { + return MDP3TimeStamp{m->lastUpdateTime()}; }, - [](::mdp3::MDInstrumentDefinitionOption55 const &m) { - return MDP3TimeStamp{m.lastUpdateTime()}; + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> const &m) { + return MDP3TimeStamp{m->lastUpdateTime()}; }, - [](::mdp3::MDInstrumentDefinitionSpread56 const &m) { - return MDP3TimeStamp{m.lastUpdateTime()}; + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> const &m) { + return MDP3TimeStamp{m->lastUpdateTime()}; }}, _def); } size_t tot_num_reports() const { return std::visit( - overloaded{[](::mdp3::MDInstrumentDefinitionFuture54 const &m) { - return m.totNumReports(); + overloaded{[](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> const &m) { + return m->totNumReports(); }, - [](::mdp3::MDInstrumentDefinitionOption55 const &m) { - return m.totNumReports(); + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> const &m) { + return m->totNumReports(); }, - [](::mdp3::MDInstrumentDefinitionSpread56 const &m) { - return m.totNumReports(); - }}, - _def); - } - - std::string to_string() { - return std::visit( - overloaded{[](::mdp3::MDInstrumentDefinitionFuture54 const &m) { - return std::string{m.buffer(), m.sbePosition()}; - }, - [](::mdp3::MDInstrumentDefinitionOption55 const &m) { - return std::string{m.buffer(), m.sbePosition()}; - }, - [](::mdp3::MDInstrumentDefinitionSpread56 const &m) { - return std::string{m.buffer(), m.sbePosition()}; + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> const &m) { + return m->totNumReports(); }}, _def); } MDP3SecurityID security_id() const { return std::visit( - overloaded{[](::mdp3::MDInstrumentDefinitionFuture54 const &m) { - return MDP3SecurityID{m.securityID()}; + overloaded{[](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> const &m) { + return MDP3SecurityID{m->securityID()}; }, - [](::mdp3::MDInstrumentDefinitionOption55 const &m) { - return MDP3SecurityID{m.securityID()}; + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> const &m) { + return MDP3SecurityID{m->securityID()}; }, - [](::mdp3::MDInstrumentDefinitionSpread56 const &m) { - return MDP3SecurityID{m.securityID()}; + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> const &m) { + return MDP3SecurityID{m->securityID()}; }}, _def); } ::mdp3::MatchEventIndicator match_event_indicator() const { return std::visit( - overloaded{[](::mdp3::MDInstrumentDefinitionFuture54 &m) { - return m.matchEventIndicator(); + overloaded{[](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> &m) { + return m->matchEventIndicator(); }, - [](::mdp3::MDInstrumentDefinitionOption55 &m) { - return m.matchEventIndicator(); + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> &m) { + return m->matchEventIndicator(); }, - [](::mdp3::MDInstrumentDefinitionSpread56 &m) { - return m.matchEventIndicator(); + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> &m) { + return m->matchEventIndicator(); }}, _def); } const char *security_group() const { return std::visit( - overloaded{[](::mdp3::MDInstrumentDefinitionFuture54 &m) { - return m.securityGroup(); + overloaded{[](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> &m) { + return m->securityGroup(); }, - [](::mdp3::MDInstrumentDefinitionOption55 &m) { - return m.securityGroup(); + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> &m) { + return m->securityGroup(); }, - [](::mdp3::MDInstrumentDefinitionSpread56 &m) { - return m.securityGroup(); + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> &m) { + return m->securityGroup(); }}, _def); } const char *asset() const { return std::visit( - overloaded{[](::mdp3::MDInstrumentDefinitionFuture54 &m) { - return m.asset(); + overloaded{[](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> &m) { + return m->asset(); }, - [](::mdp3::MDInstrumentDefinitionOption55 &m) { - return m.asset(); + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> &m) { + return m->asset(); }, - [](::mdp3::MDInstrumentDefinitionSpread56 &m) { - return m.asset(); + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> &m) { + return m->asset(); }}, _def); } ::mdp3::SecurityTradingStatus::Value trading_status() const { return std::visit( - overloaded{[](::mdp3::MDInstrumentDefinitionFuture54 &m) { - return m.mDSecurityTradingStatus(); + overloaded{[](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> &m) { + return m->mDSecurityTradingStatus(); }, - [](::mdp3::MDInstrumentDefinitionOption55 &m) { - return m.mDSecurityTradingStatus(); + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> &m) { + return m->mDSecurityTradingStatus(); }, - [](::mdp3::MDInstrumentDefinitionSpread56 &m) { - return m.mDSecurityTradingStatus(); + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> &m) { + return m->mDSecurityTradingStatus(); }}, _def); } private: - using DefVar = std::variant<::mdp3::MDInstrumentDefinitionFuture54, - ::mdp3::MDInstrumentDefinitionOption55, - ::mdp3::MDInstrumentDefinitionSpread56>; - void init_def_from_store(DefVar const &d) { - std::visit( - overloaded{[this](::mdp3::MDInstrumentDefinitionFuture54 const &) { - _def = ::mdp3::MDInstrumentDefinitionFuture54{ - _store.data(), _store.size()}; - }, - - [this](::mdp3::MDInstrumentDefinitionOption55 const &) { - _def = ::mdp3::MDInstrumentDefinitionOption55{ - _store.data(), _store.size()}; - }, - - [this](::mdp3::MDInstrumentDefinitionSpread56 const &) { - _def = ::mdp3::MDInstrumentDefinitionSpread56{ - _store.data(), _store.size()}; - }}, - d); - } - - std::string _store; + using DefVar = std::variant, + MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55>, + MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56>>; mutable DefVar _def; }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp index f6227762c..83fc08c72 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp @@ -168,10 +168,12 @@ template void InstrumentSnapshot::insert(api::Side s, std::array &books, size_t idx, BookLevel const &level) { auto &book = books[s - api::SIDE_BUY]; - bts_assert_d(idx - 1 < book.size()); - for (size_t i = book.size() - 2; i >= idx - 1; i++) + int zidx = (int) (idx-1); + bts_assert_d(zidx < (int) book.size()); + + for (int i = book.size() - 2; i >= zidx; i--) book[i + 1] = book[i]; - book[idx - 1] = level; + book[zidx] = level; } template @@ -186,8 +188,9 @@ template void InstrumentSnapshot::remove(api::Side s, std::array &books, size_t idx) { auto &book = books[s - api::SIDE_BUY]; - bts_assert_d(idx - 1 < book.size()); - for (size_t i = idx - 1; i < book.size() - 2; i++) + int zidx = (int) (idx-1); + bts_assert_d(zidx < (int)book.size()); + for (int i = zidx; i < (int) book.size() - 2; i++) book[i] = book[i + 1]; book[book.size() - 1] = {}; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp new file mode 100644 index 000000000..5b992c1b1 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp @@ -0,0 +1 @@ +#include "MDP3MsgBox.h" diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3MsgBox.h b/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3MsgBox.h new file mode 100644 index 000000000..058ceaa0c --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3MsgBox.h @@ -0,0 +1,44 @@ +#pragma once + +#include "MDP3Parser.h" + +namespace bts::spartan::mdp3 { + +template class MDP3MsgBox { + public: + MDP3MsgBox(M const &m) { + auto header = (MDP3MsgHeader*) (m.buffer() - sizeof(MDP3MsgHeader)); + _store = std::string((char*)header,header->msg_size); + _m = M(_store.data()+sizeof(MDP3MsgHeader), _store.size()-sizeof(MDP3MsgHeader)); + } + + MDP3MsgBox(MDP3MsgBox const&other) { + operator=(other); + } + + MDP3MsgBox(MDP3MsgBox &&other) { + operator=(std::move(other)); + } + + MDP3MsgBox& operator=(MDP3MsgBox const&other) { + _store = other._store; + _m = M(_store.data()+sizeof(MDP3MsgHeader), _store.size()-sizeof(MDP3MsgHeader)); + return *this; + } + + MDP3MsgBox& operator=(MDP3MsgBox &&other) { + _store = std::move(other._store); + _m = M(_store.data()+sizeof(MDP3MsgHeader), _store.size()-sizeof(MDP3MsgHeader)); + return *this; + } + + M* operator->() {return &_m;} + M const* operator->() const {return &_m;} + M operator*() const {return _m;} + + private: + M _m; + std::string _store; +}; + +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h b/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h index aa93b0378..a748a7f97 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h @@ -160,8 +160,7 @@ class MDP3Packet { }; using AllMDTypes = meta::list< - ::mdp3::MDIncrementalRefreshOrderBook43, - ::mdp3::MDIncrementalRefreshOrderBook47, + ::mdp3::MDIncrementalRefreshBook46, ::mdp3::MDIncrementalRefreshOrderBook47, ::mdp3::MDIncrementalRefreshTradeSummary42, ::mdp3::MDIncrementalRefreshTradeSummary48, ::mdp3::MDIncrementalRefreshVolume37, @@ -169,7 +168,7 @@ using AllMDTypes = meta::list< ::mdp3::MDIncrementalRefreshDailyStatistics49, ::mdp3::MDIncrementalRefreshLimitsBanding34, ::mdp3::MDIncrementalRefreshLimitsBanding50, - ::mdp3::MDIncrementalRefreshBook46, ::mdp3::MDInstrumentDefinitionFuture54, + ::mdp3::MDInstrumentDefinitionFuture54, ::mdp3::MDIncrementalRefreshSessionStatistics35, ::mdp3::MDIncrementalRefreshSessionStatistics51, ::mdp3::MDInstrumentDefinitionFuture27, ::mdp3::ChannelReset4, @@ -177,8 +176,9 @@ using AllMDTypes = meta::list< ::mdp3::MDInstrumentDefinitionOption55, ::mdp3::MDInstrumentDefinitionSpread29, ::mdp3::MDInstrumentDefinitionSpread56, ::mdp3::QuoteRequest39, - ::mdp3::SecurityStatus30, ::mdp3::SnapshotFullRefresh38, - ::mdp3::SnapshotFullRefresh52, ::mdp3::SnapshotFullRefreshOrderBook44, + ::mdp3::SecurityStatus30, ::mdp3::MDIncrementalRefreshOrderBook43, + ::mdp3::SnapshotFullRefresh38, ::mdp3::SnapshotFullRefresh52, + ::mdp3::SnapshotFullRefreshOrderBook44, ::mdp3::SnapshotFullRefreshOrderBook53>; template class ParserImpl { @@ -269,29 +269,4 @@ template class MsgDispatcher { } }; -class MDP3MsgWrapper { - public: - template MDP3MsgWrapper(MSG &m) { - _store = {m.buffer(), m.sbePosition()}; - _template_id = m.sbeTemplateId(); - } - - template void visit(Visitor &v) { - MsgDispatcher{_template_id, _store, v}; - } - - private: - std::string _store; - std::uint16_t _template_id; -}; - -template class MDP3MsgBox { - public: - MDP3MsgBox(M const &m) : _store(m.buffer(), m.sbePosition()) {} - M unbox() const { return M((char *)_store.data(), _store.size()); } - - private: - std::string _store; -}; - } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp index 02f0e84e5..92e6d010f 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp @@ -1,9 +1,9 @@ #include "RealTimeHandler.h" -#include "IRealTimeObserver.h" #include "../common/InstrumentSnapshot.h" #include "../common/mdp3_common.h" #include "../gen/MDIncrementalRefreshBook46.h" #include "../parsing/MDP3Parser.h" +#include "IRealTimeObserver.h" #include namespace bts::spartan::mdp3 { @@ -26,9 +26,9 @@ struct RealTimeHandler::PImpl { PImpl(IRealTimeObserver &o) : instruments(), current_packet_send_time(MDP3TimeStamp{0}), - current_packet_sn(0), current_event(), observer(o), - pending_ts(), pending_order_detail() {} - + current_packet_sn(0), current_event(), observer(o), pending_ts(), + pending_order_detail() {} + std::unordered_map> instruments; @@ -42,12 +42,11 @@ struct RealTimeHandler::PImpl { struct RealTimeHandler::EventGuard { - template - EventGuard(RealTimeHandler &owner, M &m) : owner(owner) { + template EventGuard(RealTimeHandler &owner, M &m) : owner(owner) { bts_assert(owner.me->current_packet_sn); auto &info = owner.me->current_event; if (info.transact_time == MDP3TimeStamp{0}) { - // Starting a new event + // Starting a new event info.transact_time = MDP3TimeStamp{m.transactTime()}; info.first_packet_send_time = MDP3TimeStamp{owner.me->current_packet_send_time}; @@ -136,9 +135,7 @@ RealTimeHandler::move_all_instruments_and_reset() { return result; } -void RealTimeHandler::reset() { - me = std::make_unique (me->observer); -} +void RealTimeHandler::reset() { me = std::make_unique(me->observer); } void RealTimeHandler::on_begin_packet(MDP3PacketView const &h) { me->current_packet_sn = h.seq_num(); @@ -164,17 +161,28 @@ api::Side convert_side(::mdp3::MDEntryTypeBook::Value v) { } } - void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) { EventGuard ev(*this, m); - m.noMDEntries().forEach([&ev, this](auto &md_entry) { + struct MDEntryCache { + MDP3SecurityID id; + api::Side side; + }; + std::array cache; + size_t cache_idx = 0; + + m.noMDEntries().forEach([&ev, this, &cache, &cache_idx](auto &md_entry) { auto sid = MDP3SecurityID{md_entry.securityID()}; auto iptr = find_instrument(sid); if (!iptr) return; ev.record_security_change(iptr); auto side = convert_side(md_entry.mDEntryType()); + + cache[cache_idx].id = sid; + cache[cache_idx].side = side; + cache_idx++; + auto level = BookLevel{md::IntPrice{md_entry.mDEntryPx().mantissa(), md_entry.mDEntryPx().exponent()}, MDP3Quantity{md_entry.mDEntrySize()}, @@ -227,14 +235,15 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) { } }); - m.noOrderIDEntries().forEach([this, &m](auto &order_id_entry) { - auto &md_entry = m.noMDEntriesCount(order_id_entry.referenceID()); - auto instrument_id = md_entry.securityID(); - auto &iptr = me->instruments[MDP3SecurityID{instrument_id}]; + m.noOrderIDEntries().forEach([this, &cache](auto &order_id_entry) { + auto sid = cache[order_id_entry.referenceID()].id; + auto iptr = find_instrument(sid); + if (!iptr) + return; MDP3Order order; order.key = MDP3OrderKey{order_id_entry.orderID()}; - order.security = MDP3SecurityID{instrument_id}; - order.side = convert_side(md_entry.mDEntryType()); + order.security = sid; + order.side = cache[order_id_entry.referenceID()].side; order.priority = MDP3OrderPriority{order_id_entry.mDOrderPriority()}; order.quantity = MDP3Quantity{order_id_entry.mDDisplayQty()}; switch (order_id_entry.orderUpdateAction()) { @@ -308,8 +317,7 @@ void RealTimeHandler::operator()( }); } -void RealTimeHandler::operator()( - ::mdp3::MDIncrementalRefreshOrderBook47 &m) { +void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshOrderBook47 &m) { EventGuard ev(*this, m); m.noMDEntries().forEach([&ev, this](auto &md_entry) { @@ -446,8 +454,7 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshVolume37 &m) { }); } -template -void RealTimeHandler::on_instrument_definition(Def &d) { +template void RealTimeHandler::on_instrument_definition(Def &d) { InstrumentDefinition def{d}; EventGuard ev(*this, def); @@ -461,31 +468,26 @@ void RealTimeHandler::on_instrument_definition(Def &d) { iptr = new_snapshot.get(); me->instruments[sid] = std::move(new_snapshot); ev.record_security_change(iptr); - } - else { + } else { // We have not observed the instrument creation so it will be // someone else's responsability to build a snapshot for this // instrument and add it via add_snapshot() } - } - else { + } else { iptr->set_definition(d); ev.record_security_change(iptr); } } -void RealTimeHandler::operator()( - ::mdp3::MDInstrumentDefinitionFuture54 &d) { +void RealTimeHandler::operator()(::mdp3::MDInstrumentDefinitionFuture54 &d) { on_instrument_definition(d); } -void RealTimeHandler::operator()( - ::mdp3::MDInstrumentDefinitionOption55 &d) { +void RealTimeHandler::operator()(::mdp3::MDInstrumentDefinitionOption55 &d) { on_instrument_definition(d); } -void RealTimeHandler::operator()( - ::mdp3::MDInstrumentDefinitionSpread56 &d) { +void RealTimeHandler::operator()(::mdp3::MDInstrumentDefinitionSpread56 &d) { on_instrument_definition(d); } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp index 8deb6d858..ced51c39b 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp @@ -73,7 +73,7 @@ void BookSnapshot::operator()(::mdp3::SnapshotFullRefreshOrderBook53 &chunk) { case State::ACCUMULATING_BOOKS: { bts_assert(me->books.size()); auto &ibs = me->books.back(); - auto book_msg = ibs.book.unbox(); + auto book_msg = *(ibs.book); ibs.book_chunks.push_back(chunk); LOG(Spartan, Diag, "Recovered chunk {}/{} of book {}/{}", ibs.book_chunks.size(), chunk.noChunks(), @@ -97,13 +97,13 @@ bool BookSnapshot::is_complete() const { if (me->books.empty()) return false; - if (me->books[0].book.unbox().totNumReports() != me->books.size()) + if (me->books[0].book->totNumReports() != me->books.size()) return false; for (auto &book : me->books) { if (book.book_chunks.empty()) return false; - if (book.book_chunks.size() != book.book_chunks[0].unbox().noChunks()) + if (book.book_chunks.size() != book.book_chunks[0]->noChunks()) return false; } return true; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.h b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.h index 1edf850bf..ccf3a8e5a 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.h @@ -3,6 +3,7 @@ #include #include #include "../parsing/MDP3Parser.h" +#include "../parsing/MDP3MsgBox.h" #include "../common/mdp3_common.h" namespace mdp3 { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp index 91ed648e5..6df9bd6c5 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp @@ -12,14 +12,10 @@ struct RecoveryHandler::PImpl { PImpl(RecoveryHandler &owner) : owner(owner), - book_snapshot(), - definition_snapshot(), - incremental_queue(), + book_snapshot(), definition_snapshot(), incremental_queue(), - instrument_feed_handler(*this), - snapshot_feed_handler(*this), - incremental_feed_handler(*this) - {} + instrument_feed_handler(*this), snapshot_feed_handler(*this), + incremental_feed_handler(*this) {} RecoveryHandler &owner; @@ -60,7 +56,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot() { for (auto &d : definitions_ref) definition_ids.insert(d.security_id()); for (auto &b : books) - if (definition_ids.count(MDP3SecurityID{b.book.unbox().securityID()}) == 0) { + if (definition_ids.count(MDP3SecurityID{b.book->securityID()}) == 0) { // We are missing a definition for one of the books so we // need to start over and get a new round of definitions me->definition_snapshot.reset(); @@ -108,7 +104,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot() { // Index books so we can join them with the definitions std::unordered_map book_snaps_idx; for (auto &bs : book_snaps) - book_snaps_idx[MDP3SecurityID{bs.book.unbox().securityID()}] = &bs; + book_snaps_idx[MDP3SecurityID{bs.book->securityID()}] = &bs; // Now iterate over all definitions, locate the corresponding book, // if any, and use both to build an InstrumentSnapshot instance. @@ -118,12 +114,12 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot() { auto this_book_snap = book_snaps_idx[instrument_snapshot->security_id()]; uint32_t last_processed_sn = 0; if (this_book_snap) { - last_processed_sn = this_book_snap->book.unbox().lastMsgSeqNumProcessed(); - auto book_snap_msg = this_book_snap->book.unbox(); - update_book_in_snapshot(*instrument_snapshot,book_snap_msg); + last_processed_sn = this_book_snap->book->lastMsgSeqNumProcessed(); + auto book_snap_msg = *(this_book_snap->book); + update_book_in_snapshot(*instrument_snapshot, book_snap_msg); for (auto &bchunk : this_book_snap->book_chunks) { - auto chunk_msg = bchunk.unbox(); - update_book_chunk_in_snapshot(*instrument_snapshot,chunk_msg); + auto chunk_msg = *bchunk; + update_book_chunk_in_snapshot(*instrument_snapshot, chunk_msg); } } sortable_snapshots.push_back( @@ -144,16 +140,21 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot() { auto pending_packets = me->incremental_queue.get_queued_packets_and_reset(); auto current_snapshot = sortable_snapshots.begin(); for (auto &p : pending_packets) { - auto const &pv = p.view(); + auto pv = p.view(); + + // First add all the snapshots that are ready to see the next packet for (; current_snapshot != sortable_snapshots.end(); current_snapshot++) { auto &this_snapshot = *current_snapshot; if (this_snapshot.last_processed_sn == 0 || - this_snapshot.last_processed_sn + 1 == - pv.seq_num()) { + this_snapshot.last_processed_sn + 1 == pv.seq_num()) { rt_handler.add_snapshot(std::move(this_snapshot.snapshot)); } - parse_packet(pv, rt_handler); + else + break; } + + // Now parse the next packet + parse_packet(pv, rt_handler); } bts_assert(current_snapshot == sortable_snapshots.end()); @@ -161,7 +162,9 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot() { // have been shown the incremental messages that have occurred // since their books were recovered from the snapshot feed. We are // ready to go - return rt_handler.move_all_instruments_and_reset(); + auto result = rt_handler.move_all_instruments_and_reset(); + reset(); + return std::move(result); } // Instrument feed ------------------------------------------- @@ -172,7 +175,6 @@ void RecoveryHandler::InstrumentFeedHandler::on_begin_packet( void RecoveryHandler::InstrumentFeedHandler::on_end_packet() { me.definition_snapshot.on_end_packet(); - me.owner.try_build_snapshot(); } void RecoveryHandler::InstrumentFeedHandler::operator()( @@ -198,7 +200,6 @@ void RecoveryHandler::SnapshotFeedHandler::on_begin_packet( void RecoveryHandler::SnapshotFeedHandler::on_end_packet() { me.book_snapshot.on_end_packet(); - me.owner.try_build_snapshot(); } void RecoveryHandler::SnapshotFeedHandler::operator()( @@ -218,7 +219,6 @@ void RecoveryHandler::IncrementalFeedHandler::on_begin_packet( } void RecoveryHandler::IncrementalFeedHandler::on_end_packet() { me.incremental_queue.on_end_packet(); - me.owner.try_build_snapshot(); } void RecoveryHandler::IncrementalFeedHandler::operator()( ::mdp3::MDIncrementalRefreshBook46 &m) { @@ -295,7 +295,7 @@ void RecoveryHandler::update_book_in_snapshot( r.noMDEntries().forEach([&snap](auto &md_entry) { switch (md_entry.mDEntryType()) { case ::mdp3::MDEntryType::Bid: - case ::mdp3::MDEntryType::Offer: + case ::mdp3::MDEntryType::Offer: snap.update_book_level( convert_side(md_entry.mDEntryType()), md_entry.mDPriceLevel(), BookLevel{md::IntPrice{md_entry.mDEntryPx().mantissa(), @@ -304,7 +304,7 @@ void RecoveryHandler::update_book_in_snapshot( md_entry.numberOfOrders()}); break; case ::mdp3::MDEntryType::ImpliedBid: - case ::mdp3::MDEntryType::ImpliedOffer: + case ::mdp3::MDEntryType::ImpliedOffer: snap.update_implied_book_level( convert_side(md_entry.mDEntryType()), md_entry.mDPriceLevel(), BookLevel{md::IntPrice{md_entry.mDEntryPx().mantissa(), diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp index f47fff667..9d5960245 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp @@ -174,6 +174,8 @@ TEST_CASE("spartan_recovery", "[spartan_recovery]") { std::array buffers; RecoveryHandler recovery; AB_Arbitrageur ab; + RecoveryHandler::MaybeSnapshot maybe_snapshot; + SECTION("can_build_snapshot") { reader.on_packet() = [&](auto const &info, auto const &payload) { @@ -203,12 +205,14 @@ TEST_CASE("spartan_recovery", "[spartan_recovery]") { REQUIRE(false); break; } - return true; + maybe_snapshot = recovery.try_build_snapshot(); + return !maybe_snapshot; }; while (reader.read_next()) ; - REQUIRE(recovery.try_build_snapshot()); + REQUIRE(maybe_snapshot); + REQUIRE(maybe_snapshot.value().size()); } }