From c52c24c3f2d813d4d54ebf923705f6da7fef2c7d Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Sun, 9 Aug 2020 17:06:53 -0400 Subject: [PATCH] Fixed some more issues. Everything looks good now. --- src/server/sparkd/cmds/QuickTest.cpp | 70 ++++++++++++++- .../cmds/spartan/analysis/RunAnalysis.cpp | 10 --- .../cmds/spartan/analysis/RunAnalysis.h | 1 + .../md/mdp3/common/InstrumentDefinition.h | 86 +++++++++++-------- .../md/mdp3/common/InstrumentSnapshot.cpp | 72 ++++++++++++---- .../md/mdp3/common/InstrumentSnapshot.h | 8 +- .../cmds/spartan/md/mdp3/common/mdp3_common.h | 31 +++++-- .../md/mdp3/handler/ChannelHandler.cpp | 4 + .../md/mdp3/handler/IChannelObserver.h | 2 +- .../cmds/spartan/md/mdp3/parsing/MDP3Parser.h | 2 +- .../md/mdp3/realtime/IRealTimeObserver.h | 4 +- .../md/mdp3/realtime/RealTimeHandler.cpp | 12 ++- .../md/mdp3/recovery/RecoveryHandler.cpp | 19 +++- .../md/mdp3/tests/spartan_md_tests.cpp | 69 +++++++-------- 14 files changed, 270 insertions(+), 120 deletions(-) diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index 68beb3050..e364ca239 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -12,8 +12,13 @@ #include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" +#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" +#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h" #include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h" +#include "server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h" #include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/util/PCapReader.h" #include #include @@ -26,13 +31,74 @@ namespace cmd { using namespace bts::orderman; using namespace spartan::mdp3; -int QuickTest::execute(std::vector const &) { +void dump_snapshots() { using namespace bts::spartan::mdp3; + using namespace bts::spartan; + + auto file = "/home/ppinto/es_md/es_all.pcap"; + auto feed_map = guess_feed_map(file); + bts_assert(feed_map); + auto reader = PCapReader{file}; + + struct JSON_Writer { + mdp3::MDP3PacketView packet; + void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; } + void on_end_packet() {} + + void operator()(::mdp3::SnapshotFullRefresh52 const &m) { + if (m.securityID() == 15234) + std::cout << "{\"SeqNum\": " << packet.seq_num() << ", " + << " \"SendTime\": " << packet.sending_time() << ", " + << " \"Size\": " << packet.size() << "," + << " \"Msg\": " << m << "}\n"; + } + }; + + JSON_Writer writer; + reader.on_packet() = [&](auto const &, auto const &payload) { + parse_packet(mdp3::MDP3PacketView{payload}, writer); + return true; + }; + while (reader.read_next()) + ; +} + +int QuickTest::execute(std::vector const &) { + //dump_snapshots(); + + using namespace bts::spartan::mdp3; + + struct Obs : IChannelObserver { + void on_recovery_complete( + std::vector const &snaps) override { + std::cout << "Initial Snapshot\n"; + for (auto is : snaps) { + if (is->security_id() == MDP3SecurityID{12181}) + std::cout << *is << "\n"; + } + } + + void on_real_time_event(MDP3Event const &ev) override { + for (auto is : ev.affected_securities) { + if (is->security_id() == MDP3SecurityID{12181}) { + std::cout << "Seq numbers: " << ev.first_packet_sn << " to " + << ev.last_packet_sn << "\n"; + std::cout << *is << "\n"; + } + } + } + void on_real_time_channel_loss() override { + std::cout << "Observed loss, resetting\n"; + } + } obs; + + spartan::observe_channel("/home/ppinto/es_md/es_all.pcap", obs); return 0; } std::string QuickTest::help() { - return "This is a place holder to make it easy to quickly write discardable " + return "This is a place holder to make it easy to quickly write " + "discardable " "test code"; } } // namespace cmd diff --git a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp index 35ad6dc9f..533fd61cd 100644 --- a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp +++ b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp @@ -21,20 +21,10 @@ namespace spartan { struct PacketBuffers { mdp3::MDP3Packet i; mdp3::MDP3Packet n; - mdp3::MDP3Packet smbo; }; mdp3::AB_Arbitrageur ab; -auto IA_ip = "224.0.31.1"; -auto IA_port = 14310; -auto IB_ip = "224.0.32.1"; -auto IB_port = 15310; -auto SAMBO_ip = "233.72.75.1"; -auto SAMBO_port = 23310; -auto SBMBO_ip = "233.72.75.64"; -auto SBMBO_port = 22310; - void cme_ts_to_str(uint64_t cme_ts, std::stringstream *out) { auto one_billion = 1000000000; time_t unix_ts = cme_ts / one_billion; diff --git a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.h b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.h index fdb8ad3c2..ef34cafeb 100644 --- a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.h +++ b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.h @@ -9,5 +9,6 @@ class IChannelObserver; namespace bts::spartan { int convert_pcap_to_csv_summary(std::vector const &args); int convert_pcap_to_json(std::vector const &args ); + void observe_channel(std::string const &, mdp3::IChannelObserver &) ; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h index 96d0b2ce4..6472575e1 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h @@ -1,10 +1,10 @@ #pragma once -#include "mdp3_common.h" -#include "../parsing/MDP3MsgBox.h" #include "../gen/MDInstrumentDefinitionFuture54.h" #include "../gen/MDInstrumentDefinitionOption55.h" #include "../gen/MDInstrumentDefinitionSpread56.h" +#include "../parsing/MDP3MsgBox.h" +#include "mdp3_common.h" namespace bts::spartan::mdp3 { @@ -19,52 +19,67 @@ class InstrumentDefinition { InstrumentDefinition &operator=(InstrumentDefinition const &other) = default; template - InstrumentDefinition(M const &d) - : _def (MDP3MsgBox (d)) - {} + InstrumentDefinition(M const &d) : _def(MDP3MsgBox(d)) {} MDP3TimeStamp last_update_time() const { return std::visit( - overloaded{[](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> const &m) { - return MDP3TimeStamp{m->lastUpdateTime()}; - }, - [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> const &m) { - return MDP3TimeStamp{m->lastUpdateTime()}; - }, - [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> const &m) { - return MDP3TimeStamp{m->lastUpdateTime()}; - }}, + overloaded{ + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> const &m) { + return MDP3TimeStamp{m->lastUpdateTime()}; + }, + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> const &m) { + return MDP3TimeStamp{m->lastUpdateTime()}; + }, + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> const &m) { + return MDP3TimeStamp{m->lastUpdateTime()}; + }}, _def); } size_t tot_num_reports() const { return std::visit( - overloaded{[](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> const &m) { - return m->totNumReports(); - }, - [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> const &m) { - return m->totNumReports(); - }, - [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> const &m) { - return m->totNumReports(); - }}, + overloaded{ + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> const &m) { + return m->totNumReports(); + }, + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> const &m) { + return m->totNumReports(); + }, + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> const &m) { + return m->totNumReports(); + }}, _def); } MDP3SecurityID security_id() const { return std::visit( - overloaded{[](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> const &m) { - return MDP3SecurityID{m->securityID()}; - }, - [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> const &m) { - return MDP3SecurityID{m->securityID()}; - }, - [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> const &m) { - return MDP3SecurityID{m->securityID()}; - }}, + overloaded{ + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> const &m) { + return MDP3SecurityID{m->securityID()}; + }, + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> const &m) { + return MDP3SecurityID{m->securityID()}; + }, + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> const &m) { + return MDP3SecurityID{m->securityID()}; + }}, _def); } + const char *symbol() const { + return std::visit( + overloaded{ + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> const &m) { + return m->symbol(); + }, + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55> const &m) { + return m->symbol(); + }, + [](MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56> const &m) { + return m->symbol(); + }}, + _def); + } ::mdp3::MatchEventIndicator match_event_indicator() const { return std::visit( overloaded{[](MDP3MsgBox<::mdp3::MDInstrumentDefinitionFuture54> &m) { @@ -122,9 +137,10 @@ class InstrumentDefinition { } private: - using DefVar = std::variant, - MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55>, - MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56>>; + using DefVar = + std::variant, + MDP3MsgBox<::mdp3::MDInstrumentDefinitionOption55>, + MDP3MsgBox<::mdp3::MDInstrumentDefinitionSpread56>>; mutable DefVar _def; }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp index b7625cc9f..dc55158de 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp @@ -5,19 +5,20 @@ namespace bts::spartan::mdp3 { struct InstrumentSnapshot::PImpl { PImpl(InstrumentDefinition const &def) : security_id(def.security_id()), books(), implied_books(), orders(), - last_processed_sn(0), fills(), high_limit_price(), low_limit_price(), - max_price_variation(), cleared_volume(), electronic_volume(), - fixing_price(), open_interest(), open_price(), high_bid(), low_offer(), - settlement_price(), high_trade_price(), low_trade_price(), - definition(def), trading_status(definition.trading_status()) {} + order_changes(), last_processed_sn(0), fills(), high_limit_price(), + low_limit_price(), max_price_variation(), cleared_volume(), + electronic_volume(), fixing_price(), open_interest(), open_price(), + high_bid(), low_offer(), settlement_price(), high_trade_price(), + low_trade_price(), definition(def), + trading_status(definition.trading_status()) {} PImpl(InstrumentDefinition &&def) : security_id(def.security_id()), books(), implied_books(), orders(), - last_processed_sn(0), fills(), high_limit_price(), low_limit_price(), - max_price_variation(), cleared_volume(), electronic_volume(), - fixing_price(), open_interest(), open_price(), high_bid(), low_offer(), - settlement_price(), high_trade_price(), low_trade_price(), - definition(std::move(def)), + order_changes(), last_processed_sn(0), fills(), high_limit_price(), + low_limit_price(), max_price_variation(), cleared_volume(), + electronic_volume(), fixing_price(), open_interest(), open_price(), + high_bid(), low_offer(), settlement_price(), high_trade_price(), + low_trade_price(), definition(std::move(def)), trading_status(definition.trading_status()) {} // Books @@ -26,7 +27,11 @@ struct InstrumentSnapshot::PImpl { std::array implied_books; // Orders Orders orders; + OrderChanges order_changes; + + // Last packet that affected this security size_t last_processed_sn; + // Fills Fills fills; @@ -235,20 +240,55 @@ void InstrumentSnapshot::remove_implied_book_level(api::Side s, size_t idx) { Orders const &InstrumentSnapshot::orders() const { return me->orders; } +OrderChanges const &InstrumentSnapshot::order_changes() const { + return me->order_changes; +} + +void InstrumentSnapshot::clear_order_changes() { me->order_changes.clear(); } + void InstrumentSnapshot::insert_order(MDP3Order const &order) { bts_assert_d(me->orders.count(order.key) == 0); me->orders.insert({order.key, order}); + me->order_changes.emplace_back(OrderChange{order.key, OrderChange::added}); } void InstrumentSnapshot::update_order(MDP3Order const &order) { auto it = me->orders.find(order.key); bts_assert_d(it != me->orders.end()); it->second = order; + me->order_changes.emplace_back(OrderChange{order.key, OrderChange::updated}); } void InstrumentSnapshot::remove_order(MDP3OrderKey id) { bts_assert_d(me->orders.count(id)); me->orders.erase(id); + me->order_changes.emplace_back(OrderChange{id, OrderChange::removed}); +} + +Book InstrumentSnapshot::build_book_from_orders (api::Side side) const +{ + auto invert = side == api::SIDE_BUY ? -1 : 1; + auto book_idx = std::map(); + for (auto &o : orders()) + if (o.second.side == side) { + auto &order = o.second; + auto &level = book_idx[invert * order.price.mantissa()]; + level.n_orders++; + level.price = order.price; + level.quantity.value() += order.quantity.value(); + } + + Book mbo_book; + size_t i = 0; + for (auto &p : book_idx) + if (i < mbo_book.size()) + mbo_book[i++] = p.second; + return mbo_book; +} + +void InstrumentSnapshot::set_books_from_orders() { + me->books[0] = build_book_from_orders(api::SIDE_BUY); + me->books[1] = build_book_from_orders(api::SIDE_SELL); } void InstrumentSnapshot::reset_books() { @@ -287,9 +327,7 @@ void InstrumentSnapshot::remove_fill(MDP3TradeEntryID tid, MDP3OrderKey oid) { me->fills.erase(it); } -void InstrumentSnapshot::clear_trades() { - me->fills.clear(); -} +void InstrumentSnapshot::clear_fills() { me->fills.clear(); } auto get_orders(InstrumentSnapshot const &snap, api::Side side) { std::vector result; @@ -312,8 +350,9 @@ auto get_orders(InstrumentSnapshot const &snap, api::Side side) { } std::ostream &operator<<(std::ostream &out, InstrumentSnapshot const &snap) { - out << "Snapshot --------------------------------------\n"; - out << "Security ID: " << snap.security_id().value() << "\n" + out << "Snapshot --------------------------------------\n" + << "Symbol: " << snap.definition().symbol() << "\n" + << "Security ID: " << snap.security_id().value() << "\n" << "Trading Status: " << snap.trading_status() << "\n" << "Seq Number: " << snap.last_processed_sn() << "\n" << "MBP Bid ---------------------------------------\n" @@ -324,6 +363,9 @@ std::ostream &operator<<(std::ostream &out, InstrumentSnapshot const &snap) { << snap.implied_book(api::SIDE_BUY) << "Implied Ask -----------------------------------\n" << snap.implied_book(api::SIDE_SELL); + out << "Order Changes -----------------------------------\n"; + for (auto &change : snap.order_changes()) + out << change << "\n"; auto orders = get_orders(snap, api::SIDE_BUY); out << "Buy Orders -----------------------------------\n"; for (auto &order : orders) diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h index f83472f44..32c04b5b9 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h @@ -52,10 +52,14 @@ class InstrumentSnapshot { // Orders Orders const &orders() const; + OrderChanges const &order_changes() const; + void clear_order_changes() ; void insert_order(MDP3Order const&); void update_order(MDP3Order const&); void remove_order(MDP3OrderKey); - + Book build_book_from_orders (api::Side ) const; + void set_books_from_orders() ; + // Reset books in response to a channel reset message void reset_books(); @@ -64,7 +68,7 @@ class InstrumentSnapshot { void add_fill(MDP3Fill const&); void update_fill(MDP3Fill const&); void remove_fill(MDP3TradeEntryID, MDP3OrderKey ); - void clear_trades(); + void clear_fills(); // Stats md::IntPrice high_limit_price() const; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h index 830672c02..e1601e023 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h @@ -20,7 +20,7 @@ using MDP3SecurityID = class InstrumentSnapshot; using InstrumentSnapshotPtr = std::unique_ptr; -class MDP3EventInfo { +class MDP3Event { public: MDP3TimeStamp transact_time = MDP3TimeStamp{0}; @@ -52,12 +52,11 @@ class BookLevel { }; inline std::ostream &operator<<(std::ostream &s, BookLevel const &l) { - s << l.quantity.value() << " @ " << l.price.price() << " (" << l.n_orders + s << " " << l.quantity.value() << " @ " << l.price.price() << " (" << l.n_orders << " orders)"; return s; } - struct MDP3Order { public: MDP3OrderKey key; @@ -68,8 +67,24 @@ struct MDP3Order { MDP3Quantity quantity; }; +struct OrderChange { + MDP3OrderKey key; + enum ChangeType { added, removed, updated } change; +}; + +inline std::ostream &operator<<(std::ostream &s, OrderChange const &ch) { + s << " ok: " << ch.key.value() << " " + << (ch.change == OrderChange::added + ? "create" + : (ch.change == OrderChange::updated ? "update" : "delete")); + + return s; +} + +using OrderChanges = std::vector; + inline std::ostream &operator<<(std::ostream &s, MDP3Order const &order) { - s << "price: " << order.price.price() << " " + s << " price: " << order.price.price() << " " << "k: " << order.key.value() << " " << "sk: " << order.security.value() << " " << "side: " << securities::side_name(order.side) << " " @@ -89,11 +104,11 @@ struct MDP3Fill { enum Aggressor { THIS, OTHER, IMPLIED } aggressor; }; inline std::ostream &operator<<(std::ostream &s, MDP3Fill const &fill) { - s << "ok: " << fill.key.value() << " " + s << " ok: " << fill.key.value() << " " << "price: " << fill.price.price() << " " << "qty: " << fill.quantity.value() << " " << "tid: " << fill.trade_entry_id.value() << " " - << "agg: " << (int) fill.aggressor; + << "agg: " << (int)fill.aggressor; return s; } @@ -104,14 +119,14 @@ const auto MDP3_MAX_IMPLIED_BOOK_DEPTH = 2; using Book = std::array; inline std::ostream &operator<<(std::ostream &s, Book const &b) { - for (auto const&l : b) + for (auto const &l : b) s << l << "\n"; return s; } using ImpliedBook = std::array; inline std::ostream &operator<<(std::ostream &s, ImpliedBook const &b) { - for (auto const&l : b) + for (auto const &l : b) s << l << "\n"; return s; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp index a028151d6..0f973f3d2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp @@ -134,6 +134,10 @@ void ChannelHandler::try_start_real_time() { snap_ptrs.push_back(p.get()); me->real_time_handler.add_snapshots(maybe_snaps.value()); me->observer.on_recovery_complete(snap_ptrs); + for (auto s : snap_ptrs) { + const_cast(s)->clear_fills(); + const_cast(s)->clear_order_changes(); + } } } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h b/src/server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h index b783a1c7f..0a099b825 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h @@ -9,7 +9,7 @@ class IChannelObserver : public IRecoveryObserver, public IRealTimeObserver { public: virtual void on_recovery_complete(std::vector const &) = 0; - virtual void on_real_time_event(MDP3EventInfo const &) = 0; + virtual void on_real_time_event(MDP3Event const &) = 0; virtual void on_real_time_channel_loss() = 0; }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h b/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h index 0d0fa733d..4cffafea3 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h @@ -188,7 +188,7 @@ using AllMDTypes = meta::list< template class ParserImpl { public: static_assert( - sizeof...(Msg) > 1, + sizeof...(Msg) > 0, "The handler used for this parser does not handle any message types"); ParserImpl(Handler &h) : handler(h) {} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/IRealTimeObserver.h b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/IRealTimeObserver.h index e370115ea..c7104dded 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/IRealTimeObserver.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/IRealTimeObserver.h @@ -2,11 +2,11 @@ namespace bts::spartan::mdp3 { -class MDP3EventInfo; +class MDP3Event; class IRealTimeObserver { public: - virtual void on_real_time_event(MDP3EventInfo const&) = 0; + virtual void on_real_time_event(MDP3Event const&) = 0; virtual void on_real_time_channel_loss() = 0; }; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp index ae9a08c5a..e448b7d54 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp @@ -34,7 +34,7 @@ struct RealTimeHandler::PImpl { MDP3TimeStamp current_packet_send_time; uint32_t current_packet_sn; - MDP3EventInfo current_event; + MDP3Event current_event; IRealTimeObserver &observer; std::vector pending_ts; std::vector pending_order_detail; @@ -51,6 +51,7 @@ struct RealTimeHandler::EventGuard { info.first_packet_send_time = MDP3TimeStamp{owner.me->current_packet_send_time}; info.first_packet_sn = owner.me->current_packet_sn; + info.last_packet_send_time = MDP3TimeStamp{0}; info.last_packet_sn = 0; } @@ -101,7 +102,7 @@ struct RealTimeHandler::EventGuard { s->set_last_msg_processed_sn(owner.me->current_packet_sn); for (auto ss : info.affected_securities) if (ss == s) - break; + return; info.affected_securities.push_back(s); } @@ -586,8 +587,11 @@ void RealTimeHandler::on_end_of_event() { } } me->observer.on_real_time_event(me->current_event); - for (auto s : me->current_event.affected_securities) - const_cast(s)->clear_trades(); + + for (auto s : me->current_event.affected_securities) { + const_cast(s)->clear_fills(); + const_cast(s)->clear_order_changes(); + } } InstrumentSnapshot *RealTimeHandler::find_instrument(MDP3SecurityID id) { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp index 39f99bbc5..87309a9ad 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp @@ -59,6 +59,10 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { if (definition_ids.count(MDP3SecurityID{b.book->securityID()}) == 0) { // We are missing a definition for one of the books so we // need to start over and get a new round of definitions + LOG(Spartan, Warn, + "Missed instrument definition for security {}. Recoverying " + "instruments again", + b.book->securityID()); me->definition_snapshot.reset(); return {}; } @@ -72,6 +76,12 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { // Our queue is not guaranteed to contain all the events that // follow all the snapshots so we need to get a new round of // snapshots. + LOG(Spartan, Warn, + "First packet in incremental queue ({}) is more recent than oldest " + "last processed packet in snapshot ({}). Snapshot is too old, will " + "try getting a new one.", + me->incremental_queue.oldest_packet_seq_num(), + me->book_snapshot.oldest_snapshot_seq_num()); me->book_snapshot.reset(); return {}; } @@ -82,6 +92,12 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { if (me->book_snapshot.newest_snapshot_seq_num() >= me->incremental_queue.newest_packet_seq_num()) { // Accumulate a few more + LOG(Spartan, Warn, + "Most recent packet in incremental queue ({}) is older than most " + "recent last processed packet in snapshot ({}). Will accumulate a " + "few more packets in incremental queue", + me->incremental_queue.newest_packet_seq_num(), + me->book_snapshot.newest_snapshot_seq_num()); return {}; } @@ -121,6 +137,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { auto chunk_msg = *bchunk; update_book_chunk_in_snapshot(*instrument_snapshot, chunk_msg); } + instrument_snapshot->set_books_from_orders(); } sortable_snapshots.push_back( {std::move(instrument_snapshot), last_processed_sn}); @@ -133,7 +150,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { // that we only add an instrument when the next event in the queue // has not yet been seen by the snapshot. struct NullHandler : IRealTimeObserver { - void on_real_time_event(MDP3EventInfo const &) {} + void on_real_time_event(MDP3Event const &) {} void on_real_time_channel_loss() {} } null_handler; RealTimeHandler rt_handler(null_handler); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp index 94ce02a25..aabb7e0be 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp @@ -29,40 +29,9 @@ struct PacketBuffers { MDP3Packet i; }; -bool books_are_consistent(InstrumentSnapshot const &snap, api::Side side) { - if (snap.definition().trading_status() == - ::mdp3::SecurityTradingStatus::Close || - snap.definition().trading_status() == - ::mdp3::SecurityTradingStatus::PreOpen) - return true; - - auto &orders = snap.orders(); - auto invert = side == api::SIDE_BUY ? -1 : 1; - auto book_idx = std::map(); - for (auto &o : orders) - if (o.second.side == side) { - auto &order = o.second; - auto &level = book_idx[invert * order.price.mantissa()]; - level.n_orders++; - level.price = order.price; - level.quantity.value() += order.quantity.value(); - } - - Book mbo_book; - size_t i = 0; - for (auto &p : book_idx) - if (i < mbo_book.size()) - mbo_book[i++] = p.second; - - return mbo_book == snap.book(side); -} - TEST_CASE("spartan_md", "[spartan_md]") { + //auto file = "/home/ppinto/es_md/es_all.pcap"; auto file = os::sparkd_wd() + "/data_files/small_es.pcap"; - // auto cmd = - // fmt::format("xz -d {} -c > {}", compressed_file, decompressed.path()); - // bts_assert(std::system(cmd.c_str()) == 0); - auto feed_map = guess_feed_map(file); auto reader = PCapReader{file}; std::array buffers; @@ -86,7 +55,7 @@ TEST_CASE("spartan_md", "[spartan_md]") { for (auto &i : snaps) instruments[i->security_id()] = i; } - void on_real_time_event(MDP3EventInfo const &ev) override { + void on_real_time_event(MDP3Event const &ev) override { real_time = true; for (auto i : ev.affected_securities) instruments[i->security_id()] = i; @@ -115,7 +84,7 @@ TEST_CASE("spartan_md", "[spartan_md]") { } return true; }; - while (reader.read_next()) + while (reader.read_next()) ; REQUIRE(obs.instruments.size() == 17); } @@ -125,14 +94,36 @@ TEST_CASE("spartan_md", "[spartan_md]") { void on_recovery_complete( std::vector const &snaps) override { for (auto is : snaps) { - REQUIRE(books_are_consistent(*is, api::SIDE_BUY)); - REQUIRE(books_are_consistent(*is, api::SIDE_SELL)); + if (is->trading_status() == + ::mdp3::SecurityTradingStatus::ReadyToTrade) { + auto buy_ok = is->build_book_from_orders(api::SIDE_BUY) == + is->book(api::SIDE_BUY); + if (!buy_ok) + std::cout << "shit"; + REQUIRE(buy_ok); + auto sell_ok = is->build_book_from_orders(api::SIDE_SELL) == + is->book(api::SIDE_SELL); + if (!sell_ok) + std::cout << "shit"; + REQUIRE(sell_ok); + } } } - void on_real_time_event(MDP3EventInfo const &ev) override { + void on_real_time_event(MDP3Event const &ev) override { for (auto is : ev.affected_securities) { - REQUIRE(books_are_consistent(*is, api::SIDE_BUY)); - REQUIRE(books_are_consistent(*is, api::SIDE_SELL)); + if (is->trading_status() == + ::mdp3::SecurityTradingStatus::ReadyToTrade) { + auto buy_ok = is->build_book_from_orders(api::SIDE_BUY) == + is->book(api::SIDE_BUY); + if (!buy_ok) + std::cout << "shit"; + REQUIRE(buy_ok); + auto sell_ok = is->build_book_from_orders(api::SIDE_SELL) == + is->book(api::SIDE_SELL); + if (!sell_ok) + std::cout << "shit"; + REQUIRE(sell_ok); + } } } void on_real_time_channel_loss() override { REQUIRE(false); }