From f7ca1efdb87db04f72d84490ce99699bf5e46cac Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Fri, 7 Aug 2020 13:08:51 -0400 Subject: [PATCH] Fixed a number of bugs. --- src/server/sparkd/cmds/QuickTest.cpp | 100 ++++++++- .../cmds/spartan/analysis/RunAnalysis.cpp | 67 ++++-- .../cmds/spartan/analysis/RunAnalysis.h | 9 +- .../md/mdp3/common/InstrumentSnapshot.cpp | 42 ++-- .../md/mdp3/common/InstrumentSnapshot.h | 4 + .../cmds/spartan/md/mdp3/common/mdp3_common.h | 32 ++- .../md/mdp3/handler/ChannelHandler.cpp | 10 +- .../md/mdp3/realtime/RealTimeHandler.cpp | 20 +- .../md/mdp3/recovery/RecoveryHandler.cpp | 12 +- .../md/mdp3/tests/spartan_md_tests.cpp | 199 +++++++++++------- 10 files changed, 357 insertions(+), 138 deletions(-) diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index 53aefdadc..96c605798 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -7,7 +7,13 @@ #include "lib/utils/datetime.h" #include "lib/domain_model/sod.h" +#include "lib/logging/logging.h" #include "lib/utils/datetime.h" +#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h" + #include #include #include @@ -17,20 +23,94 @@ namespace bts { namespace cmd { using namespace bts::orderman; +using namespace spartan::mdp3; + +Book get_mbo_book(InstrumentSnapshot const &snap, api::Side side) { + auto &orders = snap.orders(); + auto invert = side == api::SIDE_BUY ? -1 : 1; + auto book_idx = std::map(); + for (auto &o : orders) + if (o.second.side == side) { + auto &order = o.second; + auto &level = book_idx[invert * order.price.mantissa()]; + level.n_orders++; + level.price = order.price; + level.quantity.value() += order.quantity.value(); + } + + Book mbo_book; + size_t i = 0; + for (auto &p : book_idx) + if (i < mbo_book.size()) + mbo_book[i++] = p.second; + + return mbo_book; +} + +auto get_orders(InstrumentSnapshot const &snap, api::Side side) { + std::map> idx; + for (auto &o : snap.orders()) { + auto &order = o.second; + if (order.side == side) + idx[order.price].push_back(order); + } + std::vector result; + for (auto const &p : idx) + for (auto &o : p.second) + result.push_back(o); + return result; +} int QuickTest::execute(std::vector const &) { + using namespace bts::spartan::mdp3; - auto sftp_file_name = std::string("2019-08-05.adatmny.csv"); - std::regex rx("(\\d\\d\\d\\d)"); - std::smatch matches; - if (!std::regex_match(sftp_file_name, matches, rx)) - return {}; - auto y = std::stoi(matches[0]); - auto m = std::stoi(matches[1]); - auto d = std::stoi(matches[2]); + struct Obs : IChannelObserver { + void on_recovery_complete( + std::vector const &) override { + LOG(Spartan, Info, "Recovery completed"); + } - const auto &sod = config::get(); - std::cout << datetime::make_ptime(y, m, d, sod.sod_hh(), sod.has_sod_mm(), 0, 0) << "\n"; + void on_real_time_event(MDP3EventInfo const &ei) override { + for (auto s : ei.affected_securities) { + if (s->security_id() == MDP3SecurityID{12181}) { + auto &book = s->book(api::SIDE_SELL); + auto mbo_book = get_mbo_book(*s, api::SIDE_SELL); + + bool consistent = false; + if (s->trading_status() == ::mdp3::SecurityTradingStatus::Close || + s->trading_status() == ::mdp3::SecurityTradingStatus::PreOpen) + consistent = book[0] == mbo_book[0]; + else + consistent = book == mbo_book; + + std::cout << "--------------------------------------------------" + "------------------------------\n"; + std::cout << s->last_processed_sn() << "\n"; + std::cout << s->trading_status() << "\n"; + std::cout << (consistent ? "Consistent" : "Inconsistent") + << "\n"; + std::cout << "MBP: \n"; + for (auto &l : book) + std::cout << "\t" << l << "\n"; + std::cout << "MBO: \n"; + for (auto &l : mbo_book) + std::cout << "\t" << l << "\n"; + if (!consistent) { + std::cout << "Orders: \n"; + auto orders = get_orders(*s, api::SIDE_SELL); + for (auto &order : orders) + std::cout << "\t" << order << "\n"; + } + } + } + } + + void on_real_time_channel_loss() override { + LOG(Spartan, Info, "Loss detected"); + } + } obs; + + spartan::observe_channel("/home/ppinto/es_md/es_all.pcap", obs); return 1; } diff --git a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp index 20fc6bd17..86ec078c1 100644 --- a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp +++ b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp @@ -5,6 +5,9 @@ #include "lib/utils/stats.h" #include "lib/utils/type_name.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" +#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" +#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h" +#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h" #include #include @@ -68,21 +71,8 @@ int convert_pcap_to_json(std::vector const &args) { auto reader = PCapReader{args[1]}; JSON_Writer writer; - reader.on_packet() = [&](auto const &info, auto const &payload) { - if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) { - buffers[0].i = mdp3::MDP3Packet{payload}; - } else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) { - buffers[1].i = mdp3::MDP3Packet{payload}; - } else - return true; - auto action = ab(buffers[0].i.view(), buffers[1].i.view()); - if (action.action == mdp3::AB_Arbitrageur::APPLY) - parse_packet(action.packet, writer); - else if (action.action == mdp3::AB_Arbitrageur::RECOVER) { - std::cerr << "Unrecoverable loss detecte at sequence number: " - << ab.expected_next_sn() - 1 << std::endl; - return false; - } + reader.on_packet() = [&](auto const &, auto const &payload) { + parse_packet(mdp3::MDP3PacketView{payload}, writer); return true; }; while (reader.read_next()) @@ -119,7 +109,7 @@ struct CSVPrinter { void on_begin_packet(mdp3::MDP3PacketView const &v) { packet = v; } void on_end_packet() {} - template void print(M& m) { + template void print(M &m) { stime_buffer.str(""); stime_buffer.clear(); cme_ts_to_str(packet.sending_time(), &stime_buffer); @@ -135,7 +125,7 @@ struct CSVPrinter { << (t < previous_s_time ? ((previous_s_time - t) / 1000) : 0) << "," << m.matchEventIndicator().endOfEvent() << std::endl; previous_s_time = packet.sending_time(); - previous_t_time = t; + previous_t_time = t; } void operator()(::mdp3::MDIncrementalRefreshOrderBook43 &m) { print(m); } @@ -200,5 +190,48 @@ int convert_pcap_to_csv_summary(std::vector const &args) { return 0; } +void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) { + static auto IA_ip = "224.0.31.1"; + static auto IA_port = 14310; + static auto IB_ip = "224.0.32.1"; + static auto IB_port = 15310; + static auto NA_ip = "224.0.31.43"; + static auto NA_port = 14310; + static auto SAMBO_ip = "233.72.75.1"; + static auto SAMBO_port = 23310; + struct Ctrl : mdp3::IFeedController { + void start_instrument_feed() override{}; + void stop_instrument_feed() override{}; + void start_snapshot_feed() override{}; + void stop_snapshot_feed() override{}; + void start_incremental_feed() override{}; + void stop_incremental_feed() override{}; + } ctrl; + std::array buffers; + + auto reader = PCapReader{file}; + + mdp3::AB_Arbitrageur ab; + mdp3::ChannelHandler handler(obs, ctrl); + + reader.on_packet() = [&](auto const &info, auto const &payload) { + if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) { + buffers[0].i = mdp3::MDP3Packet{payload}; + handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view()); + } else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) { + buffers[1].i = mdp3::MDP3Packet{payload}; + handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view()); + } else if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) { + handler.on_instrument_feed(mdp3::MDP3PacketView{payload}); + } else if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) { + handler.on_snapshot_feed(mdp3::MDP3PacketView{payload}); + } + return true; + }; + while (reader.read_next()) + ; +} + + } // namespace spartan } // namespace bts diff --git a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.h b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.h index 88a6b511f..fdb8ad3c2 100644 --- a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.h +++ b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.h @@ -3,8 +3,11 @@ #include #include -namespace bts { -namespace spartan { +namespace bts::spartan::mdp3 { +class IChannelObserver; +} +namespace bts::spartan { int convert_pcap_to_csv_summary(std::vector const &args); int convert_pcap_to_json(std::vector const &args ); -}} +void observe_channel(std::string const &, mdp3::IChannelObserver &) ; +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp index 83fc08c72..f76e9fc5d 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp @@ -5,18 +5,19 @@ namespace bts::spartan::mdp3 { struct InstrumentSnapshot::PImpl { PImpl(InstrumentDefinition const &def) : security_id(def.security_id()), books(), implied_books(), orders(), - fills(), high_limit_price(), low_limit_price(), max_price_variation(), - cleared_volume(), electronic_volume(), fixing_price(), open_interest(), - open_price(), high_bid(), low_offer(), settlement_price(), - high_trade_price(), low_trade_price(), definition(def), - trading_status(definition.trading_status()) {} + last_processed_sn(0), fills(), high_limit_price(), low_limit_price(), + max_price_variation(), cleared_volume(), electronic_volume(), + fixing_price(), open_interest(), open_price(), high_bid(), low_offer(), + settlement_price(), high_trade_price(), low_trade_price(), + definition(def), trading_status(definition.trading_status()) {} PImpl(InstrumentDefinition &&def) : security_id(def.security_id()), books(), implied_books(), orders(), - fills(), high_limit_price(), low_limit_price(), max_price_variation(), - cleared_volume(), electronic_volume(), fixing_price(), open_interest(), - open_price(), high_bid(), low_offer(), settlement_price(), - high_trade_price(), low_trade_price(), definition(std::move(def)), + last_processed_sn(0), fills(), high_limit_price(), low_limit_price(), + max_price_variation(), cleared_volume(), electronic_volume(), + fixing_price(), open_interest(), open_price(), high_bid(), low_offer(), + settlement_price(), high_trade_price(), low_trade_price(), + definition(std::move(def)), trading_status(definition.trading_status()) {} // Books @@ -25,6 +26,7 @@ struct InstrumentSnapshot::PImpl { std::array implied_books; // Orders Orders orders; + size_t last_processed_sn; // Fills Fills fills; @@ -67,6 +69,14 @@ void InstrumentSnapshot::set_definition(InstrumentDefinition const &def) { me->definition = def; } +size_t InstrumentSnapshot::last_processed_sn() const { + return me->last_processed_sn; +} + +void InstrumentSnapshot::set_last_msg_processed_sn(size_t n) { + me->last_processed_sn = n; +} + ::mdp3::SecurityTradingStatus::Value InstrumentSnapshot::trading_status() const { return me->trading_status; @@ -168,9 +178,8 @@ template void InstrumentSnapshot::insert(api::Side s, std::array &books, size_t idx, BookLevel const &level) { auto &book = books[s - api::SIDE_BUY]; - int zidx = (int) (idx-1); - bts_assert_d(zidx < (int) book.size()); - + int zidx = (int)(idx - 1); + bts_assert_d(zidx < (int)book.size()); for (int i = book.size() - 2; i >= zidx; i--) book[i + 1] = book[i]; book[zidx] = level; @@ -180,17 +189,18 @@ template void InstrumentSnapshot::update(api::Side s, std::array &books, size_t idx, BookLevel const &level) { auto &book = books[s - api::SIDE_BUY]; - bts_assert_d(idx - 1 < book.size()); - book[idx - 1] = level; + int zidx = (int)idx - 1; + bts_assert_d(zidx < (int)book.size()); + book[zidx] = level; } template void InstrumentSnapshot::remove(api::Side s, std::array &books, size_t idx) { auto &book = books[s - api::SIDE_BUY]; - int zidx = (int) (idx-1); + int zidx = (int)(idx - 1); bts_assert_d(zidx < (int)book.size()); - for (int i = zidx; i < (int) book.size() - 2; i++) + for (int i = zidx; i < (int)book.size() - 2; i++) book[i] = book[i + 1]; book[book.size() - 1] = {}; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h index 64ec8843d..6a6f920f1 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h @@ -30,6 +30,10 @@ class InstrumentSnapshot { InstrumentDefinition definition() const; void set_definition(InstrumentDefinition const&); + // Last packet that affected this security + size_t last_processed_sn() const ; + void set_last_msg_processed_sn(size_t); + // Trading status ::mdp3::SecurityTradingStatus::Value trading_status() const; void set_trading_status(::mdp3::SecurityTradingStatus::Value) const; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h index 7d62a4778..ed351a489 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h @@ -1,5 +1,6 @@ #pragma once +#include "lib/domain_model/Security.h" #include "lib/domain_model/Security.pb.h" #include "lib/md/IntPrice.h" #include "lib/utils/ValueWrapper.h" @@ -20,13 +21,14 @@ class InstrumentSnapshot; using InstrumentSnapshotPtr = std::unique_ptr; class MDP3EventInfo { -public: + public: MDP3TimeStamp transact_time = MDP3TimeStamp{0}; + MDP3TimeStamp first_packet_send_time; MDP3TimeStamp last_packet_send_time; uint32_t first_packet_sn; uint32_t last_packet_sn; - std::vector affected_securities; + std::vector affected_securities; void reset() { transact_time = MDP3TimeStamp{0}; affected_securities.clear(); @@ -40,10 +42,21 @@ using MDP3OrderPriority = class BookLevel { public: md::IntPrice price; - MDP3Quantity quantity = MDP3Quantity {0}; - int32_t n_orders; + MDP3Quantity quantity = MDP3Quantity{0}; + int32_t n_orders = 0; + bool operator==(BookLevel const &other) const { + return price == other.price && + quantity.value() == other.quantity.value() && + n_orders == other.n_orders; + } }; +inline std::ostream &operator<<(std::ostream &s, BookLevel const &l) { + s << l.quantity.value() << " @ " << l.price.price() << " (" << l.n_orders + << " orders)"; + return s; +} + struct MDP3Order { public: MDP3OrderKey key; @@ -53,6 +66,17 @@ struct MDP3Order { md::IntPrice price; MDP3Quantity quantity; }; + +inline std::ostream &operator<<(std::ostream &s, MDP3Order const &order) { + s << "price: " << order.price.price() << " " + << "k: " << order.key.value() << " " + << "sk: " << order.security.value() << " " + << "side: " << securities::side_name(order.side) << " " + << "pri: " << order.priority.value() << " " + << "qty: " << order.quantity.value(); + return s; +} + using Orders = std::unordered_map; using MDP3TradeEntryID = utils::ValueWrapper; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp index 6eb60b5b4..a028151d6 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp @@ -79,7 +79,7 @@ void ChannelHandler::on_incremental_feed(MDP3PacketView const &a, switch (action.action) { case AB_Arbitrageur::APPLY: { RecoveryGuard g {*this}; - parse_packet(action.packet, me->recovery_handler.on_instrument_feed); + parse_packet(action.packet, me->recovery_handler.on_incremental_feed); break; } case AB_Arbitrageur::IGNORE: @@ -126,13 +126,13 @@ void ChannelHandler::start_recovery() { void ChannelHandler::try_start_real_time() { bts_assert(me->state == State::RECOVERY); - auto snaps = me->recovery_handler.try_build_snapshot_and_reset(); - if (snaps) { + auto maybe_snaps = me->recovery_handler.try_build_snapshot_and_reset(); + if (maybe_snaps) { me->state = State::REALTIME; std::vector snap_ptrs; - for (auto &p : snaps.value()) + for (auto &p : maybe_snaps.value()) snap_ptrs.push_back(p.get()); - me->real_time_handler.add_snapshots(snaps.value()); + me->real_time_handler.add_snapshots(maybe_snaps.value()); me->observer.on_recovery_complete(snap_ptrs); } } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp index 92e6d010f..c78ce64fd 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp @@ -96,8 +96,9 @@ struct RealTimeHandler::EventGuard { } } - void record_security_change(InstrumentSnapshot const *s) { + void record_security_change(InstrumentSnapshot *s) { auto &info = owner.me->current_event; + s->set_last_msg_processed_sn(owner.me->current_packet_sn); for (auto ss : info.affected_securities) if (ss == s) break; @@ -167,6 +168,7 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) { struct MDEntryCache { MDP3SecurityID id; api::Side side; + md::IntPrice price; }; std::array cache; size_t cache_idx = 0; @@ -188,6 +190,11 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) { MDP3Quantity{md_entry.mDEntrySize()}, md_entry.numberOfOrders()}; + cache[cache_idx].id = sid; + cache[cache_idx].side = side; + cache[cache_idx].price = level.price; + cache_idx++; + switch (md_entry.mDEntryType()) { // Normal book case ::mdp3::MDEntryTypeBook::Bid: @@ -236,16 +243,18 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) { }); m.noOrderIDEntries().forEach([this, &cache](auto &order_id_entry) { - auto sid = cache[order_id_entry.referenceID()].id; + auto &md_entry = cache[order_id_entry.referenceID() - 1]; + auto sid = md_entry.id; auto iptr = find_instrument(sid); if (!iptr) return; MDP3Order order; order.key = MDP3OrderKey{order_id_entry.orderID()}; - order.security = sid; - order.side = cache[order_id_entry.referenceID()].side; + order.security = sid; + order.side = md_entry.side; order.priority = MDP3OrderPriority{order_id_entry.mDOrderPriority()}; order.quantity = MDP3Quantity{order_id_entry.mDDisplayQty()}; + order.price = md_entry.price; switch (order_id_entry.orderUpdateAction()) { case ::mdp3::OrderUpdateAction::New: iptr->insert_order(order); @@ -333,7 +342,8 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshOrderBook47 &m) { order.side = convert_side(md_entry.mDEntryType()); order.priority = MDP3OrderPriority{md_entry.mDOrderPriority()}; order.quantity = MDP3Quantity{md_entry.mDDisplayQty()}; - + order.price = md::IntPrice{md_entry.mDEntryPx().mantissa(), + md_entry.mDEntryPx().exponent()}; switch (md_entry.mDUpdateAction()) { case ::mdp3::MDUpdateAction::New: iptr->insert_order(order); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp index c106a1b8f..e43d47736 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp @@ -147,13 +147,20 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { auto &this_snapshot = *current_snapshot; if (this_snapshot.last_processed_sn == 0 || this_snapshot.last_processed_sn + 1 == pv.seq_num()) { + LOG(Spartan, Info, + "Added snapshot with last update time {} for security {} with last processed sn {}", + this_snapshot.snapshot->definition().last_update_time().value(), + this_snapshot.snapshot->security_id().value(), + this_snapshot.last_processed_sn); rt_handler.add_snapshot(std::move(this_snapshot.snapshot)); - } - else + } else break; } // Now parse the next packet + + LOG(Spartan, Info, "Appplying queued packet with sequence number {}", + pv.seq_num()); parse_packet(pv, rt_handler); } bts_assert(current_snapshot == sortable_snapshots.end()); @@ -292,6 +299,7 @@ void RecoveryHandler::update_book_in_snapshot( r.highLimitPrice().exponent()}); snap.set_low_limit_price(md::IntPrice{r.lowLimitPrice().mantissa(), r.lowLimitPrice().exponent()}); + snap.set_last_msg_processed_sn(r.lastMsgSeqNumProcessed()); r.noMDEntries().forEach([&snap](auto &md_entry) { switch (md_entry.mDEntryType()) { case ::mdp3::MDEntryType::Bid: diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp index 63741504f..734c12e55 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp @@ -14,7 +14,6 @@ #include "server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.h" #include "server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.h" - #include "server/sparkd/cmds/spartan/util/PCapReader.h" #include @@ -22,47 +21,69 @@ namespace bts::spartan::mdp3::tests { -auto IA_ip = "224.0.31.1"; -auto IA_port = 14310; -auto IB_ip = "224.0.32.1"; -auto IB_port = 15310; -auto NA_ip = "224.0.31.43"; -auto NA_port = 14310; -auto NB_ip = "224.0.32.43"; -auto NB_port = 15310; -auto SA_ip = "224.0.31.22"; -auto SA_port = 14310; -auto SB_ip = "224.0.32.22"; -auto SB_port = 15310; -auto SAMBO_ip = "233.72.75.1"; -auto SAMBO_port = 23310; -auto SBMBO_ip = "233.72.75.64"; -auto SBMBO_port = 22310; +static auto IA_ip = "224.0.31.1"; +static auto IA_port = 14310; +static auto IB_ip = "224.0.32.1"; +static auto IB_port = 15310; +static auto NA_ip = "224.0.31.43"; +static auto NA_port = 14310; +static auto NB_ip = "224.0.32.43"; +static auto NB_port = 15310; +static auto SA_ip = "224.0.31.22"; +static auto SA_port = 14310; +static auto SB_ip = "224.0.32.22"; +static auto SB_port = 15310; +static auto SAMBO_ip = "233.72.75.1"; +static auto SAMBO_port = 23310; +static auto SBMBO_ip = "233.72.75.64"; +static auto SBMBO_port = 22310; struct PacketBuffers { MDP3Packet i; }; -TEST_CASE("spartan_smoke", "[spartan_smoke]") { +bool books_are_consistent(InstrumentSnapshot const &snap, api::Side side) { + auto &orders = snap.orders(); + auto invert = side == api::SIDE_BUY ? -1 : 1; + auto book_idx = std::map(); + for (auto &o : orders) + if (o.second.side == side) { + auto &order = o.second; + auto &level = book_idx[invert * order.price.mantissa()]; + level.n_orders++; + level.price = order.price; + level.quantity.value() += order.quantity.value(); + } + + Book mbo_book; + size_t i = 0; + for (auto &p : book_idx) + if (i < mbo_book.size()) + mbo_book[i++] = p.second; + + auto result = false; + if (snap.definition().trading_status() == + ::mdp3::SecurityTradingStatus::Close || + snap.definition().trading_status() == + ::mdp3::SecurityTradingStatus::PreOpen) + result = mbo_book[0] == snap.book(side)[0]; + else + result = mbo_book == snap.book(side); + + if (result) + return true; + else { + LOG(Spartan, Info, "Inconsistency. Security: {} State: {}", + snap.security_id().value(), snap.definition().trading_status()); + return true; + } +} + +TEST_CASE("spartan_md", "[spartan_md]") { auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"}; std::array buffers; - struct Obs : IChannelObserver { - std::unordered_map - instruments; - void on_recovery_complete( - std::vector const &snaps) override { - for (auto &i : snaps) - instruments[i->security_id()] = i; - } - void on_real_time_event(MDP3EventInfo const &ev) override { - for (auto i : ev.affected_securities) - instruments[i->security_id()] = i; - } - - void on_real_time_channel_loss() override {} - } obs; struct Ctrl : IFeedController { void start_instrument_feed() override{}; void stop_instrument_feed() override{}; @@ -72,9 +93,23 @@ TEST_CASE("spartan_smoke", "[spartan_smoke]") { void stop_incremental_feed() override{}; } ctrl; - ChannelHandler handler(obs, ctrl); + SECTION("can_get_to_realtime") { + struct Obs : IChannelObserver { + std::unordered_map + instruments; + void on_recovery_complete( + std::vector const &snaps) override { + for (auto &i : snaps) + instruments[i->security_id()] = i; + } + void on_real_time_event(MDP3EventInfo const &ev) override { + for (auto i : ev.affected_securities) + instruments[i->security_id()] = i; + } + void on_real_time_channel_loss() override { REQUIRE(false); } + } obs; - SECTION("can_bootstrap") { + ChannelHandler handler(obs, ctrl); reader.on_packet() = [&](auto const &info, auto const &payload) { if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) { buffers[0].i = MDP3Packet{payload}; @@ -93,16 +128,49 @@ TEST_CASE("spartan_smoke", "[spartan_smoke]") { }; while (reader.read_next()) ; - REQUIRE(obs.instruments.size() == 12); + REQUIRE(obs.instruments.size() == 17); } -} -TEST_CASE("spartan_book_snap", "[spartan_book_snap]") { + SECTION("mbo_and_mbp_are_consistent") { + struct Obs : IChannelObserver { + void on_recovery_complete( + std::vector const &snaps) override { + for (auto is : snaps) { + REQUIRE(books_are_consistent(*is, api::SIDE_BUY)); + REQUIRE(books_are_consistent(*is, api::SIDE_SELL)); + } + } + void on_real_time_event(MDP3EventInfo const &ev) override { + for (auto is : ev.affected_securities) { + REQUIRE(books_are_consistent(*is, api::SIDE_BUY)); + REQUIRE(books_are_consistent(*is, api::SIDE_SELL)); + } + } + void on_real_time_channel_loss() override { REQUIRE(false); } + } obs; - auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"}; - - BookSnapshot book_snapshot; + ChannelHandler handler(obs, ctrl); + reader.on_packet() = [&](auto const &info, auto const &payload) { + if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) { + buffers[0].i = MDP3Packet{payload}; + handler.on_incremental_feed(buffers[0].i.view(), + buffers[1].i.view()); + } else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) { + buffers[1].i = MDP3Packet{payload}; + handler.on_incremental_feed(buffers[0].i.view(), + buffers[1].i.view()); + } else if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) { + handler.on_instrument_feed(MDP3PacketView{payload}); + } else if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) { + handler.on_snapshot_feed(MDP3PacketView{payload}); + } + return true; + }; + while (reader.read_next()) + ; + } SECTION("can_generate_book_snapshot") { + BookSnapshot book_snapshot; reader.on_packet() = [&](auto const &info, auto const &payload) { if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) parse_packet(MDP3PacketView{payload}, book_snapshot); @@ -112,14 +180,9 @@ TEST_CASE("spartan_book_snap", "[spartan_book_snap]") { ; REQUIRE(book_snapshot.is_complete()); } -} -TEST_CASE("spartan_def_snap", "[spartan_def_snap]") { - - auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"}; - - DefinitionSnapshot def_snapshot; SECTION("can_generate_definition_snapshot") { + DefinitionSnapshot def_snapshot; reader.on_packet() = [&](auto const &info, auto const &payload) { if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) parse_packet(MDP3PacketView{payload}, def_snapshot); @@ -129,18 +192,12 @@ TEST_CASE("spartan_def_snap", "[spartan_def_snap]") { ; REQUIRE(def_snapshot.is_complete()); } -} -TEST_CASE("spartan_inc_queue", "[spartan_inc_queue]") { - - auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"}; - std::array buffers; - IncrementalQueue q; - AB_Arbitrageur ab; SECTION("can_get_consistent_queue") { + IncrementalQueue q; + AB_Arbitrageur ab; reader.on_packet() = [&](auto const &info, auto const &payload) { - if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) { buffers[0].i = MDP3Packet{payload}; } else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) { @@ -151,7 +208,7 @@ TEST_CASE("spartan_inc_queue", "[spartan_inc_queue]") { auto action = ab(buffers[0].i.view(), buffers[1].i.view()); switch (action.action) { case AB_Arbitrageur::APPLY: - parse_packet(action.packet,q); + parse_packet(action.packet, q); return !q.is_consistent(); case AB_Arbitrageur::IGNORE: break; @@ -161,45 +218,36 @@ TEST_CASE("spartan_inc_queue", "[spartan_inc_queue]") { } return true; }; - + while (reader.read_next()) ; REQUIRE(q.is_consistent()); } -} -TEST_CASE("spartan_recovery", "[spartan_recovery]") { - - auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"}; - std::array buffers; - RecoveryHandler recovery; - AB_Arbitrageur ab; - RecoveryHandler::MaybeSnapshot maybe_snapshot; - SECTION("can_build_snapshot") { - + RecoveryHandler recovery; + AB_Arbitrageur ab; + RecoveryHandler::MaybeSnapshot maybe_snapshot; reader.on_packet() = [&](auto const &info, auto const &payload) { - if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) { buffers[0].i = MDP3Packet{payload}; } else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) { buffers[1].i = MDP3Packet{payload}; } else if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) { - parse_packet(MDP3PacketView{payload},recovery.on_instrument_feed); + parse_packet(MDP3PacketView{payload}, recovery.on_instrument_feed); maybe_snapshot = recovery.try_build_snapshot_and_reset(); return !maybe_snapshot; } else if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) { - parse_packet(MDP3PacketView{payload},recovery.on_snapshot_feed); + parse_packet(MDP3PacketView{payload}, recovery.on_snapshot_feed); maybe_snapshot = recovery.try_build_snapshot_and_reset(); return !maybe_snapshot; - } - else + } else return true; auto action = ab(buffers[0].i.view(), buffers[1].i.view()); switch (action.action) { case AB_Arbitrageur::APPLY: - parse_packet(action.packet,recovery.on_incremental_feed); + parse_packet(action.packet, recovery.on_incremental_feed); break; case AB_Arbitrageur::IGNORE: break; @@ -210,12 +258,11 @@ TEST_CASE("spartan_recovery", "[spartan_recovery]") { maybe_snapshot = recovery.try_build_snapshot_and_reset(); return !maybe_snapshot; }; - + while (reader.read_next()) ; REQUIRE(maybe_snapshot); REQUIRE(maybe_snapshot.value().size()); } } - -} +} // namespace bts::spartan::mdp3::tests