From 6a4e06873b60735a00e0d8ea8ae91aab0c6c83cb Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 1 Sep 2020 11:02:17 -0400 Subject: [PATCH] Added diagnostics, fixed order priority insertion --- src/lib/logging/Sink.cpp | 8 ++++++- src/server/sparkd/cmds/Spartan.cpp | 9 +++++++- .../back_testing/FifoPriceLevelMatcher.cpp | 2 +- .../mdp3/back_testing/FifoSecurityMatcher.cpp | 22 ++++++++++++++----- .../md/mdp3/common/InstrumentSnapshot.cpp | 7 +++--- .../cmds/spartan/md/mdp3/common/mdp3_common.h | 7 ++++-- .../md/mdp3/handler/ChannelHandler.cpp | 4 ++-- .../spartan/md/mdp3/recovery/BookSnapshot.cpp | 6 ++--- .../md/mdp3/recovery/DefinitionSnapshot.cpp | 6 ++--- .../md/mdp3/recovery/IncrementalQueue.cpp | 2 +- .../md/mdp3/recovery/RecoveryHandler.cpp | 6 ++--- .../spartan/md/mdp3/scripts/download_pcaps.sh | 7 ++++++ tools/docker_tools/run_rdm | 7 +++--- 13 files changed, 64 insertions(+), 29 deletions(-) create mode 100755 src/server/sparkd/cmds/spartan/md/mdp3/scripts/download_pcaps.sh diff --git a/src/lib/logging/Sink.cpp b/src/lib/logging/Sink.cpp index d95c96f37..46f739b5c 100644 --- a/src/lib/logging/Sink.cpp +++ b/src/lib/logging/Sink.cpp @@ -57,7 +57,13 @@ auto formatter(LoggingConfig const &cfg) { Sink console_sink(LoggingConfig const &) { return [](LogMsg const &m) { - std::cout << fmt::format("{:<8} | {:<5} | {}", m.subsystem_name, m.level_name,m.msg) + std::cout << fmt::format( + "{:<8}|{:<5}|{:<35}| {}", m.subsystem_name, m.level_name, + fmt::format( + "{}:{}", + boost::filesystem::path(m.file).filename().string(), + m.line), + m.msg) << std::endl; }; } diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index 00a1b08ea..e6c7077e5 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -44,6 +44,7 @@ std::map const &)>> } void on_real_time_event(MDP3Event const &ev) override { + for (auto s : ev.affected_securities) { auto it = orders.find(s->security_id()); if (it == orders.end()) { @@ -72,7 +73,13 @@ std::map const &)>> } auto current_orders = s->orders(); add_iceberg_annotation(current_orders); - bts_assert(matcher.has_same_orders(current_orders)); + if (!matcher.has_same_orders(current_orders)) { + std::cout << "Failing on event:\n " << ev.event_id << std::endl; + std::cout << "Old Orders:\n " << std::endl << old_orders ; + std::cout << "Deduced:\n " << deduced << std::endl; + std::cout << "Expected:\n " << s->order_changes() << std::endl; + bts_assert(false); + } old_orders = current_orders; } else { it->second = s->orders(); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp index f3c4060b6..9e6935ae7 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp @@ -64,7 +64,7 @@ std::tuple if(o.is_active){ auto[o_rem, cross_rem] = o.cross(cross_qty.value()); cross_qty = MDP3Quantity(cross_rem); - LOG(OrderMan, Info, "FifoPriceLevelMatcher::_one_pass_cross_order " + LOG(OrderMan, Diag, "FifoPriceLevelMatcher::_one_pass_cross_order " "{} (new_qty:{}) crossed by {} (new_qty:{})", o.key,o_rem,cross_ord_key,cross_rem); if(cross_rem > 0) diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp index 9b9969e47..dc6c1c81e 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp @@ -88,8 +88,18 @@ FifoSecurityMatcher(snp.orders()){ FifoSecurityMatcher::FifoSecurityMatcher(Orders const&orders) : FifoSecurityMatcher() { + std::vector sorted_orders; + sorted_orders.reserve(orders.size()); for (const auto& kv : orders) - on_new_order(kv.second); + sorted_orders.emplace_back(kv.second); + + std::sort(sorted_orders.begin(), sorted_orders.end(), + [](const auto &o1, const auto &o2) { + return o1.priority < o2.priority; + }); + + for (auto const & o : sorted_orders) + on_new_order(o); } // accessors ////////////////////////////////////////////////////////////////// @@ -136,7 +146,7 @@ std::string FifoSecurityMatcher::print_top_book() const{ // order interactions ///////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// void FifoSecurityMatcher::on_new_order(const MDP3Order& ord){ - LOG(OrderMan, Info, "FifoSecurityMatcher::on_new_order " + LOG(OrderMan, Diag, "FifoSecurityMatcher::on_new_order " "key:{} side:{} qty:{} ib_qty:{} price:{} pre_book:{} ord_no:{}", ord.key, static_cast(ord.side), @@ -240,9 +250,11 @@ bool FifoSecurityMatcher::has_same_orders(const Orders& ord_map){ plevel->side()==exp_ord.side && ord->quantity == exp_ord.quantity; if(!match){ - auto pl = _find_price_level(exp_ord.side,exp_ord.price); - bts_assert(false); - bts_assert(pl != nullptr); + auto pl = _find_price_level(exp_ord.side,exp_ord.price); + (void) pl; + // debug_break(); + // bts_assert(false); + // bts_assert(pl != nullptr); return false; } } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp index 254001913..212a33695 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp @@ -405,8 +405,7 @@ OrderChanges InstrumentSnapshot::deduced_order_changes() const { } std::ostream &operator<<(std::ostream &out, InstrumentSnapshot const &snap) { - out << "Snapshot --------------------------------------\n" - << "Symbol: " << snap.definition().symbol() << "\n" + out << "Snapshot " << snap.definition().symbol() << " --------------------------------------\n" << "Security ID: " << snap.security_id().value() << "\n" << "Trading Status: " << snap.trading_status() << "\n" << "Seq Number: " << snap.last_processed_sn() << "\n" @@ -424,11 +423,11 @@ std::ostream &operator<<(std::ostream &out, InstrumentSnapshot const &snap) { auto orders = sort_orders(snap.orders(), api::SIDE_BUY); out << "Buy Orders -----------------------------------\n"; for (auto &order : orders) - out << order << "\n"; + out << " " << order << "\n"; orders = sort_orders(snap.orders(), api::SIDE_SELL); out << "Sell Orders ----------------------------------\n"; for (auto &order : orders) - out << order << "\n"; + out << " " << order << "\n"; out << "Fills ----------------------------------------\n"; for (auto &f : snap.fills()) out << f << "\n"; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h index 6f261c6dd..f28012193 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h @@ -7,8 +7,8 @@ #include "lib/utils/overloaded.h" -#include #include +#include #include namespace bts::spartan::mdp3 { @@ -37,7 +37,10 @@ class MDP3Event { }; inline std::ostream &operator<<(std::ostream &s, MDP3Event const &ev) { - s << " transact_time: " << ev.transact_time.value() + s << "Event : " << ev.event_id + << " ------------------------------------------------------------" + << std::endl + << " transact_time: " << ev.transact_time.value() << " packets: " << ev.first_packet_sn << " to " << ev.last_packet_sn; return s; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp index ff01e9c35..f63cd6731 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp @@ -111,7 +111,7 @@ PickedFeed ChannelHandler::on_incremental_feed(MDP3PacketView const &a, void ChannelHandler::start_recovery() { switch (me->state) { case State::RECOVERY: - LOG(Spartan, Warn, "Restarting channel recovery."); + LOG(Spartan, Diag, "Restarting channel recovery."); me->real_time_ab = {}; me->recovery_handler.reset(); break; @@ -133,7 +133,7 @@ void ChannelHandler::try_start_real_time() { bts_assert(me->state == State::RECOVERY); auto maybe_snaps = me->recovery_handler.try_build_snapshot_and_reset(); if (maybe_snaps) { - LOG(Spartan, Warn, "Recovery complete. Starting real time feed"); + LOG(Spartan, Diag, "Recovery complete. Starting real time feed"); me->state = State::REALTIME; std::vector snap_ptrs; for (auto &p : maybe_snaps.value()) diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp index 2b0b59ce6..0c3e7455c 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp @@ -23,7 +23,7 @@ void BookSnapshot::on_begin_packet(MDP3PacketView const &pv) { switch (me->state) { case State::WFIRST_PACKET: if (pv.seq_num() == 1) { - LOG(Spartan, Info, + LOG(Spartan, Diag, "Observed packet 1 in snapshot feed. Starting to accumulate books"); transition(State::ACCUMULATING_BOOKS); me->last_sn = 1; @@ -31,7 +31,7 @@ void BookSnapshot::on_begin_packet(MDP3PacketView const &pv) { break; case State::ACCUMULATING_BOOKS: if (me->last_sn + 1 != pv.seq_num()) { - LOG(Spartan, Warn, + LOG(Spartan, Diag, "Detected loss in snapshot feed. Expected {} but received {}. " "Restarting snapshot recovery", me->last_sn + 1, pv.seq_num()); @@ -146,7 +146,7 @@ std::string BookSnapshot::state_name(State st) { } void BookSnapshot::transition(State st) { - LOG(Spartan,Info,"Book snapshot transitioning from {} to {}", + LOG(Spartan,Diag,"Book snapshot transitioning from {} to {}", state_name(me->state), state_name(st)); me->state = st; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp index 7ff308db4..ca5e8463b 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp @@ -23,7 +23,7 @@ void DefinitionSnapshot::on_begin_packet(MDP3PacketView const &pv) { switch (me->state) { case State::WFIRST_PACKET: if (pv.seq_num() == 1) { - LOG(Spartan, Info, + LOG(Spartan, Diag, "Observed packet 1 in instrument feed. Starting to accumulate instruments"); transition(State::ACCUMULATING_DEFS); me->last_sn = 1; @@ -31,7 +31,7 @@ void DefinitionSnapshot::on_begin_packet(MDP3PacketView const &pv) { break; case State::ACCUMULATING_DEFS: if (me->last_sn + 1 != pv.seq_num()) { - LOG(Spartan, Warn, + LOG(Spartan, Diag, "Detected loss in instrument feed. Expected {} but received {}. " "Restarting snapshot recovery", me->last_sn + 1, pv.seq_num()); @@ -110,7 +110,7 @@ std::string DefinitionSnapshot::state_name(State st) { } void DefinitionSnapshot::transition(State st) { - LOG(Spartan,Info,"Instrument snapshot transitioning from {} to {}", + LOG(Spartan,Diag,"Instrument snapshot transitioning from {} to {}", state_name(me->state), state_name(st)); me->state = st; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp index 756c2fa34..fd2e2d998 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp @@ -188,7 +188,7 @@ std::string IncrementalQueue::state_name(State st) { } void IncrementalQueue::transition(State st) { - LOG(Spartan,Info,"Incremental queue transitioning from {} to {}", + LOG(Spartan,Diag,"Incremental queue transitioning from {} to {}", state_name(me->state), state_name(st)); me->state = st; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp index 5a9172d94..0c723ad91 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp @@ -59,7 +59,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { if (definition_ids.count(MDP3SecurityID{b.book->securityID()}) == 0) { // We are missing a definition for one of the books so we // need to start over and get a new round of definitions - LOG(Spartan, Warn, + LOG(Spartan, Diag, "Missed instrument definition for security {}. Recoverying " "instruments again", b.book->securityID()); @@ -76,7 +76,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { // Our queue is not guaranteed to contain all the events that // follow all the snapshots so we need to get a new round of // snapshots. - LOG(Spartan, Warn, + LOG(Spartan, Diag, "First packet in incremental queue ({}) is more recent than oldest " "last processed packet in snapshot ({}). Snapshot is too old, will " "try getting a new one.", @@ -92,7 +92,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { if (me->book_snapshot.newest_snapshot_seq_num() > me->incremental_queue.newest_packet_seq_num()) { // Accumulate a few more - LOG(Spartan, Warn, + LOG(Spartan, Diag, "Most recent packet in incremental queue ({}) is older than most " "recent last processed packet in snapshot ({}). Will acumulate a " "few more packets in incremental queue.", diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/scripts/download_pcaps.sh b/src/server/sparkd/cmds/spartan/md/mdp3/scripts/download_pcaps.sh new file mode 100755 index 000000000..a783015f5 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/scripts/download_pcaps.sh @@ -0,0 +1,7 @@ +#!/bin/bash +if [ "$#" -ne 1 ]; then + echo " download_pcaps.sh " + exit 1 +fi + +AWS_ACCESS_KEY_ID=AKIAIFIGOOWGBLEMO3PQ AWS_SECRET_ACCESS_KEY=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 AWS_REGION=us-east-1 aws s3 sync s3://spark-recordings $1 diff --git a/tools/docker_tools/run_rdm b/tools/docker_tools/run_rdm index ef767cc20..ff9513ecc 100755 --- a/tools/docker_tools/run_rdm +++ b/tools/docker_tools/run_rdm @@ -1,10 +1,11 @@ #!/bin/bash DIR="$(dirname "$(readlink -f "$0")")" +COMPILE_CMDS=$DIR/../../compile_commands.json -while [ true ]; do - sleep 10 - rc -J $DIR/../../compile_commands.json +inotifywait -q -m -e close_write --format %e $COMPILE_CMDS | +while read events; do + rc -J $COMPILE_CMDS done & while [ true ]; do \