diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index 29c2d103a..ca518c550 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -36,12 +36,9 @@ using namespace spartan; struct TimeWriter { mdp3::MDP3PacketView packet; void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; } - void on_end_packet() { - std::cout << packet.sending_time() << std::endl; - } + void on_end_packet() { std::cout << packet.sending_time() << std::endl; } - template void operator()(M const &) { - } + template void operator()(M const &) {} }; void print_times(std::string const &file) { struct PacketBuffers { @@ -58,9 +55,9 @@ void print_times(std::string const &file) { if ((info.dst_ip == feed_map.incremental[0].ip && info.dst_port == feed_map.incremental[0].port)) { TimeWriter tw; - parse_packet(MDP3PacketView {payload},tw); + parse_packet(MDP3PacketView{payload}, tw); } - + return true; }; while (reader.read_next()) @@ -74,39 +71,55 @@ int QuickTest::execute(std::vector const &args) { log::configure_logging(cfg); auto file = args[0]; - print_times(file); - return 0; - using namespace bts::spartan::mdp3; struct Obs : IChannelObserver { + + Orders previous_orders; + size_t event_id = 0; + void on_recovery_complete( std::vector const &secs) override { - - for (auto s : secs) { - std::cout << "Bootstrap -----------------------------------------" - "------------------------------" - << std::endl; - std::cout << "Symbol: " << s->definition().symbol() << std::endl - << "Status: " << s->trading_status() << std::endl - << "Orders:\n" - << s->orders(); - } + previous_orders.clear(); + + for (auto s : secs) + for (auto const &o : s->orders()) + previous_orders[o.first] = o.second; } void on_real_time_event(MDP3Event const &ev) override { + event_id++; for (auto s : ev.affected_securities) { - if (s->order_changes().size() || s->fills().size()) { - std::cout << "Event " << ev.transact_time - << " -----------------------------" << std::endl; - std::cout << "Symbol: " << s->definition().symbol() << std::endl - << "Status: " << s->trading_status() << std::endl - << "Seq Num: " << ev.first_packet_sn << " to " - << ev.last_packet_sn << std::endl - << "Orders:\n" - << s->order_changes() << "Fills:\n" - << s->fills(); + if (s->trading_status() == + ::mdp3::SecurityTradingStatus::ReadyToTrade) { + for (auto f : s->fills()) { + if (f.aggressor != MDP3Fill::THIS_BUY && + f.aggressor != MDP3Fill::THIS_SELL) { + auto it = previous_orders.find(f.key); + bts_assert(it != previous_orders.end()); + it->second.quantity.value() -= f.quantity.value(); + if (it->second.quantity.value() < 0) { + std::cout << " Hidden sizes in order. " + << " eid: " << event_id + << " Event: " << ev + << " Order: " << it->first + << " Fill tid: " << f.trade_entry_id + << std::endl; + } + } + } + } + for (auto const &ch : s->order_changes()) { + switch (ch.change) { + case OrderChange::added: + case OrderChange::updated: + previous_orders[ch.order.key] = ch.order; + break; + case OrderChange::removed: + previous_orders.erase(ch.order.key); + break; + } } } } diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index 2889c2d1f..d151a2078 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -2,15 +2,26 @@ #include "server/sparkd/cmds/spartan/algo/RunTAS.h" #include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" #include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/util/PCapReader.h" +#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" +#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h" +#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/PickedFeed.h" #include #include #include +#include namespace bts { namespace cmd { +int slim_pcap(std::string const &file, std::string const &out_dir) { + return !spartan::mdp3::slim_down_pcap(file, out_dir); +} + int export_tables(std::string const &file, std::string const &out_dir) { using namespace bts::spartan::mdp3; @@ -25,15 +36,12 @@ int export_tables(std::string const &file, std::string const &out_dir) { "first_st,last_st" << std::endl; - event_items - << "eid, t_time,sid,status,type," - "change,ok,side," - "pri,price,qty,tid,agg" - << std::endl; + event_items << "eid, t_time,sid,status,type," + "change,ok,side," + "pri,price,qty,tid,agg" + << std::endl; - securities - << "sid,symbol,group,asset,last_t" - << std::endl; + securities << "sid,symbol,group,asset,last_t" << std::endl; } std::ofstream events; @@ -155,8 +163,6 @@ int export_tables(std::string const &file, std::string const &out_dir) { << std::endl; - if (s->security_id().value() == 12181 && event_id == 1065) - std::cout << "There is" << std::endl; auto deduced_order = s->deduced_order_change(); if (deduced_order) { event_items << event_id << "," @@ -211,6 +217,8 @@ int Spartan::execute(std::vector const &args) { error = spartan::convert_pcap_to_json(args); else if (args[0] == "pcap_to_tables") error = export_tables(args[1], args[2]); + else if (args[0] == "slim_pcap") + error = slim_pcap(args[1],args[2]); } if (error) std::cout << help(); @@ -220,6 +228,7 @@ int Spartan::execute(std::vector const &args) { std::string Spartan::help() { return fmt::format("\tpcap_to_msgs pcap_file\n" "\tpcap_to_json pcap_file\n" + "\tslim_pcap in_pcap out_pcap\n" "\tpcap_to_tables pcap_file output_dir\n"); } diff --git a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp index ac6993f82..200dce8ad 100644 --- a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp +++ b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp @@ -25,24 +25,6 @@ struct PacketBuffers { mdp3::AB_Arbitrageur ab; -void cme_ts_to_str(uint64_t cme_ts, std::stringstream *out) { - auto one_billion = 1000000000; - time_t unix_ts = cme_ts / one_billion; - auto boost_ts = boost::posix_time::from_time_t(unix_ts); - auto year = boost_ts.date().year(); - auto month = boost_ts.date().month().as_number(); - auto day = boost_ts.date().day().as_number(); - int hours = boost_ts.time_of_day().hours(); - int minutes = boost_ts.time_of_day().minutes(); - int seconds = boost_ts.time_of_day().seconds(); - *out << year << "-" << std::setw(2) << std::setfill('0') << month << "-" - << std::setw(2) << std::setw(2) << std::setfill('0') << day << "." - << std::setw(2) << std::setw(2) << std::setfill('0') << hours << ":" - << std::setw(2) << std::setfill('0') << minutes << ":" << std::setw(2) - << std::setfill('0') << seconds << "." << std::setw(9) - << cme_ts - (unix_ts * one_billion); -} - struct JSON_Writer { std::stringstream json_buffer; @@ -112,10 +94,10 @@ struct CSVPrinter { template void print(M &m) { stime_buffer.str(""); stime_buffer.clear(); - cme_ts_to_str(packet.sending_time(), &stime_buffer); + mdp3::cme_ts_to_date_str(packet.sending_time(), &stime_buffer); ttime_buffer.str(""); ttime_buffer.clear(); - cme_ts_to_str(get_transact_time(m), &ttime_buffer); + mdp3::cme_ts_to_date_str(get_transact_time(m), &ttime_buffer); auto t = get_transact_time(m); std::cout << ttime_buffer.str() << "," << stime_buffer.str() << "," diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h index 2a36532ca..9eec981af 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h @@ -35,6 +35,13 @@ class MDP3Event { } }; +inline std::ostream &operator<<(std::ostream &s, MDP3Event const &ev) { + s << " transact_time: " << ev.transact_time.value() + << " packets: " << ev.first_packet_sn + << " to " << ev.last_packet_sn; + return s; +} + using MDP3Quantity = utils::ValueWrapper; using MDP3OrderPriority = utils::ValueWrapper; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h b/src/server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h index 2db7cd0bd..846d40720 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h @@ -1,5 +1,7 @@ #pragma once #include "../parsing/MDP3Parser.h" +#include "../feed_control/PickedFeed.h" + #include "lib/utils/bts_assert.h" namespace bts::spartan::mdp3 { @@ -17,6 +19,7 @@ class AB_Arbitrageur { struct Action { ActionCode action; + PickedFeed picked; MDP3PacketView const packet; }; @@ -28,13 +31,13 @@ class AB_Arbitrageur { auto action_a = try_one(a); if (action_a == ActionCode::APPLY) { _last_seqn = a.seq_num(); - return {action_a, a}; + return {action_a, PickedFeed::A, a}; } auto action_b = try_one(b); if (action_b == ActionCode::APPLY) { _last_seqn = b.seq_num(); - return {action_b, b}; + return {action_b, PickedFeed::B, b}; } // the first packet defines our starting sequence number, rather @@ -43,17 +46,17 @@ class AB_Arbitrageur { if (_last_seqn == 0) { if (!a.empty()) { _last_seqn = a.seq_num(); - return {ActionCode::APPLY, a}; + return {ActionCode::APPLY, PickedFeed::A, a}; } else { _last_seqn = a.seq_num(); - return {ActionCode::APPLY, a}; + return {ActionCode::APPLY, PickedFeed::A,a}; } } // Both feeds are either ahead or behind, if they are ahead we // return RECOVER, otherwise we return IGNORE. if (action_a == action_b) - return {action_a, a}; + return {action_a, PickedFeed::None, a}; // One of the feeds is ahead of us and the other is behind. Its // possible that we will recover eventually, that is the feed @@ -69,9 +72,9 @@ class AB_Arbitrageur { how_far_ahead = b.seq_num() - _last_seqn; if (how_far_ahead > 10) - return {ActionCode::RECOVER, a}; + return {ActionCode::RECOVER, PickedFeed::None, a}; else - return {ActionCode::IGNORE, a}; + return {ActionCode::IGNORE, PickedFeed::None, a}; } private: diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/feed_control/PickedFeed.h b/src/server/sparkd/cmds/spartan/md/mdp3/feed_control/PickedFeed.h new file mode 100644 index 000000000..2b89490ce --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/feed_control/PickedFeed.h @@ -0,0 +1,5 @@ +#pragma once + +namespace bts::spartan { +enum class PickedFeed { A, B, None }; +} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp index 0f973f3d2..ff01e9c35 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.cpp @@ -50,7 +50,7 @@ ChannelHandler::~ChannelHandler() {} void ChannelHandler::on_instrument_feed(MDP3PacketView const &pv) { switch (me->state) { case State::RECOVERY: { - RecoveryGuard g {*this}; + RecoveryGuard g{*this}; parse_packet(pv, me->recovery_handler.on_instrument_feed); break; } @@ -62,7 +62,7 @@ void ChannelHandler::on_instrument_feed(MDP3PacketView const &pv) { void ChannelHandler::on_snapshot_feed(MDP3PacketView const &pv) { switch (me->state) { case State::RECOVERY: { - RecoveryGuard g {*this}; + RecoveryGuard g{*this}; parse_packet(pv, me->recovery_handler.on_snapshot_feed); break; } @@ -71,16 +71,16 @@ void ChannelHandler::on_snapshot_feed(MDP3PacketView const &pv) { } } -void ChannelHandler::on_incremental_feed(MDP3PacketView const &a, - MDP3PacketView const &b) { +PickedFeed ChannelHandler::on_incremental_feed(MDP3PacketView const &a, + MDP3PacketView const &b) { + auto action = me->real_time_ab(a, b); + switch (me->state) { case State::RECOVERY: { - auto action = me->real_time_ab(a, b); switch (action.action) { case AB_Arbitrageur::APPLY: { - RecoveryGuard g {*this}; + RecoveryGuard g{*this}; parse_packet(action.packet, me->recovery_handler.on_incremental_feed); - break; } case AB_Arbitrageur::IGNORE: break; @@ -91,7 +91,6 @@ void ChannelHandler::on_incremental_feed(MDP3PacketView const &a, break; } case State::REALTIME: { - auto action = me->real_time_ab(a, b); switch (action.action) { case AB_Arbitrageur::APPLY: parse_packet(action.packet, me->real_time_handler); @@ -105,15 +104,21 @@ void ChannelHandler::on_incremental_feed(MDP3PacketView const &a, break; } } + + return action.picked; } void ChannelHandler::start_recovery() { switch (me->state) { case State::RECOVERY: + LOG(Spartan, Warn, "Restarting channel recovery."); me->real_time_ab = {}; me->recovery_handler.reset(); break; case State::REALTIME: + LOG(Spartan, Warn, + "Loss detected. Shutting down real-time handling and switching to " + "recovery feeds."); me->real_time_ab = {}; me->real_time_handler.reset(); me->feed_controller.start_instrument_feed(); @@ -128,15 +133,18 @@ void ChannelHandler::try_start_real_time() { bts_assert(me->state == State::RECOVERY); auto maybe_snaps = me->recovery_handler.try_build_snapshot_and_reset(); if (maybe_snaps) { + LOG(Spartan, Warn, "Recovery complete. Starting real time feed"); me->state = State::REALTIME; std::vector snap_ptrs; for (auto &p : maybe_snaps.value()) snap_ptrs.push_back(p.get()); me->real_time_handler.add_snapshots(maybe_snaps.value()); + me->feed_controller.stop_instrument_feed(); + me->feed_controller.stop_snapshot_feed(); me->observer.on_recovery_complete(snap_ptrs); for (auto s : snap_ptrs) { - const_cast(s)->clear_fills(); - const_cast(s)->clear_order_changes(); + const_cast(s)->clear_fills(); + const_cast(s)->clear_order_changes(); } } } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h index 31d2d8e75..6b3ce5977 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h @@ -1,8 +1,10 @@ #pragma once -#include #include -#include "server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h" +#include + +#include "../feed_control/PickedFeed.h" +#include "../parsing/MDP3Parser.h" namespace bts::spartan::mdp3 { @@ -11,15 +13,16 @@ class IFeedController; class MDP3PacketView; class ChannelHandler { -public: - ChannelHandler(IChannelObserver&, IFeedController &); + public: + ChannelHandler(IChannelObserver &, IFeedController &); ~ChannelHandler(); - void on_instrument_feed(MDP3PacketView const&); - void on_snapshot_feed(MDP3PacketView const&); - void on_incremental_feed(MDP3PacketView const&a, MDP3PacketView const&b = {} ); + void on_instrument_feed(MDP3PacketView const &); + void on_snapshot_feed(MDP3PacketView const &); + PickedFeed on_incremental_feed(MDP3PacketView const &a, + MDP3PacketView const &b = {}); -private: + private: struct RecoveryGuard; void start_recovery(); void try_start_real_time(); @@ -28,5 +31,4 @@ private: std::unique_ptr me; }; -} - +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp index ced51c39b..2b0b59ce6 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp @@ -56,7 +56,7 @@ void BookSnapshot::operator()(::mdp3::SnapshotFullRefresh52 &book) { me->newest_snapshot_sn = std::max(book.lastMsgSeqNumProcessed(), me->newest_snapshot_sn); me->books.emplace_back(ib); - LOG(Spartan, Info, "Recovered book {}/{} from snapshot feed", + LOG(Spartan, Diag, "Recovered book {}/{} from snapshot feed", me->books.size(), book.totNumReports()); check_completion(); break; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp index a24ec9c51..5ddfad55d 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp @@ -165,7 +165,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { auto &this_snapshot = *current_snapshot; if (this_snapshot.last_processed_sn == 0 || this_snapshot.last_processed_sn + 1 == pv.seq_num()) { - LOG(Spartan, Info, + LOG(Spartan, Diag, "Added snapshot with last update time {} for security {} with " "last processed sn {}", this_snapshot.snapshot->definition().last_update_time().value(), @@ -178,7 +178,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { // Now parse the next packet - LOG(Spartan, Info, "Appplying queued packet with sequence number {}", + LOG(Spartan, Diag, "Appplying queued packet with sequence number {}", pv.seq_num()); parse_packet(pv, rt_handler); last_applied_packet = pv.seq_num(); @@ -187,7 +187,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { for (; current_snapshot != sortable_snapshots.end(); current_snapshot++) { auto &this_snapshot = *current_snapshot; bts_assert(this_snapshot.last_processed_sn == last_applied_packet); - LOG(Spartan, Info, + LOG(Spartan, Diag, "Added snapshot with last update time {} for security {} with " "last processed sn {}", this_snapshot.snapshot->definition().last_update_time().value(), diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp index 4c47763ac..cd0e14eaa 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp @@ -1,96 +1,272 @@ #include "mdp3_utils.h" +#include "../../../util/PCapReader.h" +#include "../common/InstrumentSnapshot.h" +#include "../feed_control/AB_Arbitrageur.h" +#include "../feed_control/IFeedController.h" #include "../gen/MDIncrementalRefreshOrderBook47.h" #include "../gen/MDInstrumentDefinitionFuture54.h" #include "../gen/MDInstrumentDefinitionOption55.h" #include "../gen/MDInstrumentDefinitionSpread56.h" #include "../gen/SnapshotFullRefreshOrderBook53.h" +#include "../handler/ChannelHandler.h" +#include "../handler/IChannelObserver.h" #include "../parsing/MDP3Parser.h" -#include "server/sparkd/cmds/spartan/util/PCapReader.h" +#include "lib/utils/bts_chrono.h" #include "lib/utils/overloaded.h" +#include +#include +#include +#include +#include +#include + namespace bts::spartan::mdp3 { +struct Guesser { + FeedMap result; + PCapReader::PacketInfo current_packet; + + void on_begin_packet(MDP3PacketView const &) {} + void on_end_packet() {} + + void set_current_source(const PCapReader::PacketInfo &info) { + current_packet = info; + } + + template void operator()(M const &) { + if (result.incremental[0].port == 0) { + result.incremental[0].port = current_packet.dst_port; + result.incremental[0].ip = current_packet.dst_ip; + return; + } + if (result.incremental[0].port == current_packet.dst_port && + result.incremental[0].ip == current_packet.dst_ip) + return; + + if (result.incremental[1].port == 0) { + result.incremental[1].port = current_packet.dst_port; + result.incremental[1].ip = current_packet.dst_ip; + } + } + + void operator()(::mdp3::SnapshotFullRefresh52 const &) { + if (result.snapshot.port == 0) { + result.snapshot.port = current_packet.dst_port; + result.snapshot.ip = current_packet.dst_ip; + } + } + + void operator()(::mdp3::SnapshotFullRefreshOrderBook53 const &) { + if (result.snapshot.port == 0) { + result.snapshot.port = current_packet.dst_port; + result.snapshot.ip = current_packet.dst_ip; + } + } + + void operator()(::mdp3::MDInstrumentDefinitionFuture54 const &) { + on_definition(); + } + + void operator()(::mdp3::MDInstrumentDefinitionOption55 const &) { + on_definition(); + } + + void operator()(::mdp3::MDInstrumentDefinitionSpread56 const &) { + on_definition(); + } + + void on_definition() { + if (result.instrument.port) + return; + if (result.incremental[0].port == 0) + return; + if (result.incremental[1].port == 0) + return; + if (result.incremental[0].ip == current_packet.dst_ip && + result.incremental[0].port == current_packet.dst_port) + return; + if (result.incremental[1].ip == current_packet.dst_ip && + result.incremental[1].port == current_packet.dst_port) + return; + result.instrument.port = current_packet.dst_port; + result.instrument.ip = current_packet.dst_ip; + } +}; + FeedMap guess_feed_map(std::string const &pcap_path) { - - struct Guesser { - FeedMap result; - PCapReader::PacketInfo current_packet; - - void on_begin_packet(MDP3PacketView const&) {} - void on_end_packet() {} - - void set_current_source(const PCapReader::PacketInfo &info) { - current_packet = info; - } - - void operator()(::mdp3::MDIncrementalRefreshOrderBook47 const &) { - if (result.incremental[0].port == 0) { - result.incremental[0].port = current_packet.dst_port; - result.incremental[0].ip = current_packet.dst_ip; - return; - } - if (result.incremental[0].port == current_packet.dst_port && - result.incremental[0].ip == current_packet.dst_ip) - return; - - if (result.incremental[1].port == 0) { - result.incremental[1].port = current_packet.dst_port; - result.incremental[1].ip = current_packet.dst_ip; - } - } - - void operator()(::mdp3::SnapshotFullRefreshOrderBook53 const &) { - if (result.snapshot.port == 0) { - result.snapshot.port = current_packet.dst_port; - result.snapshot.ip = current_packet.dst_ip; - } - } - - void operator()(::mdp3::MDInstrumentDefinitionFuture54 const &) { - on_definition(); - } - - void operator()(::mdp3::MDInstrumentDefinitionOption55 const &) { - on_definition(); - } - - void operator()(::mdp3::MDInstrumentDefinitionSpread56 const &) { - on_definition(); - } - - void on_definition() { - if (result.instrument.port) - return; - if (result.incremental[0].port == 0) - return; - if (result.incremental[1].port == 0) - return; - if (result.incremental[0].ip == current_packet.dst_ip && - result.incremental[0].port == current_packet.dst_port) - return; - if (result.incremental[1].ip == current_packet.dst_ip && - result.incremental[1].port == current_packet.dst_port) - return; - result.instrument.port = current_packet.dst_port; - result.instrument.ip = current_packet.dst_ip; - } - } guesser; - + Guesser guesser; auto reader = PCapReader{pcap_path}; reader.on_packet() = [&](auto const &info, auto const &payload) { guesser.set_current_source(info); - parse_packet(MDP3PacketView {payload},guesser); - if (guesser.result) + parse_packet(MDP3PacketView{payload}, guesser); + if (guesser.result) return false; else return true; }; - + while (reader.read_next()) ; return guesser.result; } +std::ostream &operator<<(std::ostream &o, FeedMap const &fm) { + o << "IA:" << fm.incremental[0].ip << ":" << fm.incremental[0].port + << " IB:" << fm.incremental[1].ip << ":" << fm.incremental[1].port + << " N:" << fm.instrument.ip << ":" << fm.instrument.port + << " SMBO:" << fm.snapshot.ip << ":" << fm.snapshot.port; + return o; +} + +bool slim_down_pcap(std::string const &file, std::string const &out_file) { + using namespace bts::spartan; + using namespace bts::spartan::mdp3; + + struct Ctrl : IFeedController { + bool instrument_feed = false; + bool snapshot_feed = false; + bool incremental_feed = false; + + void start_instrument_feed() override { instrument_feed = true; } + void stop_instrument_feed() override { instrument_feed = false; } + void start_snapshot_feed() override { snapshot_feed = true; } + void stop_snapshot_feed() override { snapshot_feed = false; } + void start_incremental_feed() override { incremental_feed = true; } + void stop_incremental_feed() override { incremental_feed = false; } + } ctrl; + + struct Obs : IChannelObserver { + bool observed_loss = false; + bool recovery_complete = false; + + size_t previous_packet = 0; + size_t previous_send_ts = 0; + TimeStamp last_print; + + void on_recovery_complete( + std::vector const &) override { + recovery_complete = true; + } + + void on_real_time_event(MDP3Event const &ev) override { + if (ms_diff(Timer::now(), last_print) >= 1000) { + last_print = Timer::now(); + std::stringstream str; + cme_ts_to_time_str(ev.transact_time.value(), &str); + LOG(Spartan, Info, + "Time: {} -> {} pkt/sec at {}x real time", + str.str(), + (ev.first_packet_sn - previous_packet), + (ev.first_packet_send_time.value() - previous_send_ts) / 1000000000); + previous_packet = ev.first_packet_sn; + previous_send_ts = ev.first_packet_send_time.value(); + } + } + void on_real_time_channel_loss() override { observed_loss = true; } + } obs; + + struct IncrementalFeedBuffer { + bool written_once = false; + pcpp::RawPacket rp; + mdp3::MDP3Packet p; + }; + std::array inc_buffer; + + auto feed_map = mdp3::guess_feed_map(file); + bts_assert(feed_map); + auto reader = PCapReader{file}; + + mdp3::AB_Arbitrageur ab; + mdp3::ChannelHandler handler(obs, ctrl); + + pcpp::PcapNgFileWriterDevice writer(out_file.c_str()); + writer.open(); + reader.on_packet() = [&](auto const &info, auto const &payload) { + if ((info.dst_ip == feed_map.incremental[0].ip && + info.dst_port == feed_map.incremental[0].port)) { + inc_buffer[0].rp = *info.rp; + inc_buffer[0].p.set_bytes(payload); + } else if ((info.dst_ip == feed_map.incremental[1].ip && + info.dst_port == feed_map.incremental[1].port)) { + inc_buffer[1].rp = *info.rp; + inc_buffer[1].p.set_bytes(payload); + } else if (info.dst_ip == feed_map.instrument.ip && + info.dst_port == feed_map.instrument.port) { + if (ctrl.instrument_feed) { + handler.on_instrument_feed(mdp3::MDP3PacketView{payload}); + writer.writePacket(*info.rp); + } + return true; + } else if (info.dst_ip == feed_map.snapshot.ip && + info.dst_port == feed_map.snapshot.port) { + if (ctrl.snapshot_feed) { + handler.on_snapshot_feed(mdp3::MDP3PacketView{payload}); + writer.writePacket(*info.rp); + } + return true; + } else + return true; + + // Incremental feed handling + auto picked = handler.on_incremental_feed(inc_buffer[0].p.view(), + inc_buffer[1].p.view()); + if (picked == PickedFeed::A || + ((!inc_buffer[0].written_once || !obs.recovery_complete) && + !inc_buffer[0].p.view().empty())) { + inc_buffer[0].written_once = true; + writer.writePacket(inc_buffer[0].rp); + } + + if (picked == PickedFeed::B || + ((!inc_buffer[1].written_once || !obs.recovery_complete) && + !inc_buffer[1].p.view().empty())) { + inc_buffer[1].written_once = true; + writer.writePacket(inc_buffer[1].rp); + } + + return true; + }; + + while (reader.read_next()) + if (obs.observed_loss) { + LOG(Spartan, Error, "Observed loss in {}. Cannot slim it down", file); + return false; + }; + return true; +} + +void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out) { + auto one_billion = 1000000000; + time_t unix_ts = cme_ts / one_billion; + auto boost_ts = boost::posix_time::from_time_t(unix_ts); + int hours = boost_ts.time_of_day().hours(); + int minutes = boost_ts.time_of_day().minutes(); + int seconds = boost_ts.time_of_day().seconds(); + *out << std::setw(2) << std::setfill('0') << hours << ":" << std::setw(2) + << std::setfill('0') << minutes << ":" << std::setw(2) + << std::setfill('0') << seconds << "." << std::setw(9) + << cme_ts - (unix_ts * one_billion); +} + +void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out) { + auto one_billion = 1000000000; + time_t unix_ts = cme_ts / one_billion; + auto boost_ts = boost::posix_time::from_time_t(unix_ts); + auto year = boost_ts.date().year(); + auto month = boost_ts.date().month().as_number(); + auto day = boost_ts.date().day().as_number(); + int hours = boost_ts.time_of_day().hours(); + int minutes = boost_ts.time_of_day().minutes(); + int seconds = boost_ts.time_of_day().seconds(); + *out << year << "-" << std::setw(2) << std::setfill('0') << month << "-" + << std::setw(2) << std::setw(2) << std::setfill('0') << day << "." + << std::setw(2) << std::setw(2) << std::setfill('0') << hours << ":" + << std::setw(2) << std::setfill('0') << minutes << ":" << std::setw(2) + << std::setfill('0') << seconds << "." << std::setw(9) + << cme_ts - (unix_ts * one_billion); +} + } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h index 4ba99175e..3888a81ca 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h @@ -23,15 +23,12 @@ struct FeedMap { snapshot.port; } }; - -inline std::ostream &operator<<(std::ostream &o, FeedMap const &fm) { - o << "IA:" << fm.incremental[0].ip << ":" << fm.incremental[0].port - << " IB:" << fm.incremental[1].ip << ":" << fm.incremental[1].port - << " N:" << fm.instrument.ip << ":" << fm.instrument.port - << " SMBO:" << fm.snapshot.ip << ":" << fm.snapshot.port; - return o; -} +std::ostream &operator<<(std::ostream &o, FeedMap const &fm); FeedMap guess_feed_map(std::string const &pcap_path); +bool slim_down_pcap(std::string const&in, std::string const &out); +void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out); +void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out); + } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/util/PCapReader.cpp b/src/server/sparkd/cmds/spartan/util/PCapReader.cpp index 2fea3af3c..a9bd43d76 100644 --- a/src/server/sparkd/cmds/spartan/util/PCapReader.cpp +++ b/src/server/sparkd/cmds/spartan/util/PCapReader.cpp @@ -1,11 +1,11 @@ #include "PCapReader.h" #include "lib/utils/bts_assert.h" -#include -#include -#include #include +#include #include +#include +#include #include namespace bts { @@ -14,10 +14,10 @@ namespace spartan { struct PCapReader::PImpl { PImpl(std::string const &file_path) : reader(pcpp::IFileReaderDevice::getReader(file_path.c_str())), - on_packet([](auto const&, auto const &) {return true;}) {} + on_packet([](auto const &, auto const &) { return true; }) {} std::unique_ptr reader; - std::function on_packet; + OnPacket on_packet; pcpp::RawPacket rawPacket; }; @@ -31,19 +31,18 @@ PCapReader::PCapReader(std::string const &file_path, std::string const &filter) PCapReader::~PCapReader() {} -std::function &PCapReader::on_packet() { - return me->on_packet; -} +PCapReader::OnPacket &PCapReader::on_packet() { return me->on_packet; } bool PCapReader::read_next() { if (me->reader->getNextPacket(me->rawPacket)) { pcpp::Packet parsedPacket(&me->rawPacket); PacketInfo info; - + info.rp = &me->rawPacket; + auto udp = parsedPacket.getLayerOfType(); if (udp) info.dst_port = udp->getUdpHeader()->portDst; - else { + else { auto tcp = parsedPacket.getLayerOfType(); info.dst_port = tcp->getTcpHeader()->portDst; } @@ -52,12 +51,13 @@ bool PCapReader::read_next() { info.dst_ip = ip->getDstIpAddress().toString(); auto payload = parsedPacket.getLayerOfType(pcpp::GenericPayload); if (payload) - return me->on_packet(info, {(const char *)payload->getData(), payload->getDataLen()}); + return me->on_packet( + info, {(const char *)payload->getData(), payload->getDataLen()}); else return true; } else return false; } -} // namespace utils +} // namespace spartan } // namespace bts diff --git a/src/server/sparkd/cmds/spartan/util/PCapReader.h b/src/server/sparkd/cmds/spartan/util/PCapReader.h index 7cbf04948..9fe6835fb 100644 --- a/src/server/sparkd/cmds/spartan/util/PCapReader.h +++ b/src/server/sparkd/cmds/spartan/util/PCapReader.h @@ -1,26 +1,32 @@ #pragma once -#include #include +#include +namespace pcpp { +class RawPacket; +} -namespace bts { -namespace spartan { - +namespace bts::spartan { class PCapReader { -public: - PCapReader(std::string const&file_path, - std::string const&filter = ""); + public: + PCapReader(std::string const &file_path, std::string const &filter = ""); ~PCapReader(); bool read_next(); + struct PacketInfo { std::string dst_ip; int dst_port; + pcpp::RawPacket *rp; }; - std::function &on_packet(); -private: + + using OnPacket = + std::function; + OnPacket &on_packet(); + + private: struct PImpl; std::unique_ptr me; }; -}} +} // namespace bts::spartan