Added new spartan command to clean up pcaps to just what we need.
This commit is contained in:
1 parent
86ca092f18
commit
5c5bc243d2
14 files changed
+398
-190
No files matched your search
@@ -36,12 +36,9 @@ using namespace spartan;
|
||||
struct TimeWriter {
|
||||
mdp3::MDP3PacketView packet;
|
||||
void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; }
|
||||
void on_end_packet() {
|
||||
std::cout << packet.sending_time() << std::endl;
|
||||
}
|
||||
void on_end_packet() { std::cout << packet.sending_time() << std::endl; }
|
||||
|
||||
template <class M> void operator()(M const &) {
|
||||
}
|
||||
template <class M> void operator()(M const &) {}
|
||||
};
|
||||
void print_times(std::string const &file) {
|
||||
struct PacketBuffers {
|
||||
@@ -58,9 +55,9 @@ void print_times(std::string const &file) {
|
||||
if ((info.dst_ip == feed_map.incremental[0].ip &&
|
||||
info.dst_port == feed_map.incremental[0].port)) {
|
||||
TimeWriter tw;
|
||||
parse_packet(MDP3PacketView {payload},tw);
|
||||
parse_packet(MDP3PacketView{payload}, tw);
|
||||
}
|
||||
|
||||
|
||||
return true;
|
||||
};
|
||||
while (reader.read_next())
|
||||
@@ -74,39 +71,55 @@ int QuickTest::execute(std::vector<std::string> const &args) {
|
||||
log::configure_logging(cfg);
|
||||
|
||||
auto file = args[0];
|
||||
print_times(file);
|
||||
return 0;
|
||||
|
||||
using namespace bts::spartan::mdp3;
|
||||
|
||||
struct Obs : IChannelObserver {
|
||||
|
||||
Orders previous_orders;
|
||||
size_t event_id = 0;
|
||||
|
||||
void on_recovery_complete(
|
||||
std::vector<InstrumentSnapshot const *> const &secs) override {
|
||||
|
||||
for (auto s : secs) {
|
||||
std::cout << "Bootstrap -----------------------------------------"
|
||||
"------------------------------"
|
||||
<< std::endl;
|
||||
std::cout << "Symbol: " << s->definition().symbol() << std::endl
|
||||
<< "Status: " << s->trading_status() << std::endl
|
||||
<< "Orders:\n"
|
||||
<< s->orders();
|
||||
}
|
||||
previous_orders.clear();
|
||||
|
||||
for (auto s : secs)
|
||||
for (auto const &o : s->orders())
|
||||
previous_orders[o.first] = o.second;
|
||||
}
|
||||
|
||||
void on_real_time_event(MDP3Event const &ev) override {
|
||||
event_id++;
|
||||
|
||||
for (auto s : ev.affected_securities) {
|
||||
if (s->order_changes().size() || s->fills().size()) {
|
||||
std::cout << "Event " << ev.transact_time
|
||||
<< " -----------------------------" << std::endl;
|
||||
std::cout << "Symbol: " << s->definition().symbol() << std::endl
|
||||
<< "Status: " << s->trading_status() << std::endl
|
||||
<< "Seq Num: " << ev.first_packet_sn << " to "
|
||||
<< ev.last_packet_sn << std::endl
|
||||
<< "Orders:\n"
|
||||
<< s->order_changes() << "Fills:\n"
|
||||
<< s->fills();
|
||||
if (s->trading_status() ==
|
||||
::mdp3::SecurityTradingStatus::ReadyToTrade) {
|
||||
for (auto f : s->fills()) {
|
||||
if (f.aggressor != MDP3Fill::THIS_BUY &&
|
||||
f.aggressor != MDP3Fill::THIS_SELL) {
|
||||
auto it = previous_orders.find(f.key);
|
||||
bts_assert(it != previous_orders.end());
|
||||
it->second.quantity.value() -= f.quantity.value();
|
||||
if (it->second.quantity.value() < 0) {
|
||||
std::cout << " Hidden sizes in order. "
|
||||
<< " eid: " << event_id
|
||||
<< " Event: " << ev
|
||||
<< " Order: " << it->first
|
||||
<< " Fill tid: " << f.trade_entry_id
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (auto const &ch : s->order_changes()) {
|
||||
switch (ch.change) {
|
||||
case OrderChange::added:
|
||||
case OrderChange::updated:
|
||||
previous_orders[ch.order.key] = ch.order;
|
||||
break;
|
||||
case OrderChange::removed:
|
||||
previous_orders.erase(ch.order.key);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,15 +2,26 @@
|
||||
#include "server/sparkd/cmds/spartan/algo/RunTAS.h"
|
||||
#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
|
||||
#include "server/sparkd/cmds/spartan/util/PCapReader.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/PickedFeed.h"
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <Pcap++/PcapFileDevice.h>
|
||||
|
||||
namespace bts {
|
||||
namespace cmd {
|
||||
|
||||
int slim_pcap(std::string const &file, std::string const &out_dir) {
|
||||
return !spartan::mdp3::slim_down_pcap(file, out_dir);
|
||||
}
|
||||
|
||||
int export_tables(std::string const &file, std::string const &out_dir) {
|
||||
using namespace bts::spartan::mdp3;
|
||||
|
||||
@@ -25,15 +36,12 @@ int export_tables(std::string const &file, std::string const &out_dir) {
|
||||
"first_st,last_st"
|
||||
<< std::endl;
|
||||
|
||||
event_items
|
||||
<< "eid, t_time,sid,status,type,"
|
||||
"change,ok,side,"
|
||||
"pri,price,qty,tid,agg"
|
||||
<< std::endl;
|
||||
event_items << "eid, t_time,sid,status,type,"
|
||||
"change,ok,side,"
|
||||
"pri,price,qty,tid,agg"
|
||||
<< std::endl;
|
||||
|
||||
securities
|
||||
<< "sid,symbol,group,asset,last_t"
|
||||
<< std::endl;
|
||||
securities << "sid,symbol,group,asset,last_t" << std::endl;
|
||||
}
|
||||
|
||||
std::ofstream events;
|
||||
@@ -155,8 +163,6 @@ int export_tables(std::string const &file, std::string const &out_dir) {
|
||||
|
||||
<< std::endl;
|
||||
|
||||
if (s->security_id().value() == 12181 && event_id == 1065)
|
||||
std::cout << "There is" << std::endl;
|
||||
auto deduced_order = s->deduced_order_change();
|
||||
if (deduced_order) {
|
||||
event_items << event_id << ","
|
||||
@@ -211,6 +217,8 @@ int Spartan::execute(std::vector<std::string> const &args) {
|
||||
error = spartan::convert_pcap_to_json(args);
|
||||
else if (args[0] == "pcap_to_tables")
|
||||
error = export_tables(args[1], args[2]);
|
||||
else if (args[0] == "slim_pcap")
|
||||
error = slim_pcap(args[1],args[2]);
|
||||
}
|
||||
if (error)
|
||||
std::cout << help();
|
||||
@@ -220,6 +228,7 @@ int Spartan::execute(std::vector<std::string> const &args) {
|
||||
std::string Spartan::help() {
|
||||
return fmt::format("\tpcap_to_msgs pcap_file\n"
|
||||
"\tpcap_to_json pcap_file\n"
|
||||
"\tslim_pcap in_pcap out_pcap\n"
|
||||
"\tpcap_to_tables pcap_file output_dir\n");
|
||||
}
|
||||
|
||||
|
||||
@@ -25,24 +25,6 @@ struct PacketBuffers {
|
||||
|
||||
mdp3::AB_Arbitrageur ab;
|
||||
|
||||
void cme_ts_to_str(uint64_t cme_ts, std::stringstream *out) {
|
||||
auto one_billion = 1000000000;
|
||||
time_t unix_ts = cme_ts / one_billion;
|
||||
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
|
||||
auto year = boost_ts.date().year();
|
||||
auto month = boost_ts.date().month().as_number();
|
||||
auto day = boost_ts.date().day().as_number();
|
||||
int hours = boost_ts.time_of_day().hours();
|
||||
int minutes = boost_ts.time_of_day().minutes();
|
||||
int seconds = boost_ts.time_of_day().seconds();
|
||||
*out << year << "-" << std::setw(2) << std::setfill('0') << month << "-"
|
||||
<< std::setw(2) << std::setw(2) << std::setfill('0') << day << "."
|
||||
<< std::setw(2) << std::setw(2) << std::setfill('0') << hours << ":"
|
||||
<< std::setw(2) << std::setfill('0') << minutes << ":" << std::setw(2)
|
||||
<< std::setfill('0') << seconds << "." << std::setw(9)
|
||||
<< cme_ts - (unix_ts * one_billion);
|
||||
}
|
||||
|
||||
struct JSON_Writer {
|
||||
|
||||
std::stringstream json_buffer;
|
||||
@@ -112,10 +94,10 @@ struct CSVPrinter {
|
||||
template <class M> void print(M &m) {
|
||||
stime_buffer.str("");
|
||||
stime_buffer.clear();
|
||||
cme_ts_to_str(packet.sending_time(), &stime_buffer);
|
||||
mdp3::cme_ts_to_date_str(packet.sending_time(), &stime_buffer);
|
||||
ttime_buffer.str("");
|
||||
ttime_buffer.clear();
|
||||
cme_ts_to_str(get_transact_time(m), &ttime_buffer);
|
||||
mdp3::cme_ts_to_date_str(get_transact_time(m), &ttime_buffer);
|
||||
|
||||
auto t = get_transact_time(m);
|
||||
std::cout << ttime_buffer.str() << "," << stime_buffer.str() << ","
|
||||
|
||||
@@ -35,6 +35,13 @@ class MDP3Event {
|
||||
}
|
||||
};
|
||||
|
||||
inline std::ostream &operator<<(std::ostream &s, MDP3Event const &ev) {
|
||||
s << " transact_time: " << ev.transact_time.value()
|
||||
<< " packets: " << ev.first_packet_sn
|
||||
<< " to " << ev.last_packet_sn;
|
||||
return s;
|
||||
}
|
||||
|
||||
using MDP3Quantity = utils::ValueWrapper<struct MDP3QuantityTag, int32_t>;
|
||||
using MDP3OrderPriority =
|
||||
utils::ValueWrapper<struct MDP3OrderPriorityTag, uint64_t>;
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#pragma once
|
||||
#include "../parsing/MDP3Parser.h"
|
||||
#include "../feed_control/PickedFeed.h"
|
||||
|
||||
#include "lib/utils/bts_assert.h"
|
||||
|
||||
namespace bts::spartan::mdp3 {
|
||||
@@ -17,6 +19,7 @@ class AB_Arbitrageur {
|
||||
|
||||
struct Action {
|
||||
ActionCode action;
|
||||
PickedFeed picked;
|
||||
MDP3PacketView const packet;
|
||||
};
|
||||
|
||||
@@ -28,13 +31,13 @@ class AB_Arbitrageur {
|
||||
auto action_a = try_one(a);
|
||||
if (action_a == ActionCode::APPLY) {
|
||||
_last_seqn = a.seq_num();
|
||||
return {action_a, a};
|
||||
return {action_a, PickedFeed::A, a};
|
||||
}
|
||||
|
||||
auto action_b = try_one(b);
|
||||
if (action_b == ActionCode::APPLY) {
|
||||
_last_seqn = b.seq_num();
|
||||
return {action_b, b};
|
||||
return {action_b, PickedFeed::B, b};
|
||||
}
|
||||
|
||||
// the first packet defines our starting sequence number, rather
|
||||
@@ -43,17 +46,17 @@ class AB_Arbitrageur {
|
||||
if (_last_seqn == 0) {
|
||||
if (!a.empty()) {
|
||||
_last_seqn = a.seq_num();
|
||||
return {ActionCode::APPLY, a};
|
||||
return {ActionCode::APPLY, PickedFeed::A, a};
|
||||
} else {
|
||||
_last_seqn = a.seq_num();
|
||||
return {ActionCode::APPLY, a};
|
||||
return {ActionCode::APPLY, PickedFeed::A,a};
|
||||
}
|
||||
}
|
||||
|
||||
// Both feeds are either ahead or behind, if they are ahead we
|
||||
// return RECOVER, otherwise we return IGNORE.
|
||||
if (action_a == action_b)
|
||||
return {action_a, a};
|
||||
return {action_a, PickedFeed::None, a};
|
||||
|
||||
// One of the feeds is ahead of us and the other is behind. Its
|
||||
// possible that we will recover eventually, that is the feed
|
||||
@@ -69,9 +72,9 @@ class AB_Arbitrageur {
|
||||
how_far_ahead = b.seq_num() - _last_seqn;
|
||||
|
||||
if (how_far_ahead > 10)
|
||||
return {ActionCode::RECOVER, a};
|
||||
return {ActionCode::RECOVER, PickedFeed::None, a};
|
||||
else
|
||||
return {ActionCode::IGNORE, a};
|
||||
return {ActionCode::IGNORE, PickedFeed::None, a};
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
#pragma once
|
||||
|
||||
namespace bts::spartan {
|
||||
enum class PickedFeed { A, B, None };
|
||||
}
|
||||
@@ -50,7 +50,7 @@ ChannelHandler::~ChannelHandler() {}
|
||||
void ChannelHandler::on_instrument_feed(MDP3PacketView const &pv) {
|
||||
switch (me->state) {
|
||||
case State::RECOVERY: {
|
||||
RecoveryGuard g {*this};
|
||||
RecoveryGuard g{*this};
|
||||
parse_packet(pv, me->recovery_handler.on_instrument_feed);
|
||||
break;
|
||||
}
|
||||
@@ -62,7 +62,7 @@ void ChannelHandler::on_instrument_feed(MDP3PacketView const &pv) {
|
||||
void ChannelHandler::on_snapshot_feed(MDP3PacketView const &pv) {
|
||||
switch (me->state) {
|
||||
case State::RECOVERY: {
|
||||
RecoveryGuard g {*this};
|
||||
RecoveryGuard g{*this};
|
||||
parse_packet(pv, me->recovery_handler.on_snapshot_feed);
|
||||
break;
|
||||
}
|
||||
@@ -71,16 +71,16 @@ void ChannelHandler::on_snapshot_feed(MDP3PacketView const &pv) {
|
||||
}
|
||||
}
|
||||
|
||||
void ChannelHandler::on_incremental_feed(MDP3PacketView const &a,
|
||||
MDP3PacketView const &b) {
|
||||
PickedFeed ChannelHandler::on_incremental_feed(MDP3PacketView const &a,
|
||||
MDP3PacketView const &b) {
|
||||
auto action = me->real_time_ab(a, b);
|
||||
|
||||
switch (me->state) {
|
||||
case State::RECOVERY: {
|
||||
auto action = me->real_time_ab(a, b);
|
||||
switch (action.action) {
|
||||
case AB_Arbitrageur::APPLY: {
|
||||
RecoveryGuard g {*this};
|
||||
RecoveryGuard g{*this};
|
||||
parse_packet(action.packet, me->recovery_handler.on_incremental_feed);
|
||||
break;
|
||||
}
|
||||
case AB_Arbitrageur::IGNORE:
|
||||
break;
|
||||
@@ -91,7 +91,6 @@ void ChannelHandler::on_incremental_feed(MDP3PacketView const &a,
|
||||
break;
|
||||
}
|
||||
case State::REALTIME: {
|
||||
auto action = me->real_time_ab(a, b);
|
||||
switch (action.action) {
|
||||
case AB_Arbitrageur::APPLY:
|
||||
parse_packet(action.packet, me->real_time_handler);
|
||||
@@ -105,15 +104,21 @@ void ChannelHandler::on_incremental_feed(MDP3PacketView const &a,
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return action.picked;
|
||||
}
|
||||
|
||||
void ChannelHandler::start_recovery() {
|
||||
switch (me->state) {
|
||||
case State::RECOVERY:
|
||||
LOG(Spartan, Warn, "Restarting channel recovery.");
|
||||
me->real_time_ab = {};
|
||||
me->recovery_handler.reset();
|
||||
break;
|
||||
case State::REALTIME:
|
||||
LOG(Spartan, Warn,
|
||||
"Loss detected. Shutting down real-time handling and switching to "
|
||||
"recovery feeds.");
|
||||
me->real_time_ab = {};
|
||||
me->real_time_handler.reset();
|
||||
me->feed_controller.start_instrument_feed();
|
||||
@@ -128,15 +133,18 @@ void ChannelHandler::try_start_real_time() {
|
||||
bts_assert(me->state == State::RECOVERY);
|
||||
auto maybe_snaps = me->recovery_handler.try_build_snapshot_and_reset();
|
||||
if (maybe_snaps) {
|
||||
LOG(Spartan, Warn, "Recovery complete. Starting real time feed");
|
||||
me->state = State::REALTIME;
|
||||
std::vector<InstrumentSnapshot const *> snap_ptrs;
|
||||
for (auto &p : maybe_snaps.value())
|
||||
snap_ptrs.push_back(p.get());
|
||||
me->real_time_handler.add_snapshots(maybe_snaps.value());
|
||||
me->feed_controller.stop_instrument_feed();
|
||||
me->feed_controller.stop_snapshot_feed();
|
||||
me->observer.on_recovery_complete(snap_ptrs);
|
||||
for (auto s : snap_ptrs) {
|
||||
const_cast<InstrumentSnapshot*>(s)->clear_fills();
|
||||
const_cast<InstrumentSnapshot*>(s)->clear_order_changes();
|
||||
const_cast<InstrumentSnapshot *>(s)->clear_fills();
|
||||
const_cast<InstrumentSnapshot *>(s)->clear_order_changes();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
#pragma once
|
||||
|
||||
#include <string_view>
|
||||
#include <memory>
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h"
|
||||
#include <string_view>
|
||||
|
||||
#include "../feed_control/PickedFeed.h"
|
||||
#include "../parsing/MDP3Parser.h"
|
||||
|
||||
namespace bts::spartan::mdp3 {
|
||||
|
||||
@@ -11,15 +13,16 @@ class IFeedController;
|
||||
class MDP3PacketView;
|
||||
|
||||
class ChannelHandler {
|
||||
public:
|
||||
ChannelHandler(IChannelObserver&, IFeedController &);
|
||||
public:
|
||||
ChannelHandler(IChannelObserver &, IFeedController &);
|
||||
~ChannelHandler();
|
||||
|
||||
void on_instrument_feed(MDP3PacketView const&);
|
||||
void on_snapshot_feed(MDP3PacketView const&);
|
||||
void on_incremental_feed(MDP3PacketView const&a, MDP3PacketView const&b = {} );
|
||||
void on_instrument_feed(MDP3PacketView const &);
|
||||
void on_snapshot_feed(MDP3PacketView const &);
|
||||
PickedFeed on_incremental_feed(MDP3PacketView const &a,
|
||||
MDP3PacketView const &b = {});
|
||||
|
||||
private:
|
||||
private:
|
||||
struct RecoveryGuard;
|
||||
void start_recovery();
|
||||
void try_start_real_time();
|
||||
@@ -28,5 +31,4 @@ private:
|
||||
std::unique_ptr<PImpl> me;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
} // namespace bts::spartan::mdp3
|
||||
@@ -56,7 +56,7 @@ void BookSnapshot::operator()(::mdp3::SnapshotFullRefresh52 &book) {
|
||||
me->newest_snapshot_sn =
|
||||
std::max(book.lastMsgSeqNumProcessed(), me->newest_snapshot_sn);
|
||||
me->books.emplace_back(ib);
|
||||
LOG(Spartan, Info, "Recovered book {}/{} from snapshot feed",
|
||||
LOG(Spartan, Diag, "Recovered book {}/{} from snapshot feed",
|
||||
me->books.size(), book.totNumReports());
|
||||
check_completion();
|
||||
break;
|
||||
|
||||
@@ -165,7 +165,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() {
|
||||
auto &this_snapshot = *current_snapshot;
|
||||
if (this_snapshot.last_processed_sn == 0 ||
|
||||
this_snapshot.last_processed_sn + 1 == pv.seq_num()) {
|
||||
LOG(Spartan, Info,
|
||||
LOG(Spartan, Diag,
|
||||
"Added snapshot with last update time {} for security {} with "
|
||||
"last processed sn {}",
|
||||
this_snapshot.snapshot->definition().last_update_time().value(),
|
||||
@@ -178,7 +178,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() {
|
||||
|
||||
// Now parse the next packet
|
||||
|
||||
LOG(Spartan, Info, "Appplying queued packet with sequence number {}",
|
||||
LOG(Spartan, Diag, "Appplying queued packet with sequence number {}",
|
||||
pv.seq_num());
|
||||
parse_packet(pv, rt_handler);
|
||||
last_applied_packet = pv.seq_num();
|
||||
@@ -187,7 +187,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() {
|
||||
for (; current_snapshot != sortable_snapshots.end(); current_snapshot++) {
|
||||
auto &this_snapshot = *current_snapshot;
|
||||
bts_assert(this_snapshot.last_processed_sn == last_applied_packet);
|
||||
LOG(Spartan, Info,
|
||||
LOG(Spartan, Diag,
|
||||
"Added snapshot with last update time {} for security {} with "
|
||||
"last processed sn {}",
|
||||
this_snapshot.snapshot->definition().last_update_time().value(),
|
||||
|
||||
@@ -1,96 +1,272 @@
|
||||
#include "mdp3_utils.h"
|
||||
|
||||
#include "../../../util/PCapReader.h"
|
||||
#include "../common/InstrumentSnapshot.h"
|
||||
#include "../feed_control/AB_Arbitrageur.h"
|
||||
#include "../feed_control/IFeedController.h"
|
||||
#include "../gen/MDIncrementalRefreshOrderBook47.h"
|
||||
#include "../gen/MDInstrumentDefinitionFuture54.h"
|
||||
#include "../gen/MDInstrumentDefinitionOption55.h"
|
||||
#include "../gen/MDInstrumentDefinitionSpread56.h"
|
||||
#include "../gen/SnapshotFullRefreshOrderBook53.h"
|
||||
#include "../handler/ChannelHandler.h"
|
||||
#include "../handler/IChannelObserver.h"
|
||||
#include "../parsing/MDP3Parser.h"
|
||||
#include "server/sparkd/cmds/spartan/util/PCapReader.h"
|
||||
#include "lib/utils/bts_chrono.h"
|
||||
|
||||
#include "lib/utils/overloaded.h"
|
||||
|
||||
#include <Pcap++/PcapFileDevice.h>
|
||||
#include <boost/date_time/posix_time/conversion.hpp>
|
||||
#include <boost/date_time/posix_time/ptime.hpp>
|
||||
#include <fmt/format.h>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
|
||||
namespace bts::spartan::mdp3 {
|
||||
|
||||
struct Guesser {
|
||||
FeedMap result;
|
||||
PCapReader::PacketInfo current_packet;
|
||||
|
||||
void on_begin_packet(MDP3PacketView const &) {}
|
||||
void on_end_packet() {}
|
||||
|
||||
void set_current_source(const PCapReader::PacketInfo &info) {
|
||||
current_packet = info;
|
||||
}
|
||||
|
||||
template <class M> void operator()(M const &) {
|
||||
if (result.incremental[0].port == 0) {
|
||||
result.incremental[0].port = current_packet.dst_port;
|
||||
result.incremental[0].ip = current_packet.dst_ip;
|
||||
return;
|
||||
}
|
||||
if (result.incremental[0].port == current_packet.dst_port &&
|
||||
result.incremental[0].ip == current_packet.dst_ip)
|
||||
return;
|
||||
|
||||
if (result.incremental[1].port == 0) {
|
||||
result.incremental[1].port = current_packet.dst_port;
|
||||
result.incremental[1].ip = current_packet.dst_ip;
|
||||
}
|
||||
}
|
||||
|
||||
void operator()(::mdp3::SnapshotFullRefresh52 const &) {
|
||||
if (result.snapshot.port == 0) {
|
||||
result.snapshot.port = current_packet.dst_port;
|
||||
result.snapshot.ip = current_packet.dst_ip;
|
||||
}
|
||||
}
|
||||
|
||||
void operator()(::mdp3::SnapshotFullRefreshOrderBook53 const &) {
|
||||
if (result.snapshot.port == 0) {
|
||||
result.snapshot.port = current_packet.dst_port;
|
||||
result.snapshot.ip = current_packet.dst_ip;
|
||||
}
|
||||
}
|
||||
|
||||
void operator()(::mdp3::MDInstrumentDefinitionFuture54 const &) {
|
||||
on_definition();
|
||||
}
|
||||
|
||||
void operator()(::mdp3::MDInstrumentDefinitionOption55 const &) {
|
||||
on_definition();
|
||||
}
|
||||
|
||||
void operator()(::mdp3::MDInstrumentDefinitionSpread56 const &) {
|
||||
on_definition();
|
||||
}
|
||||
|
||||
void on_definition() {
|
||||
if (result.instrument.port)
|
||||
return;
|
||||
if (result.incremental[0].port == 0)
|
||||
return;
|
||||
if (result.incremental[1].port == 0)
|
||||
return;
|
||||
if (result.incremental[0].ip == current_packet.dst_ip &&
|
||||
result.incremental[0].port == current_packet.dst_port)
|
||||
return;
|
||||
if (result.incremental[1].ip == current_packet.dst_ip &&
|
||||
result.incremental[1].port == current_packet.dst_port)
|
||||
return;
|
||||
result.instrument.port = current_packet.dst_port;
|
||||
result.instrument.ip = current_packet.dst_ip;
|
||||
}
|
||||
};
|
||||
|
||||
FeedMap guess_feed_map(std::string const &pcap_path) {
|
||||
|
||||
struct Guesser {
|
||||
FeedMap result;
|
||||
PCapReader::PacketInfo current_packet;
|
||||
|
||||
void on_begin_packet(MDP3PacketView const&) {}
|
||||
void on_end_packet() {}
|
||||
|
||||
void set_current_source(const PCapReader::PacketInfo &info) {
|
||||
current_packet = info;
|
||||
}
|
||||
|
||||
void operator()(::mdp3::MDIncrementalRefreshOrderBook47 const &) {
|
||||
if (result.incremental[0].port == 0) {
|
||||
result.incremental[0].port = current_packet.dst_port;
|
||||
result.incremental[0].ip = current_packet.dst_ip;
|
||||
return;
|
||||
}
|
||||
if (result.incremental[0].port == current_packet.dst_port &&
|
||||
result.incremental[0].ip == current_packet.dst_ip)
|
||||
return;
|
||||
|
||||
if (result.incremental[1].port == 0) {
|
||||
result.incremental[1].port = current_packet.dst_port;
|
||||
result.incremental[1].ip = current_packet.dst_ip;
|
||||
}
|
||||
}
|
||||
|
||||
void operator()(::mdp3::SnapshotFullRefreshOrderBook53 const &) {
|
||||
if (result.snapshot.port == 0) {
|
||||
result.snapshot.port = current_packet.dst_port;
|
||||
result.snapshot.ip = current_packet.dst_ip;
|
||||
}
|
||||
}
|
||||
|
||||
void operator()(::mdp3::MDInstrumentDefinitionFuture54 const &) {
|
||||
on_definition();
|
||||
}
|
||||
|
||||
void operator()(::mdp3::MDInstrumentDefinitionOption55 const &) {
|
||||
on_definition();
|
||||
}
|
||||
|
||||
void operator()(::mdp3::MDInstrumentDefinitionSpread56 const &) {
|
||||
on_definition();
|
||||
}
|
||||
|
||||
void on_definition() {
|
||||
if (result.instrument.port)
|
||||
return;
|
||||
if (result.incremental[0].port == 0)
|
||||
return;
|
||||
if (result.incremental[1].port == 0)
|
||||
return;
|
||||
if (result.incremental[0].ip == current_packet.dst_ip &&
|
||||
result.incremental[0].port == current_packet.dst_port)
|
||||
return;
|
||||
if (result.incremental[1].ip == current_packet.dst_ip &&
|
||||
result.incremental[1].port == current_packet.dst_port)
|
||||
return;
|
||||
result.instrument.port = current_packet.dst_port;
|
||||
result.instrument.ip = current_packet.dst_ip;
|
||||
}
|
||||
} guesser;
|
||||
|
||||
Guesser guesser;
|
||||
auto reader = PCapReader{pcap_path};
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
guesser.set_current_source(info);
|
||||
parse_packet(MDP3PacketView {payload},guesser);
|
||||
if (guesser.result)
|
||||
parse_packet(MDP3PacketView{payload}, guesser);
|
||||
if (guesser.result)
|
||||
return false;
|
||||
else
|
||||
return true;
|
||||
};
|
||||
|
||||
|
||||
while (reader.read_next())
|
||||
;
|
||||
return guesser.result;
|
||||
}
|
||||
|
||||
std::ostream &operator<<(std::ostream &o, FeedMap const &fm) {
|
||||
o << "IA:" << fm.incremental[0].ip << ":" << fm.incremental[0].port
|
||||
<< " IB:" << fm.incremental[1].ip << ":" << fm.incremental[1].port
|
||||
<< " N:" << fm.instrument.ip << ":" << fm.instrument.port
|
||||
<< " SMBO:" << fm.snapshot.ip << ":" << fm.snapshot.port;
|
||||
return o;
|
||||
}
|
||||
|
||||
bool slim_down_pcap(std::string const &file, std::string const &out_file) {
|
||||
using namespace bts::spartan;
|
||||
using namespace bts::spartan::mdp3;
|
||||
|
||||
struct Ctrl : IFeedController {
|
||||
bool instrument_feed = false;
|
||||
bool snapshot_feed = false;
|
||||
bool incremental_feed = false;
|
||||
|
||||
void start_instrument_feed() override { instrument_feed = true; }
|
||||
void stop_instrument_feed() override { instrument_feed = false; }
|
||||
void start_snapshot_feed() override { snapshot_feed = true; }
|
||||
void stop_snapshot_feed() override { snapshot_feed = false; }
|
||||
void start_incremental_feed() override { incremental_feed = true; }
|
||||
void stop_incremental_feed() override { incremental_feed = false; }
|
||||
} ctrl;
|
||||
|
||||
struct Obs : IChannelObserver {
|
||||
bool observed_loss = false;
|
||||
bool recovery_complete = false;
|
||||
|
||||
size_t previous_packet = 0;
|
||||
size_t previous_send_ts = 0;
|
||||
TimeStamp last_print;
|
||||
|
||||
void on_recovery_complete(
|
||||
std::vector<InstrumentSnapshot const *> const &) override {
|
||||
recovery_complete = true;
|
||||
}
|
||||
|
||||
void on_real_time_event(MDP3Event const &ev) override {
|
||||
if (ms_diff(Timer::now(), last_print) >= 1000) {
|
||||
last_print = Timer::now();
|
||||
std::stringstream str;
|
||||
cme_ts_to_time_str(ev.transact_time.value(), &str);
|
||||
LOG(Spartan, Info,
|
||||
"Time: {} -> {} pkt/sec at {}x real time",
|
||||
str.str(),
|
||||
(ev.first_packet_sn - previous_packet),
|
||||
(ev.first_packet_send_time.value() - previous_send_ts) / 1000000000);
|
||||
previous_packet = ev.first_packet_sn;
|
||||
previous_send_ts = ev.first_packet_send_time.value();
|
||||
}
|
||||
}
|
||||
void on_real_time_channel_loss() override { observed_loss = true; }
|
||||
} obs;
|
||||
|
||||
struct IncrementalFeedBuffer {
|
||||
bool written_once = false;
|
||||
pcpp::RawPacket rp;
|
||||
mdp3::MDP3Packet p;
|
||||
};
|
||||
std::array<IncrementalFeedBuffer, 2> inc_buffer;
|
||||
|
||||
auto feed_map = mdp3::guess_feed_map(file);
|
||||
bts_assert(feed_map);
|
||||
auto reader = PCapReader{file};
|
||||
|
||||
mdp3::AB_Arbitrageur ab;
|
||||
mdp3::ChannelHandler handler(obs, ctrl);
|
||||
|
||||
pcpp::PcapNgFileWriterDevice writer(out_file.c_str());
|
||||
writer.open();
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
if ((info.dst_ip == feed_map.incremental[0].ip &&
|
||||
info.dst_port == feed_map.incremental[0].port)) {
|
||||
inc_buffer[0].rp = *info.rp;
|
||||
inc_buffer[0].p.set_bytes(payload);
|
||||
} else if ((info.dst_ip == feed_map.incremental[1].ip &&
|
||||
info.dst_port == feed_map.incremental[1].port)) {
|
||||
inc_buffer[1].rp = *info.rp;
|
||||
inc_buffer[1].p.set_bytes(payload);
|
||||
} else if (info.dst_ip == feed_map.instrument.ip &&
|
||||
info.dst_port == feed_map.instrument.port) {
|
||||
if (ctrl.instrument_feed) {
|
||||
handler.on_instrument_feed(mdp3::MDP3PacketView{payload});
|
||||
writer.writePacket(*info.rp);
|
||||
}
|
||||
return true;
|
||||
} else if (info.dst_ip == feed_map.snapshot.ip &&
|
||||
info.dst_port == feed_map.snapshot.port) {
|
||||
if (ctrl.snapshot_feed) {
|
||||
handler.on_snapshot_feed(mdp3::MDP3PacketView{payload});
|
||||
writer.writePacket(*info.rp);
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
return true;
|
||||
|
||||
// Incremental feed handling
|
||||
auto picked = handler.on_incremental_feed(inc_buffer[0].p.view(),
|
||||
inc_buffer[1].p.view());
|
||||
if (picked == PickedFeed::A ||
|
||||
((!inc_buffer[0].written_once || !obs.recovery_complete) &&
|
||||
!inc_buffer[0].p.view().empty())) {
|
||||
inc_buffer[0].written_once = true;
|
||||
writer.writePacket(inc_buffer[0].rp);
|
||||
}
|
||||
|
||||
if (picked == PickedFeed::B ||
|
||||
((!inc_buffer[1].written_once || !obs.recovery_complete) &&
|
||||
!inc_buffer[1].p.view().empty())) {
|
||||
inc_buffer[1].written_once = true;
|
||||
writer.writePacket(inc_buffer[1].rp);
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
while (reader.read_next())
|
||||
if (obs.observed_loss) {
|
||||
LOG(Spartan, Error, "Observed loss in {}. Cannot slim it down", file);
|
||||
return false;
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out) {
|
||||
auto one_billion = 1000000000;
|
||||
time_t unix_ts = cme_ts / one_billion;
|
||||
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
|
||||
int hours = boost_ts.time_of_day().hours();
|
||||
int minutes = boost_ts.time_of_day().minutes();
|
||||
int seconds = boost_ts.time_of_day().seconds();
|
||||
*out << std::setw(2) << std::setfill('0') << hours << ":" << std::setw(2)
|
||||
<< std::setfill('0') << minutes << ":" << std::setw(2)
|
||||
<< std::setfill('0') << seconds << "." << std::setw(9)
|
||||
<< cme_ts - (unix_ts * one_billion);
|
||||
}
|
||||
|
||||
void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out) {
|
||||
auto one_billion = 1000000000;
|
||||
time_t unix_ts = cme_ts / one_billion;
|
||||
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
|
||||
auto year = boost_ts.date().year();
|
||||
auto month = boost_ts.date().month().as_number();
|
||||
auto day = boost_ts.date().day().as_number();
|
||||
int hours = boost_ts.time_of_day().hours();
|
||||
int minutes = boost_ts.time_of_day().minutes();
|
||||
int seconds = boost_ts.time_of_day().seconds();
|
||||
*out << year << "-" << std::setw(2) << std::setfill('0') << month << "-"
|
||||
<< std::setw(2) << std::setw(2) << std::setfill('0') << day << "."
|
||||
<< std::setw(2) << std::setw(2) << std::setfill('0') << hours << ":"
|
||||
<< std::setw(2) << std::setfill('0') << minutes << ":" << std::setw(2)
|
||||
<< std::setfill('0') << seconds << "." << std::setw(9)
|
||||
<< cme_ts - (unix_ts * one_billion);
|
||||
}
|
||||
|
||||
} // namespace bts::spartan::mdp3
|
||||
@@ -23,15 +23,12 @@ struct FeedMap {
|
||||
snapshot.port;
|
||||
}
|
||||
};
|
||||
|
||||
inline std::ostream &operator<<(std::ostream &o, FeedMap const &fm) {
|
||||
o << "IA:" << fm.incremental[0].ip << ":" << fm.incremental[0].port
|
||||
<< " IB:" << fm.incremental[1].ip << ":" << fm.incremental[1].port
|
||||
<< " N:" << fm.instrument.ip << ":" << fm.instrument.port
|
||||
<< " SMBO:" << fm.snapshot.ip << ":" << fm.snapshot.port;
|
||||
return o;
|
||||
}
|
||||
std::ostream &operator<<(std::ostream &o, FeedMap const &fm);
|
||||
|
||||
FeedMap guess_feed_map(std::string const &pcap_path);
|
||||
bool slim_down_pcap(std::string const&in, std::string const &out);
|
||||
void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out);
|
||||
void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out);
|
||||
|
||||
|
||||
} // namespace bts::spartan::mdp3
|
||||
@@ -1,11 +1,11 @@
|
||||
#include "PCapReader.h"
|
||||
#include "lib/utils/bts_assert.h"
|
||||
|
||||
#include <Pcap++/Packet.h>
|
||||
#include <Pcap++/UdpLayer.h>
|
||||
#include <Pcap++/TcpLayer.h>
|
||||
#include <Pcap++/IPv4Layer.h>
|
||||
#include <Pcap++/Packet.h>
|
||||
#include <Pcap++/PcapFileDevice.h>
|
||||
#include <Pcap++/TcpLayer.h>
|
||||
#include <Pcap++/UdpLayer.h>
|
||||
#include <arpa/inet.h>
|
||||
|
||||
namespace bts {
|
||||
@@ -14,10 +14,10 @@ namespace spartan {
|
||||
struct PCapReader::PImpl {
|
||||
PImpl(std::string const &file_path)
|
||||
: reader(pcpp::IFileReaderDevice::getReader(file_path.c_str())),
|
||||
on_packet([](auto const&, auto const &) {return true;}) {}
|
||||
on_packet([](auto const &, auto const &) { return true; }) {}
|
||||
|
||||
std::unique_ptr<pcpp::IFileReaderDevice> reader;
|
||||
std::function<bool(PacketInfo const&, std::string_view const &)> on_packet;
|
||||
OnPacket on_packet;
|
||||
pcpp::RawPacket rawPacket;
|
||||
};
|
||||
|
||||
@@ -31,19 +31,18 @@ PCapReader::PCapReader(std::string const &file_path, std::string const &filter)
|
||||
|
||||
PCapReader::~PCapReader() {}
|
||||
|
||||
std::function<bool(const PCapReader::PacketInfo &, std::string_view const &)> &PCapReader::on_packet() {
|
||||
return me->on_packet;
|
||||
}
|
||||
PCapReader::OnPacket &PCapReader::on_packet() { return me->on_packet; }
|
||||
|
||||
bool PCapReader::read_next() {
|
||||
if (me->reader->getNextPacket(me->rawPacket)) {
|
||||
pcpp::Packet parsedPacket(&me->rawPacket);
|
||||
PacketInfo info;
|
||||
|
||||
info.rp = &me->rawPacket;
|
||||
|
||||
auto udp = parsedPacket.getLayerOfType<pcpp::UdpLayer>();
|
||||
if (udp)
|
||||
info.dst_port = udp->getUdpHeader()->portDst;
|
||||
else {
|
||||
else {
|
||||
auto tcp = parsedPacket.getLayerOfType<pcpp::TcpLayer>();
|
||||
info.dst_port = tcp->getTcpHeader()->portDst;
|
||||
}
|
||||
@@ -52,12 +51,13 @@ bool PCapReader::read_next() {
|
||||
info.dst_ip = ip->getDstIpAddress().toString();
|
||||
auto payload = parsedPacket.getLayerOfType(pcpp::GenericPayload);
|
||||
if (payload)
|
||||
return me->on_packet(info, {(const char *)payload->getData(), payload->getDataLen()});
|
||||
return me->on_packet(
|
||||
info, {(const char *)payload->getData(), payload->getDataLen()});
|
||||
else
|
||||
return true;
|
||||
} else
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace utils
|
||||
} // namespace spartan
|
||||
} // namespace bts
|
||||
@@ -1,26 +1,32 @@
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
|
||||
namespace pcpp {
|
||||
class RawPacket;
|
||||
}
|
||||
|
||||
namespace bts {
|
||||
namespace spartan {
|
||||
|
||||
namespace bts::spartan {
|
||||
class PCapReader {
|
||||
public:
|
||||
PCapReader(std::string const&file_path,
|
||||
std::string const&filter = "");
|
||||
public:
|
||||
PCapReader(std::string const &file_path, std::string const &filter = "");
|
||||
~PCapReader();
|
||||
bool read_next();
|
||||
|
||||
struct PacketInfo {
|
||||
std::string dst_ip;
|
||||
int dst_port;
|
||||
pcpp::RawPacket *rp;
|
||||
};
|
||||
std::function<bool(const PacketInfo&,std::string_view const&)> &on_packet();
|
||||
private:
|
||||
|
||||
using OnPacket =
|
||||
std::function<bool(const PacketInfo &, std::string_view const &)>;
|
||||
OnPacket &on_packet();
|
||||
|
||||
private:
|
||||
struct PImpl;
|
||||
std::unique_ptr<PImpl> me;
|
||||
};
|
||||
|
||||
}}
|
||||
} // namespace bts::spartan
|
||||
Reference in new issue
Block a user