Added new spartan command to clean up pcaps to just what we need.

This commit is contained in:
Pedro Pinto committed 2020-08-19 16:29:49 -04:00
1 parent 86ca092f18
commit 5c5bc243d2
14 files changed
+398 -190

No files matched your search

+43 -30
View File
@@ -36,12 +36,9 @@ using namespace spartan;
struct TimeWriter {
mdp3::MDP3PacketView packet;
void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; }
void on_end_packet() {
std::cout << packet.sending_time() << std::endl;
}
void on_end_packet() { std::cout << packet.sending_time() << std::endl; }
template <class M> void operator()(M const &) {
}
template <class M> void operator()(M const &) {}
};
void print_times(std::string const &file) {
struct PacketBuffers {
@@ -58,9 +55,9 @@ void print_times(std::string const &file) {
if ((info.dst_ip == feed_map.incremental[0].ip &&
info.dst_port == feed_map.incremental[0].port)) {
TimeWriter tw;
parse_packet(MDP3PacketView {payload},tw);
parse_packet(MDP3PacketView{payload}, tw);
}
return true;
};
while (reader.read_next())
@@ -74,39 +71,55 @@ int QuickTest::execute(std::vector<std::string> const &args) {
log::configure_logging(cfg);
auto file = args[0];
print_times(file);
return 0;
using namespace bts::spartan::mdp3;
struct Obs : IChannelObserver {
Orders previous_orders;
size_t event_id = 0;
void on_recovery_complete(
std::vector<InstrumentSnapshot const *> const &secs) override {
for (auto s : secs) {
std::cout << "Bootstrap -----------------------------------------"
"------------------------------"
<< std::endl;
std::cout << "Symbol: " << s->definition().symbol() << std::endl
<< "Status: " << s->trading_status() << std::endl
<< "Orders:\n"
<< s->orders();
}
previous_orders.clear();
for (auto s : secs)
for (auto const &o : s->orders())
previous_orders[o.first] = o.second;
}
void on_real_time_event(MDP3Event const &ev) override {
event_id++;
for (auto s : ev.affected_securities) {
if (s->order_changes().size() || s->fills().size()) {
std::cout << "Event " << ev.transact_time
<< " -----------------------------" << std::endl;
std::cout << "Symbol: " << s->definition().symbol() << std::endl
<< "Status: " << s->trading_status() << std::endl
<< "Seq Num: " << ev.first_packet_sn << " to "
<< ev.last_packet_sn << std::endl
<< "Orders:\n"
<< s->order_changes() << "Fills:\n"
<< s->fills();
if (s->trading_status() ==
::mdp3::SecurityTradingStatus::ReadyToTrade) {
for (auto f : s->fills()) {
if (f.aggressor != MDP3Fill::THIS_BUY &&
f.aggressor != MDP3Fill::THIS_SELL) {
auto it = previous_orders.find(f.key);
bts_assert(it != previous_orders.end());
it->second.quantity.value() -= f.quantity.value();
if (it->second.quantity.value() < 0) {
std::cout << " Hidden sizes in order. "
<< " eid: " << event_id
<< " Event: " << ev
<< " Order: " << it->first
<< " Fill tid: " << f.trade_entry_id
<< std::endl;
}
}
}
}
for (auto const &ch : s->order_changes()) {
switch (ch.change) {
case OrderChange::added:
case OrderChange::updated:
previous_orders[ch.order.key] = ch.order;
break;
case OrderChange::removed:
previous_orders.erase(ch.order.key);
break;
}
}
}
}
+19 -10
View File
@@ -2,15 +2,26 @@
#include "server/sparkd/cmds/spartan/algo/RunTAS.h"
#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h"
#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/util/PCapReader.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/PickedFeed.h"
#include <fmt/format.h>
#include <fstream>
#include <iostream>
#include <Pcap++/PcapFileDevice.h>
namespace bts {
namespace cmd {
int slim_pcap(std::string const &file, std::string const &out_dir) {
return !spartan::mdp3::slim_down_pcap(file, out_dir);
}
int export_tables(std::string const &file, std::string const &out_dir) {
using namespace bts::spartan::mdp3;
@@ -25,15 +36,12 @@ int export_tables(std::string const &file, std::string const &out_dir) {
"first_st,last_st"
<< std::endl;
event_items
<< "eid, t_time,sid,status,type,"
"change,ok,side,"
"pri,price,qty,tid,agg"
<< std::endl;
event_items << "eid, t_time,sid,status,type,"
"change,ok,side,"
"pri,price,qty,tid,agg"
<< std::endl;
securities
<< "sid,symbol,group,asset,last_t"
<< std::endl;
securities << "sid,symbol,group,asset,last_t" << std::endl;
}
std::ofstream events;
@@ -155,8 +163,6 @@ int export_tables(std::string const &file, std::string const &out_dir) {
<< std::endl;
if (s->security_id().value() == 12181 && event_id == 1065)
std::cout << "There is" << std::endl;
auto deduced_order = s->deduced_order_change();
if (deduced_order) {
event_items << event_id << ","
@@ -211,6 +217,8 @@ int Spartan::execute(std::vector<std::string> const &args) {
error = spartan::convert_pcap_to_json(args);
else if (args[0] == "pcap_to_tables")
error = export_tables(args[1], args[2]);
else if (args[0] == "slim_pcap")
error = slim_pcap(args[1],args[2]);
}
if (error)
std::cout << help();
@@ -220,6 +228,7 @@ int Spartan::execute(std::vector<std::string> const &args) {
std::string Spartan::help() {
return fmt::format("\tpcap_to_msgs pcap_file\n"
"\tpcap_to_json pcap_file\n"
"\tslim_pcap in_pcap out_pcap\n"
"\tpcap_to_tables pcap_file output_dir\n");
}
@@ -25,24 +25,6 @@ struct PacketBuffers {
mdp3::AB_Arbitrageur ab;
void cme_ts_to_str(uint64_t cme_ts, std::stringstream *out) {
auto one_billion = 1000000000;
time_t unix_ts = cme_ts / one_billion;
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
auto year = boost_ts.date().year();
auto month = boost_ts.date().month().as_number();
auto day = boost_ts.date().day().as_number();
int hours = boost_ts.time_of_day().hours();
int minutes = boost_ts.time_of_day().minutes();
int seconds = boost_ts.time_of_day().seconds();
*out << year << "-" << std::setw(2) << std::setfill('0') << month << "-"
<< std::setw(2) << std::setw(2) << std::setfill('0') << day << "."
<< std::setw(2) << std::setw(2) << std::setfill('0') << hours << ":"
<< std::setw(2) << std::setfill('0') << minutes << ":" << std::setw(2)
<< std::setfill('0') << seconds << "." << std::setw(9)
<< cme_ts - (unix_ts * one_billion);
}
struct JSON_Writer {
std::stringstream json_buffer;
@@ -112,10 +94,10 @@ struct CSVPrinter {
template <class M> void print(M &m) {
stime_buffer.str("");
stime_buffer.clear();
cme_ts_to_str(packet.sending_time(), &stime_buffer);
mdp3::cme_ts_to_date_str(packet.sending_time(), &stime_buffer);
ttime_buffer.str("");
ttime_buffer.clear();
cme_ts_to_str(get_transact_time(m), &ttime_buffer);
mdp3::cme_ts_to_date_str(get_transact_time(m), &ttime_buffer);
auto t = get_transact_time(m);
std::cout << ttime_buffer.str() << "," << stime_buffer.str() << ","
@@ -35,6 +35,13 @@ class MDP3Event {
}
};
inline std::ostream &operator<<(std::ostream &s, MDP3Event const &ev) {
s << " transact_time: " << ev.transact_time.value()
<< " packets: " << ev.first_packet_sn
<< " to " << ev.last_packet_sn;
return s;
}
using MDP3Quantity = utils::ValueWrapper<struct MDP3QuantityTag, int32_t>;
using MDP3OrderPriority =
utils::ValueWrapper<struct MDP3OrderPriorityTag, uint64_t>;
@@ -1,5 +1,7 @@
#pragma once
#include "../parsing/MDP3Parser.h"
#include "../feed_control/PickedFeed.h"
#include "lib/utils/bts_assert.h"
namespace bts::spartan::mdp3 {
@@ -17,6 +19,7 @@ class AB_Arbitrageur {
struct Action {
ActionCode action;
PickedFeed picked;
MDP3PacketView const packet;
};
@@ -28,13 +31,13 @@ class AB_Arbitrageur {
auto action_a = try_one(a);
if (action_a == ActionCode::APPLY) {
_last_seqn = a.seq_num();
return {action_a, a};
return {action_a, PickedFeed::A, a};
}
auto action_b = try_one(b);
if (action_b == ActionCode::APPLY) {
_last_seqn = b.seq_num();
return {action_b, b};
return {action_b, PickedFeed::B, b};
}
// the first packet defines our starting sequence number, rather
@@ -43,17 +46,17 @@ class AB_Arbitrageur {
if (_last_seqn == 0) {
if (!a.empty()) {
_last_seqn = a.seq_num();
return {ActionCode::APPLY, a};
return {ActionCode::APPLY, PickedFeed::A, a};
} else {
_last_seqn = a.seq_num();
return {ActionCode::APPLY, a};
return {ActionCode::APPLY, PickedFeed::A,a};
}
}
// Both feeds are either ahead or behind, if they are ahead we
// return RECOVER, otherwise we return IGNORE.
if (action_a == action_b)
return {action_a, a};
return {action_a, PickedFeed::None, a};
// One of the feeds is ahead of us and the other is behind. Its
// possible that we will recover eventually, that is the feed
@@ -69,9 +72,9 @@ class AB_Arbitrageur {
how_far_ahead = b.seq_num() - _last_seqn;
if (how_far_ahead > 10)
return {ActionCode::RECOVER, a};
return {ActionCode::RECOVER, PickedFeed::None, a};
else
return {ActionCode::IGNORE, a};
return {ActionCode::IGNORE, PickedFeed::None, a};
}
private:
@@ -0,0 +1,5 @@
#pragma once
namespace bts::spartan {
enum class PickedFeed { A, B, None };
}
@@ -50,7 +50,7 @@ ChannelHandler::~ChannelHandler() {}
void ChannelHandler::on_instrument_feed(MDP3PacketView const &pv) {
switch (me->state) {
case State::RECOVERY: {
RecoveryGuard g {*this};
RecoveryGuard g{*this};
parse_packet(pv, me->recovery_handler.on_instrument_feed);
break;
}
@@ -62,7 +62,7 @@ void ChannelHandler::on_instrument_feed(MDP3PacketView const &pv) {
void ChannelHandler::on_snapshot_feed(MDP3PacketView const &pv) {
switch (me->state) {
case State::RECOVERY: {
RecoveryGuard g {*this};
RecoveryGuard g{*this};
parse_packet(pv, me->recovery_handler.on_snapshot_feed);
break;
}
@@ -71,16 +71,16 @@ void ChannelHandler::on_snapshot_feed(MDP3PacketView const &pv) {
}
}
void ChannelHandler::on_incremental_feed(MDP3PacketView const &a,
MDP3PacketView const &b) {
PickedFeed ChannelHandler::on_incremental_feed(MDP3PacketView const &a,
MDP3PacketView const &b) {
auto action = me->real_time_ab(a, b);
switch (me->state) {
case State::RECOVERY: {
auto action = me->real_time_ab(a, b);
switch (action.action) {
case AB_Arbitrageur::APPLY: {
RecoveryGuard g {*this};
RecoveryGuard g{*this};
parse_packet(action.packet, me->recovery_handler.on_incremental_feed);
break;
}
case AB_Arbitrageur::IGNORE:
break;
@@ -91,7 +91,6 @@ void ChannelHandler::on_incremental_feed(MDP3PacketView const &a,
break;
}
case State::REALTIME: {
auto action = me->real_time_ab(a, b);
switch (action.action) {
case AB_Arbitrageur::APPLY:
parse_packet(action.packet, me->real_time_handler);
@@ -105,15 +104,21 @@ void ChannelHandler::on_incremental_feed(MDP3PacketView const &a,
break;
}
}
return action.picked;
}
void ChannelHandler::start_recovery() {
switch (me->state) {
case State::RECOVERY:
LOG(Spartan, Warn, "Restarting channel recovery.");
me->real_time_ab = {};
me->recovery_handler.reset();
break;
case State::REALTIME:
LOG(Spartan, Warn,
"Loss detected. Shutting down real-time handling and switching to "
"recovery feeds.");
me->real_time_ab = {};
me->real_time_handler.reset();
me->feed_controller.start_instrument_feed();
@@ -128,15 +133,18 @@ void ChannelHandler::try_start_real_time() {
bts_assert(me->state == State::RECOVERY);
auto maybe_snaps = me->recovery_handler.try_build_snapshot_and_reset();
if (maybe_snaps) {
LOG(Spartan, Warn, "Recovery complete. Starting real time feed");
me->state = State::REALTIME;
std::vector<InstrumentSnapshot const *> snap_ptrs;
for (auto &p : maybe_snaps.value())
snap_ptrs.push_back(p.get());
me->real_time_handler.add_snapshots(maybe_snaps.value());
me->feed_controller.stop_instrument_feed();
me->feed_controller.stop_snapshot_feed();
me->observer.on_recovery_complete(snap_ptrs);
for (auto s : snap_ptrs) {
const_cast<InstrumentSnapshot*>(s)->clear_fills();
const_cast<InstrumentSnapshot*>(s)->clear_order_changes();
const_cast<InstrumentSnapshot *>(s)->clear_fills();
const_cast<InstrumentSnapshot *>(s)->clear_order_changes();
}
}
}
@@ -1,8 +1,10 @@
#pragma once
#include <string_view>
#include <memory>
#include "server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h"
#include <string_view>
#include "../feed_control/PickedFeed.h"
#include "../parsing/MDP3Parser.h"
namespace bts::spartan::mdp3 {
@@ -11,15 +13,16 @@ class IFeedController;
class MDP3PacketView;
class ChannelHandler {
public:
ChannelHandler(IChannelObserver&, IFeedController &);
public:
ChannelHandler(IChannelObserver &, IFeedController &);
~ChannelHandler();
void on_instrument_feed(MDP3PacketView const&);
void on_snapshot_feed(MDP3PacketView const&);
void on_incremental_feed(MDP3PacketView const&a, MDP3PacketView const&b = {} );
void on_instrument_feed(MDP3PacketView const &);
void on_snapshot_feed(MDP3PacketView const &);
PickedFeed on_incremental_feed(MDP3PacketView const &a,
MDP3PacketView const &b = {});
private:
private:
struct RecoveryGuard;
void start_recovery();
void try_start_real_time();
@@ -28,5 +31,4 @@ private:
std::unique_ptr<PImpl> me;
};
}
} // namespace bts::spartan::mdp3
@@ -56,7 +56,7 @@ void BookSnapshot::operator()(::mdp3::SnapshotFullRefresh52 &book) {
me->newest_snapshot_sn =
std::max(book.lastMsgSeqNumProcessed(), me->newest_snapshot_sn);
me->books.emplace_back(ib);
LOG(Spartan, Info, "Recovered book {}/{} from snapshot feed",
LOG(Spartan, Diag, "Recovered book {}/{} from snapshot feed",
me->books.size(), book.totNumReports());
check_completion();
break;
@@ -165,7 +165,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() {
auto &this_snapshot = *current_snapshot;
if (this_snapshot.last_processed_sn == 0 ||
this_snapshot.last_processed_sn + 1 == pv.seq_num()) {
LOG(Spartan, Info,
LOG(Spartan, Diag,
"Added snapshot with last update time {} for security {} with "
"last processed sn {}",
this_snapshot.snapshot->definition().last_update_time().value(),
@@ -178,7 +178,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() {
// Now parse the next packet
LOG(Spartan, Info, "Appplying queued packet with sequence number {}",
LOG(Spartan, Diag, "Appplying queued packet with sequence number {}",
pv.seq_num());
parse_packet(pv, rt_handler);
last_applied_packet = pv.seq_num();
@@ -187,7 +187,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() {
for (; current_snapshot != sortable_snapshots.end(); current_snapshot++) {
auto &this_snapshot = *current_snapshot;
bts_assert(this_snapshot.last_processed_sn == last_applied_packet);
LOG(Spartan, Info,
LOG(Spartan, Diag,
"Added snapshot with last update time {} for security {} with "
"last processed sn {}",
this_snapshot.snapshot->definition().last_update_time().value(),
@@ -1,96 +1,272 @@
#include "mdp3_utils.h"
#include "../../../util/PCapReader.h"
#include "../common/InstrumentSnapshot.h"
#include "../feed_control/AB_Arbitrageur.h"
#include "../feed_control/IFeedController.h"
#include "../gen/MDIncrementalRefreshOrderBook47.h"
#include "../gen/MDInstrumentDefinitionFuture54.h"
#include "../gen/MDInstrumentDefinitionOption55.h"
#include "../gen/MDInstrumentDefinitionSpread56.h"
#include "../gen/SnapshotFullRefreshOrderBook53.h"
#include "../handler/ChannelHandler.h"
#include "../handler/IChannelObserver.h"
#include "../parsing/MDP3Parser.h"
#include "server/sparkd/cmds/spartan/util/PCapReader.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/overloaded.h"
#include <Pcap++/PcapFileDevice.h>
#include <boost/date_time/posix_time/conversion.hpp>
#include <boost/date_time/posix_time/ptime.hpp>
#include <fmt/format.h>
#include <fstream>
#include <iostream>
namespace bts::spartan::mdp3 {
struct Guesser {
FeedMap result;
PCapReader::PacketInfo current_packet;
void on_begin_packet(MDP3PacketView const &) {}
void on_end_packet() {}
void set_current_source(const PCapReader::PacketInfo &info) {
current_packet = info;
}
template <class M> void operator()(M const &) {
if (result.incremental[0].port == 0) {
result.incremental[0].port = current_packet.dst_port;
result.incremental[0].ip = current_packet.dst_ip;
return;
}
if (result.incremental[0].port == current_packet.dst_port &&
result.incremental[0].ip == current_packet.dst_ip)
return;
if (result.incremental[1].port == 0) {
result.incremental[1].port = current_packet.dst_port;
result.incremental[1].ip = current_packet.dst_ip;
}
}
void operator()(::mdp3::SnapshotFullRefresh52 const &) {
if (result.snapshot.port == 0) {
result.snapshot.port = current_packet.dst_port;
result.snapshot.ip = current_packet.dst_ip;
}
}
void operator()(::mdp3::SnapshotFullRefreshOrderBook53 const &) {
if (result.snapshot.port == 0) {
result.snapshot.port = current_packet.dst_port;
result.snapshot.ip = current_packet.dst_ip;
}
}
void operator()(::mdp3::MDInstrumentDefinitionFuture54 const &) {
on_definition();
}
void operator()(::mdp3::MDInstrumentDefinitionOption55 const &) {
on_definition();
}
void operator()(::mdp3::MDInstrumentDefinitionSpread56 const &) {
on_definition();
}
void on_definition() {
if (result.instrument.port)
return;
if (result.incremental[0].port == 0)
return;
if (result.incremental[1].port == 0)
return;
if (result.incremental[0].ip == current_packet.dst_ip &&
result.incremental[0].port == current_packet.dst_port)
return;
if (result.incremental[1].ip == current_packet.dst_ip &&
result.incremental[1].port == current_packet.dst_port)
return;
result.instrument.port = current_packet.dst_port;
result.instrument.ip = current_packet.dst_ip;
}
};
FeedMap guess_feed_map(std::string const &pcap_path) {
struct Guesser {
FeedMap result;
PCapReader::PacketInfo current_packet;
void on_begin_packet(MDP3PacketView const&) {}
void on_end_packet() {}
void set_current_source(const PCapReader::PacketInfo &info) {
current_packet = info;
}
void operator()(::mdp3::MDIncrementalRefreshOrderBook47 const &) {
if (result.incremental[0].port == 0) {
result.incremental[0].port = current_packet.dst_port;
result.incremental[0].ip = current_packet.dst_ip;
return;
}
if (result.incremental[0].port == current_packet.dst_port &&
result.incremental[0].ip == current_packet.dst_ip)
return;
if (result.incremental[1].port == 0) {
result.incremental[1].port = current_packet.dst_port;
result.incremental[1].ip = current_packet.dst_ip;
}
}
void operator()(::mdp3::SnapshotFullRefreshOrderBook53 const &) {
if (result.snapshot.port == 0) {
result.snapshot.port = current_packet.dst_port;
result.snapshot.ip = current_packet.dst_ip;
}
}
void operator()(::mdp3::MDInstrumentDefinitionFuture54 const &) {
on_definition();
}
void operator()(::mdp3::MDInstrumentDefinitionOption55 const &) {
on_definition();
}
void operator()(::mdp3::MDInstrumentDefinitionSpread56 const &) {
on_definition();
}
void on_definition() {
if (result.instrument.port)
return;
if (result.incremental[0].port == 0)
return;
if (result.incremental[1].port == 0)
return;
if (result.incremental[0].ip == current_packet.dst_ip &&
result.incremental[0].port == current_packet.dst_port)
return;
if (result.incremental[1].ip == current_packet.dst_ip &&
result.incremental[1].port == current_packet.dst_port)
return;
result.instrument.port = current_packet.dst_port;
result.instrument.ip = current_packet.dst_ip;
}
} guesser;
Guesser guesser;
auto reader = PCapReader{pcap_path};
reader.on_packet() = [&](auto const &info, auto const &payload) {
guesser.set_current_source(info);
parse_packet(MDP3PacketView {payload},guesser);
if (guesser.result)
parse_packet(MDP3PacketView{payload}, guesser);
if (guesser.result)
return false;
else
return true;
};
while (reader.read_next())
;
return guesser.result;
}
std::ostream &operator<<(std::ostream &o, FeedMap const &fm) {
o << "IA:" << fm.incremental[0].ip << ":" << fm.incremental[0].port
<< " IB:" << fm.incremental[1].ip << ":" << fm.incremental[1].port
<< " N:" << fm.instrument.ip << ":" << fm.instrument.port
<< " SMBO:" << fm.snapshot.ip << ":" << fm.snapshot.port;
return o;
}
bool slim_down_pcap(std::string const &file, std::string const &out_file) {
using namespace bts::spartan;
using namespace bts::spartan::mdp3;
struct Ctrl : IFeedController {
bool instrument_feed = false;
bool snapshot_feed = false;
bool incremental_feed = false;
void start_instrument_feed() override { instrument_feed = true; }
void stop_instrument_feed() override { instrument_feed = false; }
void start_snapshot_feed() override { snapshot_feed = true; }
void stop_snapshot_feed() override { snapshot_feed = false; }
void start_incremental_feed() override { incremental_feed = true; }
void stop_incremental_feed() override { incremental_feed = false; }
} ctrl;
struct Obs : IChannelObserver {
bool observed_loss = false;
bool recovery_complete = false;
size_t previous_packet = 0;
size_t previous_send_ts = 0;
TimeStamp last_print;
void on_recovery_complete(
std::vector<InstrumentSnapshot const *> const &) override {
recovery_complete = true;
}
void on_real_time_event(MDP3Event const &ev) override {
if (ms_diff(Timer::now(), last_print) >= 1000) {
last_print = Timer::now();
std::stringstream str;
cme_ts_to_time_str(ev.transact_time.value(), &str);
LOG(Spartan, Info,
"Time: {} -> {} pkt/sec at {}x real time",
str.str(),
(ev.first_packet_sn - previous_packet),
(ev.first_packet_send_time.value() - previous_send_ts) / 1000000000);
previous_packet = ev.first_packet_sn;
previous_send_ts = ev.first_packet_send_time.value();
}
}
void on_real_time_channel_loss() override { observed_loss = true; }
} obs;
struct IncrementalFeedBuffer {
bool written_once = false;
pcpp::RawPacket rp;
mdp3::MDP3Packet p;
};
std::array<IncrementalFeedBuffer, 2> inc_buffer;
auto feed_map = mdp3::guess_feed_map(file);
bts_assert(feed_map);
auto reader = PCapReader{file};
mdp3::AB_Arbitrageur ab;
mdp3::ChannelHandler handler(obs, ctrl);
pcpp::PcapNgFileWriterDevice writer(out_file.c_str());
writer.open();
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == feed_map.incremental[0].ip &&
info.dst_port == feed_map.incremental[0].port)) {
inc_buffer[0].rp = *info.rp;
inc_buffer[0].p.set_bytes(payload);
} else if ((info.dst_ip == feed_map.incremental[1].ip &&
info.dst_port == feed_map.incremental[1].port)) {
inc_buffer[1].rp = *info.rp;
inc_buffer[1].p.set_bytes(payload);
} else if (info.dst_ip == feed_map.instrument.ip &&
info.dst_port == feed_map.instrument.port) {
if (ctrl.instrument_feed) {
handler.on_instrument_feed(mdp3::MDP3PacketView{payload});
writer.writePacket(*info.rp);
}
return true;
} else if (info.dst_ip == feed_map.snapshot.ip &&
info.dst_port == feed_map.snapshot.port) {
if (ctrl.snapshot_feed) {
handler.on_snapshot_feed(mdp3::MDP3PacketView{payload});
writer.writePacket(*info.rp);
}
return true;
} else
return true;
// Incremental feed handling
auto picked = handler.on_incremental_feed(inc_buffer[0].p.view(),
inc_buffer[1].p.view());
if (picked == PickedFeed::A ||
((!inc_buffer[0].written_once || !obs.recovery_complete) &&
!inc_buffer[0].p.view().empty())) {
inc_buffer[0].written_once = true;
writer.writePacket(inc_buffer[0].rp);
}
if (picked == PickedFeed::B ||
((!inc_buffer[1].written_once || !obs.recovery_complete) &&
!inc_buffer[1].p.view().empty())) {
inc_buffer[1].written_once = true;
writer.writePacket(inc_buffer[1].rp);
}
return true;
};
while (reader.read_next())
if (obs.observed_loss) {
LOG(Spartan, Error, "Observed loss in {}. Cannot slim it down", file);
return false;
};
return true;
}
void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out) {
auto one_billion = 1000000000;
time_t unix_ts = cme_ts / one_billion;
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
int hours = boost_ts.time_of_day().hours();
int minutes = boost_ts.time_of_day().minutes();
int seconds = boost_ts.time_of_day().seconds();
*out << std::setw(2) << std::setfill('0') << hours << ":" << std::setw(2)
<< std::setfill('0') << minutes << ":" << std::setw(2)
<< std::setfill('0') << seconds << "." << std::setw(9)
<< cme_ts - (unix_ts * one_billion);
}
void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out) {
auto one_billion = 1000000000;
time_t unix_ts = cme_ts / one_billion;
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
auto year = boost_ts.date().year();
auto month = boost_ts.date().month().as_number();
auto day = boost_ts.date().day().as_number();
int hours = boost_ts.time_of_day().hours();
int minutes = boost_ts.time_of_day().minutes();
int seconds = boost_ts.time_of_day().seconds();
*out << year << "-" << std::setw(2) << std::setfill('0') << month << "-"
<< std::setw(2) << std::setw(2) << std::setfill('0') << day << "."
<< std::setw(2) << std::setw(2) << std::setfill('0') << hours << ":"
<< std::setw(2) << std::setfill('0') << minutes << ":" << std::setw(2)
<< std::setfill('0') << seconds << "." << std::setw(9)
<< cme_ts - (unix_ts * one_billion);
}
} // namespace bts::spartan::mdp3
@@ -23,15 +23,12 @@ struct FeedMap {
snapshot.port;
}
};
inline std::ostream &operator<<(std::ostream &o, FeedMap const &fm) {
o << "IA:" << fm.incremental[0].ip << ":" << fm.incremental[0].port
<< " IB:" << fm.incremental[1].ip << ":" << fm.incremental[1].port
<< " N:" << fm.instrument.ip << ":" << fm.instrument.port
<< " SMBO:" << fm.snapshot.ip << ":" << fm.snapshot.port;
return o;
}
std::ostream &operator<<(std::ostream &o, FeedMap const &fm);
FeedMap guess_feed_map(std::string const &pcap_path);
bool slim_down_pcap(std::string const&in, std::string const &out);
void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out);
void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out);
} // namespace bts::spartan::mdp3
@@ -1,11 +1,11 @@
#include "PCapReader.h"
#include "lib/utils/bts_assert.h"
#include <Pcap++/Packet.h>
#include <Pcap++/UdpLayer.h>
#include <Pcap++/TcpLayer.h>
#include <Pcap++/IPv4Layer.h>
#include <Pcap++/Packet.h>
#include <Pcap++/PcapFileDevice.h>
#include <Pcap++/TcpLayer.h>
#include <Pcap++/UdpLayer.h>
#include <arpa/inet.h>
namespace bts {
@@ -14,10 +14,10 @@ namespace spartan {
struct PCapReader::PImpl {
PImpl(std::string const &file_path)
: reader(pcpp::IFileReaderDevice::getReader(file_path.c_str())),
on_packet([](auto const&, auto const &) {return true;}) {}
on_packet([](auto const &, auto const &) { return true; }) {}
std::unique_ptr<pcpp::IFileReaderDevice> reader;
std::function<bool(PacketInfo const&, std::string_view const &)> on_packet;
OnPacket on_packet;
pcpp::RawPacket rawPacket;
};
@@ -31,19 +31,18 @@ PCapReader::PCapReader(std::string const &file_path, std::string const &filter)
PCapReader::~PCapReader() {}
std::function<bool(const PCapReader::PacketInfo &, std::string_view const &)> &PCapReader::on_packet() {
return me->on_packet;
}
PCapReader::OnPacket &PCapReader::on_packet() { return me->on_packet; }
bool PCapReader::read_next() {
if (me->reader->getNextPacket(me->rawPacket)) {
pcpp::Packet parsedPacket(&me->rawPacket);
PacketInfo info;
info.rp = &me->rawPacket;
auto udp = parsedPacket.getLayerOfType<pcpp::UdpLayer>();
if (udp)
info.dst_port = udp->getUdpHeader()->portDst;
else {
else {
auto tcp = parsedPacket.getLayerOfType<pcpp::TcpLayer>();
info.dst_port = tcp->getTcpHeader()->portDst;
}
@@ -52,12 +51,13 @@ bool PCapReader::read_next() {
info.dst_ip = ip->getDstIpAddress().toString();
auto payload = parsedPacket.getLayerOfType(pcpp::GenericPayload);
if (payload)
return me->on_packet(info, {(const char *)payload->getData(), payload->getDataLen()});
return me->on_packet(
info, {(const char *)payload->getData(), payload->getDataLen()});
else
return true;
} else
return false;
}
} // namespace utils
} // namespace spartan
} // namespace bts
@@ -1,26 +1,32 @@
#pragma once
#include <memory>
#include <functional>
#include <memory>
namespace pcpp {
class RawPacket;
}
namespace bts {
namespace spartan {
namespace bts::spartan {
class PCapReader {
public:
PCapReader(std::string const&file_path,
std::string const&filter = "");
public:
PCapReader(std::string const &file_path, std::string const &filter = "");
~PCapReader();
bool read_next();
struct PacketInfo {
std::string dst_ip;
int dst_port;
pcpp::RawPacket *rp;
};
std::function<bool(const PacketInfo&,std::string_view const&)> &on_packet();
private:
using OnPacket =
std::function<bool(const PacketInfo &, std::string_view const &)>;
OnPacket &on_packet();
private:
struct PImpl;
std::unique_ptr<PImpl> me;
};
}}
} // namespace bts::spartan