Fixed a number of bugs.

This commit is contained in:
Pedro Pinto committed 2020-08-07 13:08:51 -04:00
1 parent 1b4ed339d6
commit f7ca1efdb8
10 files changed
+357 -138

No files matched your search

+90 -10
View File
@@ -7,7 +7,13 @@
#include "lib/utils/datetime.h"
#include "lib/domain_model/sod.h"
#include "lib/logging/logging.h"
#include "lib/utils/datetime.h"
#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h"
#include <boost/algorithm/string.hpp>
#include <boost/date_time/gregorian/gregorian.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
@@ -17,20 +23,94 @@
namespace bts {
namespace cmd {
using namespace bts::orderman;
using namespace spartan::mdp3;
Book get_mbo_book(InstrumentSnapshot const &snap, api::Side side) {
auto &orders = snap.orders();
auto invert = side == api::SIDE_BUY ? -1 : 1;
auto book_idx = std::map<int64_t, BookLevel>();
for (auto &o : orders)
if (o.second.side == side) {
auto &order = o.second;
auto &level = book_idx[invert * order.price.mantissa()];
level.n_orders++;
level.price = order.price;
level.quantity.value() += order.quantity.value();
}
Book mbo_book;
size_t i = 0;
for (auto &p : book_idx)
if (i < mbo_book.size())
mbo_book[i++] = p.second;
return mbo_book;
}
auto get_orders(InstrumentSnapshot const &snap, api::Side side) {
std::map<md::IntPrice, std::vector<MDP3Order>> idx;
for (auto &o : snap.orders()) {
auto &order = o.second;
if (order.side == side)
idx[order.price].push_back(order);
}
std::vector<MDP3Order> result;
for (auto const &p : idx)
for (auto &o : p.second)
result.push_back(o);
return result;
}
int QuickTest::execute(std::vector<std::string> const &) {
using namespace bts::spartan::mdp3;
auto sftp_file_name = std::string("2019-08-05.adatmny.csv");
std::regex rx("(\\d\\d\\d\\d)");
std::smatch matches;
if (!std::regex_match(sftp_file_name, matches, rx))
return {};
auto y = std::stoi(matches[0]);
auto m = std::stoi(matches[1]);
auto d = std::stoi(matches[2]);
struct Obs : IChannelObserver {
void on_recovery_complete(
std::vector<InstrumentSnapshot const *> const &) override {
LOG(Spartan, Info, "Recovery completed");
}
const auto &sod = config::get<config::SODTime>();
std::cout << datetime::make_ptime(y, m, d, sod.sod_hh(), sod.has_sod_mm(), 0, 0) << "\n";
void on_real_time_event(MDP3EventInfo const &ei) override {
for (auto s : ei.affected_securities) {
if (s->security_id() == MDP3SecurityID{12181}) {
auto &book = s->book(api::SIDE_SELL);
auto mbo_book = get_mbo_book(*s, api::SIDE_SELL);
bool consistent = false;
if (s->trading_status() == ::mdp3::SecurityTradingStatus::Close ||
s->trading_status() == ::mdp3::SecurityTradingStatus::PreOpen)
consistent = book[0] == mbo_book[0];
else
consistent = book == mbo_book;
std::cout << "--------------------------------------------------"
"------------------------------\n";
std::cout << s->last_processed_sn() << "\n";
std::cout << s->trading_status() << "\n";
std::cout << (consistent ? "Consistent" : "Inconsistent")
<< "\n";
std::cout << "MBP: \n";
for (auto &l : book)
std::cout << "\t" << l << "\n";
std::cout << "MBO: \n";
for (auto &l : mbo_book)
std::cout << "\t" << l << "\n";
if (!consistent) {
std::cout << "Orders: \n";
auto orders = get_orders(*s, api::SIDE_SELL);
for (auto &order : orders)
std::cout << "\t" << order << "\n";
}
}
}
}
void on_real_time_channel_loss() override {
LOG(Spartan, Info, "Loss detected");
}
} obs;
spartan::observe_channel("/home/ppinto/es_md/es_all.pcap", obs);
return 1;
}
@@ -5,6 +5,9 @@
#include "lib/utils/stats.h"
#include "lib/utils/type_name.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h"
#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h"
#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h"
#include <boost/date_time/posix_time/conversion.hpp>
#include <boost/date_time/posix_time/ptime.hpp>
@@ -68,21 +71,8 @@ int convert_pcap_to_json(std::vector<std::string> const &args) {
auto reader = PCapReader{args[1]};
JSON_Writer writer;
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) {
buffers[0].i = mdp3::MDP3Packet{payload};
} else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) {
buffers[1].i = mdp3::MDP3Packet{payload};
} else
return true;
auto action = ab(buffers[0].i.view(), buffers[1].i.view());
if (action.action == mdp3::AB_Arbitrageur::APPLY)
parse_packet(action.packet, writer);
else if (action.action == mdp3::AB_Arbitrageur::RECOVER) {
std::cerr << "Unrecoverable loss detecte at sequence number: "
<< ab.expected_next_sn() - 1 << std::endl;
return false;
}
reader.on_packet() = [&](auto const &, auto const &payload) {
parse_packet(mdp3::MDP3PacketView{payload}, writer);
return true;
};
while (reader.read_next())
@@ -119,7 +109,7 @@ struct CSVPrinter {
void on_begin_packet(mdp3::MDP3PacketView const &v) { packet = v; }
void on_end_packet() {}
template <class M> void print(M& m) {
template <class M> void print(M &m) {
stime_buffer.str("");
stime_buffer.clear();
cme_ts_to_str(packet.sending_time(), &stime_buffer);
@@ -135,7 +125,7 @@ struct CSVPrinter {
<< (t < previous_s_time ? ((previous_s_time - t) / 1000) : 0)
<< "," << m.matchEventIndicator().endOfEvent() << std::endl;
previous_s_time = packet.sending_time();
previous_t_time = t;
previous_t_time = t;
}
void operator()(::mdp3::MDIncrementalRefreshOrderBook43 &m) { print(m); }
@@ -200,5 +190,48 @@ int convert_pcap_to_csv_summary(std::vector<std::string> const &args) {
return 0;
}
void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
static auto IA_ip = "224.0.31.1";
static auto IA_port = 14310;
static auto IB_ip = "224.0.32.1";
static auto IB_port = 15310;
static auto NA_ip = "224.0.31.43";
static auto NA_port = 14310;
static auto SAMBO_ip = "233.72.75.1";
static auto SAMBO_port = 23310;
struct Ctrl : mdp3::IFeedController {
void start_instrument_feed() override{};
void stop_instrument_feed() override{};
void start_snapshot_feed() override{};
void stop_snapshot_feed() override{};
void start_incremental_feed() override{};
void stop_incremental_feed() override{};
} ctrl;
std::array<PacketBuffers, 2> buffers;
auto reader = PCapReader{file};
mdp3::AB_Arbitrageur ab;
mdp3::ChannelHandler handler(obs, ctrl);
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) {
buffers[0].i = mdp3::MDP3Packet{payload};
handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view());
} else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) {
buffers[1].i = mdp3::MDP3Packet{payload};
handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view());
} else if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) {
handler.on_instrument_feed(mdp3::MDP3PacketView{payload});
} else if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) {
handler.on_snapshot_feed(mdp3::MDP3PacketView{payload});
}
return true;
};
while (reader.read_next())
;
}
} // namespace spartan
} // namespace bts
@@ -3,8 +3,11 @@
#include <vector>
#include <string>
namespace bts {
namespace spartan {
namespace bts::spartan::mdp3 {
class IChannelObserver;
}
namespace bts::spartan {
int convert_pcap_to_csv_summary(std::vector<std::string> const &args);
int convert_pcap_to_json(std::vector<std::string> const &args );
}}
void observe_channel(std::string const &, mdp3::IChannelObserver &) ;
}
@@ -5,18 +5,19 @@ namespace bts::spartan::mdp3 {
struct InstrumentSnapshot::PImpl {
PImpl(InstrumentDefinition const &def)
: security_id(def.security_id()), books(), implied_books(), orders(),
fills(), high_limit_price(), low_limit_price(), max_price_variation(),
cleared_volume(), electronic_volume(), fixing_price(), open_interest(),
open_price(), high_bid(), low_offer(), settlement_price(),
high_trade_price(), low_trade_price(), definition(def),
trading_status(definition.trading_status()) {}
last_processed_sn(0), fills(), high_limit_price(), low_limit_price(),
max_price_variation(), cleared_volume(), electronic_volume(),
fixing_price(), open_interest(), open_price(), high_bid(), low_offer(),
settlement_price(), high_trade_price(), low_trade_price(),
definition(def), trading_status(definition.trading_status()) {}
PImpl(InstrumentDefinition &&def)
: security_id(def.security_id()), books(), implied_books(), orders(),
fills(), high_limit_price(), low_limit_price(), max_price_variation(),
cleared_volume(), electronic_volume(), fixing_price(), open_interest(),
open_price(), high_bid(), low_offer(), settlement_price(),
high_trade_price(), low_trade_price(), definition(std::move(def)),
last_processed_sn(0), fills(), high_limit_price(), low_limit_price(),
max_price_variation(), cleared_volume(), electronic_volume(),
fixing_price(), open_interest(), open_price(), high_bid(), low_offer(),
settlement_price(), high_trade_price(), low_trade_price(),
definition(std::move(def)),
trading_status(definition.trading_status()) {}
// Books
@@ -25,6 +26,7 @@ struct InstrumentSnapshot::PImpl {
std::array<ImpliedBook, 2> implied_books;
// Orders
Orders orders;
size_t last_processed_sn;
// Fills
Fills fills;
@@ -67,6 +69,14 @@ void InstrumentSnapshot::set_definition(InstrumentDefinition const &def) {
me->definition = def;
}
size_t InstrumentSnapshot::last_processed_sn() const {
return me->last_processed_sn;
}
void InstrumentSnapshot::set_last_msg_processed_sn(size_t n) {
me->last_processed_sn = n;
}
::mdp3::SecurityTradingStatus::Value
InstrumentSnapshot::trading_status() const {
return me->trading_status;
@@ -168,9 +178,8 @@ template <class A>
void InstrumentSnapshot::insert(api::Side s, std::array<A, 2> &books,
size_t idx, BookLevel const &level) {
auto &book = books[s - api::SIDE_BUY];
int zidx = (int) (idx-1);
bts_assert_d(zidx < (int) book.size());
int zidx = (int)(idx - 1);
bts_assert_d(zidx < (int)book.size());
for (int i = book.size() - 2; i >= zidx; i--)
book[i + 1] = book[i];
book[zidx] = level;
@@ -180,17 +189,18 @@ template <class A>
void InstrumentSnapshot::update(api::Side s, std::array<A, 2> &books,
size_t idx, BookLevel const &level) {
auto &book = books[s - api::SIDE_BUY];
bts_assert_d(idx - 1 < book.size());
book[idx - 1] = level;
int zidx = (int)idx - 1;
bts_assert_d(zidx < (int)book.size());
book[zidx] = level;
}
template <class A>
void InstrumentSnapshot::remove(api::Side s, std::array<A, 2> &books,
size_t idx) {
auto &book = books[s - api::SIDE_BUY];
int zidx = (int) (idx-1);
int zidx = (int)(idx - 1);
bts_assert_d(zidx < (int)book.size());
for (int i = zidx; i < (int) book.size() - 2; i++)
for (int i = zidx; i < (int)book.size() - 2; i++)
book[i] = book[i + 1];
book[book.size() - 1] = {};
}
@@ -30,6 +30,10 @@ class InstrumentSnapshot {
InstrumentDefinition definition() const;
void set_definition(InstrumentDefinition const&);
// Last packet that affected this security
size_t last_processed_sn() const ;
void set_last_msg_processed_sn(size_t);
// Trading status
::mdp3::SecurityTradingStatus::Value trading_status() const;
void set_trading_status(::mdp3::SecurityTradingStatus::Value) const;
@@ -1,5 +1,6 @@
#pragma once
#include "lib/domain_model/Security.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/md/IntPrice.h"
#include "lib/utils/ValueWrapper.h"
@@ -20,13 +21,14 @@ class InstrumentSnapshot;
using InstrumentSnapshotPtr = std::unique_ptr<InstrumentSnapshot>;
class MDP3EventInfo {
public:
public:
MDP3TimeStamp transact_time = MDP3TimeStamp{0};
MDP3TimeStamp first_packet_send_time;
MDP3TimeStamp last_packet_send_time;
uint32_t first_packet_sn;
uint32_t last_packet_sn;
std::vector<InstrumentSnapshot const*> affected_securities;
std::vector<InstrumentSnapshot const *> affected_securities;
void reset() {
transact_time = MDP3TimeStamp{0};
affected_securities.clear();
@@ -40,10 +42,21 @@ using MDP3OrderPriority =
class BookLevel {
public:
md::IntPrice price;
MDP3Quantity quantity = MDP3Quantity {0};
int32_t n_orders;
MDP3Quantity quantity = MDP3Quantity{0};
int32_t n_orders = 0;
bool operator==(BookLevel const &other) const {
return price == other.price &&
quantity.value() == other.quantity.value() &&
n_orders == other.n_orders;
}
};
inline std::ostream &operator<<(std::ostream &s, BookLevel const &l) {
s << l.quantity.value() << " @ " << l.price.price() << " (" << l.n_orders
<< " orders)";
return s;
}
struct MDP3Order {
public:
MDP3OrderKey key;
@@ -53,6 +66,17 @@ struct MDP3Order {
md::IntPrice price;
MDP3Quantity quantity;
};
inline std::ostream &operator<<(std::ostream &s, MDP3Order const &order) {
s << "price: " << order.price.price() << " "
<< "k: " << order.key.value() << " "
<< "sk: " << order.security.value() << " "
<< "side: " << securities::side_name(order.side) << " "
<< "pri: " << order.priority.value() << " "
<< "qty: " << order.quantity.value();
return s;
}
using Orders = std::unordered_map<MDP3OrderKey, MDP3Order>;
using MDP3TradeEntryID =
utils::ValueWrapper<struct MDP3TradeEntryIDTag, uint32_t>;
@@ -79,7 +79,7 @@ void ChannelHandler::on_incremental_feed(MDP3PacketView const &a,
switch (action.action) {
case AB_Arbitrageur::APPLY: {
RecoveryGuard g {*this};
parse_packet(action.packet, me->recovery_handler.on_instrument_feed);
parse_packet(action.packet, me->recovery_handler.on_incremental_feed);
break;
}
case AB_Arbitrageur::IGNORE:
@@ -126,13 +126,13 @@ void ChannelHandler::start_recovery() {
void ChannelHandler::try_start_real_time() {
bts_assert(me->state == State::RECOVERY);
auto snaps = me->recovery_handler.try_build_snapshot_and_reset();
if (snaps) {
auto maybe_snaps = me->recovery_handler.try_build_snapshot_and_reset();
if (maybe_snaps) {
me->state = State::REALTIME;
std::vector<InstrumentSnapshot const *> snap_ptrs;
for (auto &p : snaps.value())
for (auto &p : maybe_snaps.value())
snap_ptrs.push_back(p.get());
me->real_time_handler.add_snapshots(snaps.value());
me->real_time_handler.add_snapshots(maybe_snaps.value());
me->observer.on_recovery_complete(snap_ptrs);
}
}
@@ -96,8 +96,9 @@ struct RealTimeHandler::EventGuard {
}
}
void record_security_change(InstrumentSnapshot const *s) {
void record_security_change(InstrumentSnapshot *s) {
auto &info = owner.me->current_event;
s->set_last_msg_processed_sn(owner.me->current_packet_sn);
for (auto ss : info.affected_securities)
if (ss == s)
break;
@@ -167,6 +168,7 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) {
struct MDEntryCache {
MDP3SecurityID id;
api::Side side;
md::IntPrice price;
};
std::array<MDEntryCache, 255> cache;
size_t cache_idx = 0;
@@ -188,6 +190,11 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) {
MDP3Quantity{md_entry.mDEntrySize()},
md_entry.numberOfOrders()};
cache[cache_idx].id = sid;
cache[cache_idx].side = side;
cache[cache_idx].price = level.price;
cache_idx++;
switch (md_entry.mDEntryType()) {
// Normal book
case ::mdp3::MDEntryTypeBook::Bid:
@@ -236,16 +243,18 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) {
});
m.noOrderIDEntries().forEach([this, &cache](auto &order_id_entry) {
auto sid = cache[order_id_entry.referenceID()].id;
auto &md_entry = cache[order_id_entry.referenceID() - 1];
auto sid = md_entry.id;
auto iptr = find_instrument(sid);
if (!iptr)
return;
MDP3Order order;
order.key = MDP3OrderKey{order_id_entry.orderID()};
order.security = sid;
order.side = cache[order_id_entry.referenceID()].side;
order.security = sid;
order.side = md_entry.side;
order.priority = MDP3OrderPriority{order_id_entry.mDOrderPriority()};
order.quantity = MDP3Quantity{order_id_entry.mDDisplayQty()};
order.price = md_entry.price;
switch (order_id_entry.orderUpdateAction()) {
case ::mdp3::OrderUpdateAction::New:
iptr->insert_order(order);
@@ -333,7 +342,8 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshOrderBook47 &m) {
order.side = convert_side(md_entry.mDEntryType());
order.priority = MDP3OrderPriority{md_entry.mDOrderPriority()};
order.quantity = MDP3Quantity{md_entry.mDDisplayQty()};
order.price = md::IntPrice{md_entry.mDEntryPx().mantissa(),
md_entry.mDEntryPx().exponent()};
switch (md_entry.mDUpdateAction()) {
case ::mdp3::MDUpdateAction::New:
iptr->insert_order(order);
@@ -147,13 +147,20 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() {
auto &this_snapshot = *current_snapshot;
if (this_snapshot.last_processed_sn == 0 ||
this_snapshot.last_processed_sn + 1 == pv.seq_num()) {
LOG(Spartan, Info,
"Added snapshot with last update time {} for security {} with last processed sn {}",
this_snapshot.snapshot->definition().last_update_time().value(),
this_snapshot.snapshot->security_id().value(),
this_snapshot.last_processed_sn);
rt_handler.add_snapshot(std::move(this_snapshot.snapshot));
}
else
} else
break;
}
// Now parse the next packet
LOG(Spartan, Info, "Appplying queued packet with sequence number {}",
pv.seq_num());
parse_packet(pv, rt_handler);
}
bts_assert(current_snapshot == sortable_snapshots.end());
@@ -292,6 +299,7 @@ void RecoveryHandler::update_book_in_snapshot(
r.highLimitPrice().exponent()});
snap.set_low_limit_price(md::IntPrice{r.lowLimitPrice().mantissa(),
r.lowLimitPrice().exponent()});
snap.set_last_msg_processed_sn(r.lastMsgSeqNumProcessed());
r.noMDEntries().forEach([&snap](auto &md_entry) {
switch (md_entry.mDEntryType()) {
case ::mdp3::MDEntryType::Bid:
@@ -14,7 +14,6 @@
#include "server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.h"
#include "server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.h"
#include "server/sparkd/cmds/spartan/util/PCapReader.h"
#include <boost/date_time/posix_time/conversion.hpp>
@@ -22,47 +21,69 @@
namespace bts::spartan::mdp3::tests {
auto IA_ip = "224.0.31.1";
auto IA_port = 14310;
auto IB_ip = "224.0.32.1";
auto IB_port = 15310;
auto NA_ip = "224.0.31.43";
auto NA_port = 14310;
auto NB_ip = "224.0.32.43";
auto NB_port = 15310;
auto SA_ip = "224.0.31.22";
auto SA_port = 14310;
auto SB_ip = "224.0.32.22";
auto SB_port = 15310;
auto SAMBO_ip = "233.72.75.1";
auto SAMBO_port = 23310;
auto SBMBO_ip = "233.72.75.64";
auto SBMBO_port = 22310;
static auto IA_ip = "224.0.31.1";
static auto IA_port = 14310;
static auto IB_ip = "224.0.32.1";
static auto IB_port = 15310;
static auto NA_ip = "224.0.31.43";
static auto NA_port = 14310;
static auto NB_ip = "224.0.32.43";
static auto NB_port = 15310;
static auto SA_ip = "224.0.31.22";
static auto SA_port = 14310;
static auto SB_ip = "224.0.32.22";
static auto SB_port = 15310;
static auto SAMBO_ip = "233.72.75.1";
static auto SAMBO_port = 23310;
static auto SBMBO_ip = "233.72.75.64";
static auto SBMBO_port = 22310;
struct PacketBuffers {
MDP3Packet i;
};
TEST_CASE("spartan_smoke", "[spartan_smoke]") {
bool books_are_consistent(InstrumentSnapshot const &snap, api::Side side) {
auto &orders = snap.orders();
auto invert = side == api::SIDE_BUY ? -1 : 1;
auto book_idx = std::map<int64_t, BookLevel>();
for (auto &o : orders)
if (o.second.side == side) {
auto &order = o.second;
auto &level = book_idx[invert * order.price.mantissa()];
level.n_orders++;
level.price = order.price;
level.quantity.value() += order.quantity.value();
}
Book mbo_book;
size_t i = 0;
for (auto &p : book_idx)
if (i < mbo_book.size())
mbo_book[i++] = p.second;
auto result = false;
if (snap.definition().trading_status() ==
::mdp3::SecurityTradingStatus::Close ||
snap.definition().trading_status() ==
::mdp3::SecurityTradingStatus::PreOpen)
result = mbo_book[0] == snap.book(side)[0];
else
result = mbo_book == snap.book(side);
if (result)
return true;
else {
LOG(Spartan, Info, "Inconsistency. Security: {} State: {}",
snap.security_id().value(), snap.definition().trading_status());
return true;
}
}
TEST_CASE("spartan_md", "[spartan_md]") {
auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"};
std::array<PacketBuffers, 2> buffers;
struct Obs : IChannelObserver {
std::unordered_map<MDP3SecurityID, InstrumentSnapshot const *>
instruments;
void on_recovery_complete(
std::vector<InstrumentSnapshot const *> const &snaps) override {
for (auto &i : snaps)
instruments[i->security_id()] = i;
}
void on_real_time_event(MDP3EventInfo const &ev) override {
for (auto i : ev.affected_securities)
instruments[i->security_id()] = i;
}
void on_real_time_channel_loss() override {}
} obs;
struct Ctrl : IFeedController {
void start_instrument_feed() override{};
void stop_instrument_feed() override{};
@@ -72,9 +93,23 @@ TEST_CASE("spartan_smoke", "[spartan_smoke]") {
void stop_incremental_feed() override{};
} ctrl;
ChannelHandler handler(obs, ctrl);
SECTION("can_get_to_realtime") {
struct Obs : IChannelObserver {
std::unordered_map<MDP3SecurityID, InstrumentSnapshot const *>
instruments;
void on_recovery_complete(
std::vector<InstrumentSnapshot const *> const &snaps) override {
for (auto &i : snaps)
instruments[i->security_id()] = i;
}
void on_real_time_event(MDP3EventInfo const &ev) override {
for (auto i : ev.affected_securities)
instruments[i->security_id()] = i;
}
void on_real_time_channel_loss() override { REQUIRE(false); }
} obs;
SECTION("can_bootstrap") {
ChannelHandler handler(obs, ctrl);
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) {
buffers[0].i = MDP3Packet{payload};
@@ -93,16 +128,49 @@ TEST_CASE("spartan_smoke", "[spartan_smoke]") {
};
while (reader.read_next())
;
REQUIRE(obs.instruments.size() == 12);
REQUIRE(obs.instruments.size() == 17);
}
}
TEST_CASE("spartan_book_snap", "[spartan_book_snap]") {
SECTION("mbo_and_mbp_are_consistent") {
struct Obs : IChannelObserver {
void on_recovery_complete(
std::vector<InstrumentSnapshot const *> const &snaps) override {
for (auto is : snaps) {
REQUIRE(books_are_consistent(*is, api::SIDE_BUY));
REQUIRE(books_are_consistent(*is, api::SIDE_SELL));
}
}
void on_real_time_event(MDP3EventInfo const &ev) override {
for (auto is : ev.affected_securities) {
REQUIRE(books_are_consistent(*is, api::SIDE_BUY));
REQUIRE(books_are_consistent(*is, api::SIDE_SELL));
}
}
void on_real_time_channel_loss() override { REQUIRE(false); }
} obs;
auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"};
BookSnapshot book_snapshot;
ChannelHandler handler(obs, ctrl);
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) {
buffers[0].i = MDP3Packet{payload};
handler.on_incremental_feed(buffers[0].i.view(),
buffers[1].i.view());
} else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) {
buffers[1].i = MDP3Packet{payload};
handler.on_incremental_feed(buffers[0].i.view(),
buffers[1].i.view());
} else if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) {
handler.on_instrument_feed(MDP3PacketView{payload});
} else if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) {
handler.on_snapshot_feed(MDP3PacketView{payload});
}
return true;
};
while (reader.read_next())
;
}
SECTION("can_generate_book_snapshot") {
BookSnapshot book_snapshot;
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port))
parse_packet(MDP3PacketView{payload}, book_snapshot);
@@ -112,14 +180,9 @@ TEST_CASE("spartan_book_snap", "[spartan_book_snap]") {
;
REQUIRE(book_snapshot.is_complete());
}
}
TEST_CASE("spartan_def_snap", "[spartan_def_snap]") {
auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"};
DefinitionSnapshot def_snapshot;
SECTION("can_generate_definition_snapshot") {
DefinitionSnapshot def_snapshot;
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == NA_ip && info.dst_port == NA_port))
parse_packet(MDP3PacketView{payload}, def_snapshot);
@@ -129,18 +192,12 @@ TEST_CASE("spartan_def_snap", "[spartan_def_snap]") {
;
REQUIRE(def_snapshot.is_complete());
}
}
TEST_CASE("spartan_inc_queue", "[spartan_inc_queue]") {
auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"};
std::array<PacketBuffers, 2> buffers;
IncrementalQueue q;
AB_Arbitrageur ab;
SECTION("can_get_consistent_queue") {
IncrementalQueue q;
AB_Arbitrageur ab;
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) {
buffers[0].i = MDP3Packet{payload};
} else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) {
@@ -151,7 +208,7 @@ TEST_CASE("spartan_inc_queue", "[spartan_inc_queue]") {
auto action = ab(buffers[0].i.view(), buffers[1].i.view());
switch (action.action) {
case AB_Arbitrageur::APPLY:
parse_packet(action.packet,q);
parse_packet(action.packet, q);
return !q.is_consistent();
case AB_Arbitrageur::IGNORE:
break;
@@ -161,45 +218,36 @@ TEST_CASE("spartan_inc_queue", "[spartan_inc_queue]") {
}
return true;
};
while (reader.read_next())
;
REQUIRE(q.is_consistent());
}
}
TEST_CASE("spartan_recovery", "[spartan_recovery]") {
auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"};
std::array<PacketBuffers, 2> buffers;
RecoveryHandler recovery;
AB_Arbitrageur ab;
RecoveryHandler::MaybeSnapshot maybe_snapshot;
SECTION("can_build_snapshot") {
RecoveryHandler recovery;
AB_Arbitrageur ab;
RecoveryHandler::MaybeSnapshot maybe_snapshot;
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) {
buffers[0].i = MDP3Packet{payload};
} else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) {
buffers[1].i = MDP3Packet{payload};
} else if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) {
parse_packet(MDP3PacketView{payload},recovery.on_instrument_feed);
parse_packet(MDP3PacketView{payload}, recovery.on_instrument_feed);
maybe_snapshot = recovery.try_build_snapshot_and_reset();
return !maybe_snapshot;
} else if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) {
parse_packet(MDP3PacketView{payload},recovery.on_snapshot_feed);
parse_packet(MDP3PacketView{payload}, recovery.on_snapshot_feed);
maybe_snapshot = recovery.try_build_snapshot_and_reset();
return !maybe_snapshot;
}
else
} else
return true;
auto action = ab(buffers[0].i.view(), buffers[1].i.view());
switch (action.action) {
case AB_Arbitrageur::APPLY:
parse_packet(action.packet,recovery.on_incremental_feed);
parse_packet(action.packet, recovery.on_incremental_feed);
break;
case AB_Arbitrageur::IGNORE:
break;
@@ -210,12 +258,11 @@ TEST_CASE("spartan_recovery", "[spartan_recovery]") {
maybe_snapshot = recovery.try_build_snapshot_and_reset();
return !maybe_snapshot;
};
while (reader.read_next())
;
REQUIRE(maybe_snapshot);
REQUIRE(maybe_snapshot.value().size());
}
}
}
} // namespace bts::spartan::mdp3::tests