Fixed a number of bugs.
This commit is contained in:
1 parent
1b4ed339d6
commit
f7ca1efdb8
10 files changed
+357
-138
No files matched your search
@@ -7,7 +7,13 @@
|
||||
#include "lib/utils/datetime.h"
|
||||
|
||||
#include "lib/domain_model/sod.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/utils/datetime.h"
|
||||
#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h"
|
||||
|
||||
#include <boost/algorithm/string.hpp>
|
||||
#include <boost/date_time/gregorian/gregorian.hpp>
|
||||
#include <boost/date_time/posix_time/posix_time.hpp>
|
||||
@@ -17,20 +23,94 @@
|
||||
namespace bts {
|
||||
namespace cmd {
|
||||
using namespace bts::orderman;
|
||||
using namespace spartan::mdp3;
|
||||
|
||||
Book get_mbo_book(InstrumentSnapshot const &snap, api::Side side) {
|
||||
auto &orders = snap.orders();
|
||||
auto invert = side == api::SIDE_BUY ? -1 : 1;
|
||||
auto book_idx = std::map<int64_t, BookLevel>();
|
||||
for (auto &o : orders)
|
||||
if (o.second.side == side) {
|
||||
auto &order = o.second;
|
||||
auto &level = book_idx[invert * order.price.mantissa()];
|
||||
level.n_orders++;
|
||||
level.price = order.price;
|
||||
level.quantity.value() += order.quantity.value();
|
||||
}
|
||||
|
||||
Book mbo_book;
|
||||
size_t i = 0;
|
||||
for (auto &p : book_idx)
|
||||
if (i < mbo_book.size())
|
||||
mbo_book[i++] = p.second;
|
||||
|
||||
return mbo_book;
|
||||
}
|
||||
|
||||
auto get_orders(InstrumentSnapshot const &snap, api::Side side) {
|
||||
std::map<md::IntPrice, std::vector<MDP3Order>> idx;
|
||||
for (auto &o : snap.orders()) {
|
||||
auto &order = o.second;
|
||||
if (order.side == side)
|
||||
idx[order.price].push_back(order);
|
||||
}
|
||||
std::vector<MDP3Order> result;
|
||||
for (auto const &p : idx)
|
||||
for (auto &o : p.second)
|
||||
result.push_back(o);
|
||||
return result;
|
||||
}
|
||||
|
||||
int QuickTest::execute(std::vector<std::string> const &) {
|
||||
using namespace bts::spartan::mdp3;
|
||||
|
||||
auto sftp_file_name = std::string("2019-08-05.adatmny.csv");
|
||||
std::regex rx("(\\d\\d\\d\\d)");
|
||||
std::smatch matches;
|
||||
if (!std::regex_match(sftp_file_name, matches, rx))
|
||||
return {};
|
||||
auto y = std::stoi(matches[0]);
|
||||
auto m = std::stoi(matches[1]);
|
||||
auto d = std::stoi(matches[2]);
|
||||
struct Obs : IChannelObserver {
|
||||
void on_recovery_complete(
|
||||
std::vector<InstrumentSnapshot const *> const &) override {
|
||||
LOG(Spartan, Info, "Recovery completed");
|
||||
}
|
||||
|
||||
const auto &sod = config::get<config::SODTime>();
|
||||
std::cout << datetime::make_ptime(y, m, d, sod.sod_hh(), sod.has_sod_mm(), 0, 0) << "\n";
|
||||
void on_real_time_event(MDP3EventInfo const &ei) override {
|
||||
for (auto s : ei.affected_securities) {
|
||||
if (s->security_id() == MDP3SecurityID{12181}) {
|
||||
auto &book = s->book(api::SIDE_SELL);
|
||||
auto mbo_book = get_mbo_book(*s, api::SIDE_SELL);
|
||||
|
||||
bool consistent = false;
|
||||
if (s->trading_status() == ::mdp3::SecurityTradingStatus::Close ||
|
||||
s->trading_status() == ::mdp3::SecurityTradingStatus::PreOpen)
|
||||
consistent = book[0] == mbo_book[0];
|
||||
else
|
||||
consistent = book == mbo_book;
|
||||
|
||||
std::cout << "--------------------------------------------------"
|
||||
"------------------------------\n";
|
||||
std::cout << s->last_processed_sn() << "\n";
|
||||
std::cout << s->trading_status() << "\n";
|
||||
std::cout << (consistent ? "Consistent" : "Inconsistent")
|
||||
<< "\n";
|
||||
std::cout << "MBP: \n";
|
||||
for (auto &l : book)
|
||||
std::cout << "\t" << l << "\n";
|
||||
std::cout << "MBO: \n";
|
||||
for (auto &l : mbo_book)
|
||||
std::cout << "\t" << l << "\n";
|
||||
if (!consistent) {
|
||||
std::cout << "Orders: \n";
|
||||
auto orders = get_orders(*s, api::SIDE_SELL);
|
||||
for (auto &order : orders)
|
||||
std::cout << "\t" << order << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void on_real_time_channel_loss() override {
|
||||
LOG(Spartan, Info, "Loss detected");
|
||||
}
|
||||
} obs;
|
||||
|
||||
spartan::observe_channel("/home/ppinto/es_md/es_all.pcap", obs);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
#include "lib/utils/stats.h"
|
||||
#include "lib/utils/type_name.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h"
|
||||
|
||||
#include <boost/date_time/posix_time/conversion.hpp>
|
||||
#include <boost/date_time/posix_time/ptime.hpp>
|
||||
@@ -68,21 +71,8 @@ int convert_pcap_to_json(std::vector<std::string> const &args) {
|
||||
auto reader = PCapReader{args[1]};
|
||||
JSON_Writer writer;
|
||||
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) {
|
||||
buffers[0].i = mdp3::MDP3Packet{payload};
|
||||
} else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) {
|
||||
buffers[1].i = mdp3::MDP3Packet{payload};
|
||||
} else
|
||||
return true;
|
||||
auto action = ab(buffers[0].i.view(), buffers[1].i.view());
|
||||
if (action.action == mdp3::AB_Arbitrageur::APPLY)
|
||||
parse_packet(action.packet, writer);
|
||||
else if (action.action == mdp3::AB_Arbitrageur::RECOVER) {
|
||||
std::cerr << "Unrecoverable loss detecte at sequence number: "
|
||||
<< ab.expected_next_sn() - 1 << std::endl;
|
||||
return false;
|
||||
}
|
||||
reader.on_packet() = [&](auto const &, auto const &payload) {
|
||||
parse_packet(mdp3::MDP3PacketView{payload}, writer);
|
||||
return true;
|
||||
};
|
||||
while (reader.read_next())
|
||||
@@ -119,7 +109,7 @@ struct CSVPrinter {
|
||||
void on_begin_packet(mdp3::MDP3PacketView const &v) { packet = v; }
|
||||
void on_end_packet() {}
|
||||
|
||||
template <class M> void print(M& m) {
|
||||
template <class M> void print(M &m) {
|
||||
stime_buffer.str("");
|
||||
stime_buffer.clear();
|
||||
cme_ts_to_str(packet.sending_time(), &stime_buffer);
|
||||
@@ -135,7 +125,7 @@ struct CSVPrinter {
|
||||
<< (t < previous_s_time ? ((previous_s_time - t) / 1000) : 0)
|
||||
<< "," << m.matchEventIndicator().endOfEvent() << std::endl;
|
||||
previous_s_time = packet.sending_time();
|
||||
previous_t_time = t;
|
||||
previous_t_time = t;
|
||||
}
|
||||
|
||||
void operator()(::mdp3::MDIncrementalRefreshOrderBook43 &m) { print(m); }
|
||||
@@ -200,5 +190,48 @@ int convert_pcap_to_csv_summary(std::vector<std::string> const &args) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
|
||||
static auto IA_ip = "224.0.31.1";
|
||||
static auto IA_port = 14310;
|
||||
static auto IB_ip = "224.0.32.1";
|
||||
static auto IB_port = 15310;
|
||||
static auto NA_ip = "224.0.31.43";
|
||||
static auto NA_port = 14310;
|
||||
static auto SAMBO_ip = "233.72.75.1";
|
||||
static auto SAMBO_port = 23310;
|
||||
struct Ctrl : mdp3::IFeedController {
|
||||
void start_instrument_feed() override{};
|
||||
void stop_instrument_feed() override{};
|
||||
void start_snapshot_feed() override{};
|
||||
void stop_snapshot_feed() override{};
|
||||
void start_incremental_feed() override{};
|
||||
void stop_incremental_feed() override{};
|
||||
} ctrl;
|
||||
std::array<PacketBuffers, 2> buffers;
|
||||
|
||||
auto reader = PCapReader{file};
|
||||
|
||||
mdp3::AB_Arbitrageur ab;
|
||||
mdp3::ChannelHandler handler(obs, ctrl);
|
||||
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) {
|
||||
buffers[0].i = mdp3::MDP3Packet{payload};
|
||||
handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view());
|
||||
} else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) {
|
||||
buffers[1].i = mdp3::MDP3Packet{payload};
|
||||
handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view());
|
||||
} else if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) {
|
||||
handler.on_instrument_feed(mdp3::MDP3PacketView{payload});
|
||||
} else if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) {
|
||||
handler.on_snapshot_feed(mdp3::MDP3PacketView{payload});
|
||||
}
|
||||
return true;
|
||||
};
|
||||
while (reader.read_next())
|
||||
;
|
||||
}
|
||||
|
||||
|
||||
} // namespace spartan
|
||||
} // namespace bts
|
||||
@@ -3,8 +3,11 @@
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
namespace bts {
|
||||
namespace spartan {
|
||||
namespace bts::spartan::mdp3 {
|
||||
class IChannelObserver;
|
||||
}
|
||||
namespace bts::spartan {
|
||||
int convert_pcap_to_csv_summary(std::vector<std::string> const &args);
|
||||
int convert_pcap_to_json(std::vector<std::string> const &args );
|
||||
}}
|
||||
void observe_channel(std::string const &, mdp3::IChannelObserver &) ;
|
||||
}
|
||||
@@ -5,18 +5,19 @@ namespace bts::spartan::mdp3 {
|
||||
struct InstrumentSnapshot::PImpl {
|
||||
PImpl(InstrumentDefinition const &def)
|
||||
: security_id(def.security_id()), books(), implied_books(), orders(),
|
||||
fills(), high_limit_price(), low_limit_price(), max_price_variation(),
|
||||
cleared_volume(), electronic_volume(), fixing_price(), open_interest(),
|
||||
open_price(), high_bid(), low_offer(), settlement_price(),
|
||||
high_trade_price(), low_trade_price(), definition(def),
|
||||
trading_status(definition.trading_status()) {}
|
||||
last_processed_sn(0), fills(), high_limit_price(), low_limit_price(),
|
||||
max_price_variation(), cleared_volume(), electronic_volume(),
|
||||
fixing_price(), open_interest(), open_price(), high_bid(), low_offer(),
|
||||
settlement_price(), high_trade_price(), low_trade_price(),
|
||||
definition(def), trading_status(definition.trading_status()) {}
|
||||
|
||||
PImpl(InstrumentDefinition &&def)
|
||||
: security_id(def.security_id()), books(), implied_books(), orders(),
|
||||
fills(), high_limit_price(), low_limit_price(), max_price_variation(),
|
||||
cleared_volume(), electronic_volume(), fixing_price(), open_interest(),
|
||||
open_price(), high_bid(), low_offer(), settlement_price(),
|
||||
high_trade_price(), low_trade_price(), definition(std::move(def)),
|
||||
last_processed_sn(0), fills(), high_limit_price(), low_limit_price(),
|
||||
max_price_variation(), cleared_volume(), electronic_volume(),
|
||||
fixing_price(), open_interest(), open_price(), high_bid(), low_offer(),
|
||||
settlement_price(), high_trade_price(), low_trade_price(),
|
||||
definition(std::move(def)),
|
||||
trading_status(definition.trading_status()) {}
|
||||
|
||||
// Books
|
||||
@@ -25,6 +26,7 @@ struct InstrumentSnapshot::PImpl {
|
||||
std::array<ImpliedBook, 2> implied_books;
|
||||
// Orders
|
||||
Orders orders;
|
||||
size_t last_processed_sn;
|
||||
// Fills
|
||||
Fills fills;
|
||||
|
||||
@@ -67,6 +69,14 @@ void InstrumentSnapshot::set_definition(InstrumentDefinition const &def) {
|
||||
me->definition = def;
|
||||
}
|
||||
|
||||
size_t InstrumentSnapshot::last_processed_sn() const {
|
||||
return me->last_processed_sn;
|
||||
}
|
||||
|
||||
void InstrumentSnapshot::set_last_msg_processed_sn(size_t n) {
|
||||
me->last_processed_sn = n;
|
||||
}
|
||||
|
||||
::mdp3::SecurityTradingStatus::Value
|
||||
InstrumentSnapshot::trading_status() const {
|
||||
return me->trading_status;
|
||||
@@ -168,9 +178,8 @@ template <class A>
|
||||
void InstrumentSnapshot::insert(api::Side s, std::array<A, 2> &books,
|
||||
size_t idx, BookLevel const &level) {
|
||||
auto &book = books[s - api::SIDE_BUY];
|
||||
int zidx = (int) (idx-1);
|
||||
bts_assert_d(zidx < (int) book.size());
|
||||
|
||||
int zidx = (int)(idx - 1);
|
||||
bts_assert_d(zidx < (int)book.size());
|
||||
for (int i = book.size() - 2; i >= zidx; i--)
|
||||
book[i + 1] = book[i];
|
||||
book[zidx] = level;
|
||||
@@ -180,17 +189,18 @@ template <class A>
|
||||
void InstrumentSnapshot::update(api::Side s, std::array<A, 2> &books,
|
||||
size_t idx, BookLevel const &level) {
|
||||
auto &book = books[s - api::SIDE_BUY];
|
||||
bts_assert_d(idx - 1 < book.size());
|
||||
book[idx - 1] = level;
|
||||
int zidx = (int)idx - 1;
|
||||
bts_assert_d(zidx < (int)book.size());
|
||||
book[zidx] = level;
|
||||
}
|
||||
|
||||
template <class A>
|
||||
void InstrumentSnapshot::remove(api::Side s, std::array<A, 2> &books,
|
||||
size_t idx) {
|
||||
auto &book = books[s - api::SIDE_BUY];
|
||||
int zidx = (int) (idx-1);
|
||||
int zidx = (int)(idx - 1);
|
||||
bts_assert_d(zidx < (int)book.size());
|
||||
for (int i = zidx; i < (int) book.size() - 2; i++)
|
||||
for (int i = zidx; i < (int)book.size() - 2; i++)
|
||||
book[i] = book[i + 1];
|
||||
book[book.size() - 1] = {};
|
||||
}
|
||||
|
||||
@@ -30,6 +30,10 @@ class InstrumentSnapshot {
|
||||
InstrumentDefinition definition() const;
|
||||
void set_definition(InstrumentDefinition const&);
|
||||
|
||||
// Last packet that affected this security
|
||||
size_t last_processed_sn() const ;
|
||||
void set_last_msg_processed_sn(size_t);
|
||||
|
||||
// Trading status
|
||||
::mdp3::SecurityTradingStatus::Value trading_status() const;
|
||||
void set_trading_status(::mdp3::SecurityTradingStatus::Value) const;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "lib/domain_model/Security.h"
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/md/IntPrice.h"
|
||||
#include "lib/utils/ValueWrapper.h"
|
||||
@@ -20,13 +21,14 @@ class InstrumentSnapshot;
|
||||
using InstrumentSnapshotPtr = std::unique_ptr<InstrumentSnapshot>;
|
||||
|
||||
class MDP3EventInfo {
|
||||
public:
|
||||
public:
|
||||
MDP3TimeStamp transact_time = MDP3TimeStamp{0};
|
||||
|
||||
MDP3TimeStamp first_packet_send_time;
|
||||
MDP3TimeStamp last_packet_send_time;
|
||||
uint32_t first_packet_sn;
|
||||
uint32_t last_packet_sn;
|
||||
std::vector<InstrumentSnapshot const*> affected_securities;
|
||||
std::vector<InstrumentSnapshot const *> affected_securities;
|
||||
void reset() {
|
||||
transact_time = MDP3TimeStamp{0};
|
||||
affected_securities.clear();
|
||||
@@ -40,10 +42,21 @@ using MDP3OrderPriority =
|
||||
class BookLevel {
|
||||
public:
|
||||
md::IntPrice price;
|
||||
MDP3Quantity quantity = MDP3Quantity {0};
|
||||
int32_t n_orders;
|
||||
MDP3Quantity quantity = MDP3Quantity{0};
|
||||
int32_t n_orders = 0;
|
||||
bool operator==(BookLevel const &other) const {
|
||||
return price == other.price &&
|
||||
quantity.value() == other.quantity.value() &&
|
||||
n_orders == other.n_orders;
|
||||
}
|
||||
};
|
||||
|
||||
inline std::ostream &operator<<(std::ostream &s, BookLevel const &l) {
|
||||
s << l.quantity.value() << " @ " << l.price.price() << " (" << l.n_orders
|
||||
<< " orders)";
|
||||
return s;
|
||||
}
|
||||
|
||||
struct MDP3Order {
|
||||
public:
|
||||
MDP3OrderKey key;
|
||||
@@ -53,6 +66,17 @@ struct MDP3Order {
|
||||
md::IntPrice price;
|
||||
MDP3Quantity quantity;
|
||||
};
|
||||
|
||||
inline std::ostream &operator<<(std::ostream &s, MDP3Order const &order) {
|
||||
s << "price: " << order.price.price() << " "
|
||||
<< "k: " << order.key.value() << " "
|
||||
<< "sk: " << order.security.value() << " "
|
||||
<< "side: " << securities::side_name(order.side) << " "
|
||||
<< "pri: " << order.priority.value() << " "
|
||||
<< "qty: " << order.quantity.value();
|
||||
return s;
|
||||
}
|
||||
|
||||
using Orders = std::unordered_map<MDP3OrderKey, MDP3Order>;
|
||||
using MDP3TradeEntryID =
|
||||
utils::ValueWrapper<struct MDP3TradeEntryIDTag, uint32_t>;
|
||||
|
||||
@@ -79,7 +79,7 @@ void ChannelHandler::on_incremental_feed(MDP3PacketView const &a,
|
||||
switch (action.action) {
|
||||
case AB_Arbitrageur::APPLY: {
|
||||
RecoveryGuard g {*this};
|
||||
parse_packet(action.packet, me->recovery_handler.on_instrument_feed);
|
||||
parse_packet(action.packet, me->recovery_handler.on_incremental_feed);
|
||||
break;
|
||||
}
|
||||
case AB_Arbitrageur::IGNORE:
|
||||
@@ -126,13 +126,13 @@ void ChannelHandler::start_recovery() {
|
||||
|
||||
void ChannelHandler::try_start_real_time() {
|
||||
bts_assert(me->state == State::RECOVERY);
|
||||
auto snaps = me->recovery_handler.try_build_snapshot_and_reset();
|
||||
if (snaps) {
|
||||
auto maybe_snaps = me->recovery_handler.try_build_snapshot_and_reset();
|
||||
if (maybe_snaps) {
|
||||
me->state = State::REALTIME;
|
||||
std::vector<InstrumentSnapshot const *> snap_ptrs;
|
||||
for (auto &p : snaps.value())
|
||||
for (auto &p : maybe_snaps.value())
|
||||
snap_ptrs.push_back(p.get());
|
||||
me->real_time_handler.add_snapshots(snaps.value());
|
||||
me->real_time_handler.add_snapshots(maybe_snaps.value());
|
||||
me->observer.on_recovery_complete(snap_ptrs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,8 +96,9 @@ struct RealTimeHandler::EventGuard {
|
||||
}
|
||||
}
|
||||
|
||||
void record_security_change(InstrumentSnapshot const *s) {
|
||||
void record_security_change(InstrumentSnapshot *s) {
|
||||
auto &info = owner.me->current_event;
|
||||
s->set_last_msg_processed_sn(owner.me->current_packet_sn);
|
||||
for (auto ss : info.affected_securities)
|
||||
if (ss == s)
|
||||
break;
|
||||
@@ -167,6 +168,7 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) {
|
||||
struct MDEntryCache {
|
||||
MDP3SecurityID id;
|
||||
api::Side side;
|
||||
md::IntPrice price;
|
||||
};
|
||||
std::array<MDEntryCache, 255> cache;
|
||||
size_t cache_idx = 0;
|
||||
@@ -188,6 +190,11 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) {
|
||||
MDP3Quantity{md_entry.mDEntrySize()},
|
||||
md_entry.numberOfOrders()};
|
||||
|
||||
cache[cache_idx].id = sid;
|
||||
cache[cache_idx].side = side;
|
||||
cache[cache_idx].price = level.price;
|
||||
cache_idx++;
|
||||
|
||||
switch (md_entry.mDEntryType()) {
|
||||
// Normal book
|
||||
case ::mdp3::MDEntryTypeBook::Bid:
|
||||
@@ -236,16 +243,18 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) {
|
||||
});
|
||||
|
||||
m.noOrderIDEntries().forEach([this, &cache](auto &order_id_entry) {
|
||||
auto sid = cache[order_id_entry.referenceID()].id;
|
||||
auto &md_entry = cache[order_id_entry.referenceID() - 1];
|
||||
auto sid = md_entry.id;
|
||||
auto iptr = find_instrument(sid);
|
||||
if (!iptr)
|
||||
return;
|
||||
MDP3Order order;
|
||||
order.key = MDP3OrderKey{order_id_entry.orderID()};
|
||||
order.security = sid;
|
||||
order.side = cache[order_id_entry.referenceID()].side;
|
||||
order.security = sid;
|
||||
order.side = md_entry.side;
|
||||
order.priority = MDP3OrderPriority{order_id_entry.mDOrderPriority()};
|
||||
order.quantity = MDP3Quantity{order_id_entry.mDDisplayQty()};
|
||||
order.price = md_entry.price;
|
||||
switch (order_id_entry.orderUpdateAction()) {
|
||||
case ::mdp3::OrderUpdateAction::New:
|
||||
iptr->insert_order(order);
|
||||
@@ -333,7 +342,8 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshOrderBook47 &m) {
|
||||
order.side = convert_side(md_entry.mDEntryType());
|
||||
order.priority = MDP3OrderPriority{md_entry.mDOrderPriority()};
|
||||
order.quantity = MDP3Quantity{md_entry.mDDisplayQty()};
|
||||
|
||||
order.price = md::IntPrice{md_entry.mDEntryPx().mantissa(),
|
||||
md_entry.mDEntryPx().exponent()};
|
||||
switch (md_entry.mDUpdateAction()) {
|
||||
case ::mdp3::MDUpdateAction::New:
|
||||
iptr->insert_order(order);
|
||||
|
||||
@@ -147,13 +147,20 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() {
|
||||
auto &this_snapshot = *current_snapshot;
|
||||
if (this_snapshot.last_processed_sn == 0 ||
|
||||
this_snapshot.last_processed_sn + 1 == pv.seq_num()) {
|
||||
LOG(Spartan, Info,
|
||||
"Added snapshot with last update time {} for security {} with last processed sn {}",
|
||||
this_snapshot.snapshot->definition().last_update_time().value(),
|
||||
this_snapshot.snapshot->security_id().value(),
|
||||
this_snapshot.last_processed_sn);
|
||||
rt_handler.add_snapshot(std::move(this_snapshot.snapshot));
|
||||
}
|
||||
else
|
||||
} else
|
||||
break;
|
||||
}
|
||||
|
||||
// Now parse the next packet
|
||||
|
||||
LOG(Spartan, Info, "Appplying queued packet with sequence number {}",
|
||||
pv.seq_num());
|
||||
parse_packet(pv, rt_handler);
|
||||
}
|
||||
bts_assert(current_snapshot == sortable_snapshots.end());
|
||||
@@ -292,6 +299,7 @@ void RecoveryHandler::update_book_in_snapshot(
|
||||
r.highLimitPrice().exponent()});
|
||||
snap.set_low_limit_price(md::IntPrice{r.lowLimitPrice().mantissa(),
|
||||
r.lowLimitPrice().exponent()});
|
||||
snap.set_last_msg_processed_sn(r.lastMsgSeqNumProcessed());
|
||||
r.noMDEntries().forEach([&snap](auto &md_entry) {
|
||||
switch (md_entry.mDEntryType()) {
|
||||
case ::mdp3::MDEntryType::Bid:
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/recovery/IncrementalQueue.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.h"
|
||||
|
||||
|
||||
#include "server/sparkd/cmds/spartan/util/PCapReader.h"
|
||||
|
||||
#include <boost/date_time/posix_time/conversion.hpp>
|
||||
@@ -22,47 +21,69 @@
|
||||
|
||||
namespace bts::spartan::mdp3::tests {
|
||||
|
||||
auto IA_ip = "224.0.31.1";
|
||||
auto IA_port = 14310;
|
||||
auto IB_ip = "224.0.32.1";
|
||||
auto IB_port = 15310;
|
||||
auto NA_ip = "224.0.31.43";
|
||||
auto NA_port = 14310;
|
||||
auto NB_ip = "224.0.32.43";
|
||||
auto NB_port = 15310;
|
||||
auto SA_ip = "224.0.31.22";
|
||||
auto SA_port = 14310;
|
||||
auto SB_ip = "224.0.32.22";
|
||||
auto SB_port = 15310;
|
||||
auto SAMBO_ip = "233.72.75.1";
|
||||
auto SAMBO_port = 23310;
|
||||
auto SBMBO_ip = "233.72.75.64";
|
||||
auto SBMBO_port = 22310;
|
||||
static auto IA_ip = "224.0.31.1";
|
||||
static auto IA_port = 14310;
|
||||
static auto IB_ip = "224.0.32.1";
|
||||
static auto IB_port = 15310;
|
||||
static auto NA_ip = "224.0.31.43";
|
||||
static auto NA_port = 14310;
|
||||
static auto NB_ip = "224.0.32.43";
|
||||
static auto NB_port = 15310;
|
||||
static auto SA_ip = "224.0.31.22";
|
||||
static auto SA_port = 14310;
|
||||
static auto SB_ip = "224.0.32.22";
|
||||
static auto SB_port = 15310;
|
||||
static auto SAMBO_ip = "233.72.75.1";
|
||||
static auto SAMBO_port = 23310;
|
||||
static auto SBMBO_ip = "233.72.75.64";
|
||||
static auto SBMBO_port = 22310;
|
||||
|
||||
struct PacketBuffers {
|
||||
MDP3Packet i;
|
||||
};
|
||||
|
||||
TEST_CASE("spartan_smoke", "[spartan_smoke]") {
|
||||
bool books_are_consistent(InstrumentSnapshot const &snap, api::Side side) {
|
||||
auto &orders = snap.orders();
|
||||
auto invert = side == api::SIDE_BUY ? -1 : 1;
|
||||
auto book_idx = std::map<int64_t, BookLevel>();
|
||||
for (auto &o : orders)
|
||||
if (o.second.side == side) {
|
||||
auto &order = o.second;
|
||||
auto &level = book_idx[invert * order.price.mantissa()];
|
||||
level.n_orders++;
|
||||
level.price = order.price;
|
||||
level.quantity.value() += order.quantity.value();
|
||||
}
|
||||
|
||||
Book mbo_book;
|
||||
size_t i = 0;
|
||||
for (auto &p : book_idx)
|
||||
if (i < mbo_book.size())
|
||||
mbo_book[i++] = p.second;
|
||||
|
||||
auto result = false;
|
||||
if (snap.definition().trading_status() ==
|
||||
::mdp3::SecurityTradingStatus::Close ||
|
||||
snap.definition().trading_status() ==
|
||||
::mdp3::SecurityTradingStatus::PreOpen)
|
||||
result = mbo_book[0] == snap.book(side)[0];
|
||||
else
|
||||
result = mbo_book == snap.book(side);
|
||||
|
||||
if (result)
|
||||
return true;
|
||||
else {
|
||||
LOG(Spartan, Info, "Inconsistency. Security: {} State: {}",
|
||||
snap.security_id().value(), snap.definition().trading_status());
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("spartan_md", "[spartan_md]") {
|
||||
|
||||
auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"};
|
||||
std::array<PacketBuffers, 2> buffers;
|
||||
struct Obs : IChannelObserver {
|
||||
std::unordered_map<MDP3SecurityID, InstrumentSnapshot const *>
|
||||
instruments;
|
||||
|
||||
void on_recovery_complete(
|
||||
std::vector<InstrumentSnapshot const *> const &snaps) override {
|
||||
for (auto &i : snaps)
|
||||
instruments[i->security_id()] = i;
|
||||
}
|
||||
void on_real_time_event(MDP3EventInfo const &ev) override {
|
||||
for (auto i : ev.affected_securities)
|
||||
instruments[i->security_id()] = i;
|
||||
}
|
||||
|
||||
void on_real_time_channel_loss() override {}
|
||||
} obs;
|
||||
struct Ctrl : IFeedController {
|
||||
void start_instrument_feed() override{};
|
||||
void stop_instrument_feed() override{};
|
||||
@@ -72,9 +93,23 @@ TEST_CASE("spartan_smoke", "[spartan_smoke]") {
|
||||
void stop_incremental_feed() override{};
|
||||
} ctrl;
|
||||
|
||||
ChannelHandler handler(obs, ctrl);
|
||||
SECTION("can_get_to_realtime") {
|
||||
struct Obs : IChannelObserver {
|
||||
std::unordered_map<MDP3SecurityID, InstrumentSnapshot const *>
|
||||
instruments;
|
||||
void on_recovery_complete(
|
||||
std::vector<InstrumentSnapshot const *> const &snaps) override {
|
||||
for (auto &i : snaps)
|
||||
instruments[i->security_id()] = i;
|
||||
}
|
||||
void on_real_time_event(MDP3EventInfo const &ev) override {
|
||||
for (auto i : ev.affected_securities)
|
||||
instruments[i->security_id()] = i;
|
||||
}
|
||||
void on_real_time_channel_loss() override { REQUIRE(false); }
|
||||
} obs;
|
||||
|
||||
SECTION("can_bootstrap") {
|
||||
ChannelHandler handler(obs, ctrl);
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) {
|
||||
buffers[0].i = MDP3Packet{payload};
|
||||
@@ -93,16 +128,49 @@ TEST_CASE("spartan_smoke", "[spartan_smoke]") {
|
||||
};
|
||||
while (reader.read_next())
|
||||
;
|
||||
REQUIRE(obs.instruments.size() == 12);
|
||||
REQUIRE(obs.instruments.size() == 17);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("spartan_book_snap", "[spartan_book_snap]") {
|
||||
SECTION("mbo_and_mbp_are_consistent") {
|
||||
struct Obs : IChannelObserver {
|
||||
void on_recovery_complete(
|
||||
std::vector<InstrumentSnapshot const *> const &snaps) override {
|
||||
for (auto is : snaps) {
|
||||
REQUIRE(books_are_consistent(*is, api::SIDE_BUY));
|
||||
REQUIRE(books_are_consistent(*is, api::SIDE_SELL));
|
||||
}
|
||||
}
|
||||
void on_real_time_event(MDP3EventInfo const &ev) override {
|
||||
for (auto is : ev.affected_securities) {
|
||||
REQUIRE(books_are_consistent(*is, api::SIDE_BUY));
|
||||
REQUIRE(books_are_consistent(*is, api::SIDE_SELL));
|
||||
}
|
||||
}
|
||||
void on_real_time_channel_loss() override { REQUIRE(false); }
|
||||
} obs;
|
||||
|
||||
auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"};
|
||||
|
||||
BookSnapshot book_snapshot;
|
||||
ChannelHandler handler(obs, ctrl);
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) {
|
||||
buffers[0].i = MDP3Packet{payload};
|
||||
handler.on_incremental_feed(buffers[0].i.view(),
|
||||
buffers[1].i.view());
|
||||
} else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) {
|
||||
buffers[1].i = MDP3Packet{payload};
|
||||
handler.on_incremental_feed(buffers[0].i.view(),
|
||||
buffers[1].i.view());
|
||||
} else if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) {
|
||||
handler.on_instrument_feed(MDP3PacketView{payload});
|
||||
} else if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) {
|
||||
handler.on_snapshot_feed(MDP3PacketView{payload});
|
||||
}
|
||||
return true;
|
||||
};
|
||||
while (reader.read_next())
|
||||
;
|
||||
}
|
||||
SECTION("can_generate_book_snapshot") {
|
||||
BookSnapshot book_snapshot;
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port))
|
||||
parse_packet(MDP3PacketView{payload}, book_snapshot);
|
||||
@@ -112,14 +180,9 @@ TEST_CASE("spartan_book_snap", "[spartan_book_snap]") {
|
||||
;
|
||||
REQUIRE(book_snapshot.is_complete());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("spartan_def_snap", "[spartan_def_snap]") {
|
||||
|
||||
auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"};
|
||||
|
||||
DefinitionSnapshot def_snapshot;
|
||||
SECTION("can_generate_definition_snapshot") {
|
||||
DefinitionSnapshot def_snapshot;
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
if ((info.dst_ip == NA_ip && info.dst_port == NA_port))
|
||||
parse_packet(MDP3PacketView{payload}, def_snapshot);
|
||||
@@ -129,18 +192,12 @@ TEST_CASE("spartan_def_snap", "[spartan_def_snap]") {
|
||||
;
|
||||
REQUIRE(def_snapshot.is_complete());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("spartan_inc_queue", "[spartan_inc_queue]") {
|
||||
|
||||
auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"};
|
||||
std::array<PacketBuffers, 2> buffers;
|
||||
IncrementalQueue q;
|
||||
AB_Arbitrageur ab;
|
||||
SECTION("can_get_consistent_queue") {
|
||||
IncrementalQueue q;
|
||||
AB_Arbitrageur ab;
|
||||
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
|
||||
if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) {
|
||||
buffers[0].i = MDP3Packet{payload};
|
||||
} else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) {
|
||||
@@ -151,7 +208,7 @@ TEST_CASE("spartan_inc_queue", "[spartan_inc_queue]") {
|
||||
auto action = ab(buffers[0].i.view(), buffers[1].i.view());
|
||||
switch (action.action) {
|
||||
case AB_Arbitrageur::APPLY:
|
||||
parse_packet(action.packet,q);
|
||||
parse_packet(action.packet, q);
|
||||
return !q.is_consistent();
|
||||
case AB_Arbitrageur::IGNORE:
|
||||
break;
|
||||
@@ -161,45 +218,36 @@ TEST_CASE("spartan_inc_queue", "[spartan_inc_queue]") {
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
|
||||
while (reader.read_next())
|
||||
;
|
||||
REQUIRE(q.is_consistent());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("spartan_recovery", "[spartan_recovery]") {
|
||||
|
||||
auto reader = PCapReader{"/home/ppinto/es_md/es_all.pcap"};
|
||||
std::array<PacketBuffers, 2> buffers;
|
||||
RecoveryHandler recovery;
|
||||
AB_Arbitrageur ab;
|
||||
RecoveryHandler::MaybeSnapshot maybe_snapshot;
|
||||
|
||||
SECTION("can_build_snapshot") {
|
||||
|
||||
RecoveryHandler recovery;
|
||||
AB_Arbitrageur ab;
|
||||
RecoveryHandler::MaybeSnapshot maybe_snapshot;
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
|
||||
if ((info.dst_ip == IA_ip && info.dst_port == IA_port)) {
|
||||
buffers[0].i = MDP3Packet{payload};
|
||||
} else if ((info.dst_ip == IB_ip && info.dst_port == IB_port)) {
|
||||
buffers[1].i = MDP3Packet{payload};
|
||||
} else if ((info.dst_ip == NA_ip && info.dst_port == NA_port)) {
|
||||
parse_packet(MDP3PacketView{payload},recovery.on_instrument_feed);
|
||||
parse_packet(MDP3PacketView{payload}, recovery.on_instrument_feed);
|
||||
maybe_snapshot = recovery.try_build_snapshot_and_reset();
|
||||
return !maybe_snapshot;
|
||||
} else if ((info.dst_ip == SAMBO_ip && info.dst_port == SAMBO_port)) {
|
||||
parse_packet(MDP3PacketView{payload},recovery.on_snapshot_feed);
|
||||
parse_packet(MDP3PacketView{payload}, recovery.on_snapshot_feed);
|
||||
maybe_snapshot = recovery.try_build_snapshot_and_reset();
|
||||
return !maybe_snapshot;
|
||||
}
|
||||
else
|
||||
} else
|
||||
return true;
|
||||
|
||||
auto action = ab(buffers[0].i.view(), buffers[1].i.view());
|
||||
switch (action.action) {
|
||||
case AB_Arbitrageur::APPLY:
|
||||
parse_packet(action.packet,recovery.on_incremental_feed);
|
||||
parse_packet(action.packet, recovery.on_incremental_feed);
|
||||
break;
|
||||
case AB_Arbitrageur::IGNORE:
|
||||
break;
|
||||
@@ -210,12 +258,11 @@ TEST_CASE("spartan_recovery", "[spartan_recovery]") {
|
||||
maybe_snapshot = recovery.try_build_snapshot_and_reset();
|
||||
return !maybe_snapshot;
|
||||
};
|
||||
|
||||
|
||||
while (reader.read_next())
|
||||
;
|
||||
REQUIRE(maybe_snapshot);
|
||||
REQUIRE(maybe_snapshot.value().size());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
} // namespace bts::spartan::mdp3::tests
|
||||
Reference in new issue
Block a user