Added books to CSV output.

This commit is contained in:
Pedro Pinto committed 2020-08-21 00:22:36 -04:00
1 parent 056e0f8aa7
commit 291699fb9b
3 files changed
+240 -148

No files matched your search

@@ -29,32 +29,17 @@ class InstrumentSnapshot {
~InstrumentSnapshot();
// Queries ------------------------------------------------------------
// Security
MDP3SecurityID security_id() const;
InstrumentDefinition definition() const;
const char *symbol() const { return definition().symbol(); }
::mdp3::SecurityTradingStatus::Value trading_status() const;
// Last packet incorporated in this snapshot
size_t last_processed_sn() const;
// Books
Book const &book(api::Side) const;
size_t last_processed_sn() const;
Book const &book(api::Side) const;
ImpliedBook const &implied_book(api::Side) const;
// Orders
Orders const &orders() const;
// Last round of order changes
OrderChanges const &order_changes() const;
// Deduced order change, that is the order change that
// would explain the last event processed in this snapshot as
// described by order_changes() and fills()
MaybeOrderChange deduced_order_change() const;
// Last round of fills
Fills const &fills() const;
// Stats
md::IntPrice high_limit_price() const;
md::IntPrice low_limit_price() const;
md::IntPrice max_price_variation() const;
@@ -23,6 +23,86 @@ CREATE TABLE event_items(
tid numeric,
agg TEXT
);
drop table books;
CREATE TABLE books(
eid numeric,
t_time numeric,
sid numeric,
status TEXT,
bid_p_0 numeric,
bid_q_0 numeric,
bid_p_1 numeric,
bid_q_1 numeric,
bid_p_2 numeric,
bid_q_2 numeric,
bid_p_3 numeric,
bid_q_3 numeric,
bid_p_4 numeric,
bid_q_4 numeric,
bid_p_5 numeric,
bid_q_5 numeric,
bid_p_6 numeric,
bid_q_6 numeric,
bid_p_7 numeric,
bid_q_7 numeric,
bid_p_8 numeric,
bid_q_8 numeric,
bid_p_9 numeric,
bid_q_9 numeric,
ask_p_0 numeric,
ask_q_0 numeric,
ask_p_1 numeric,
ask_q_1 numeric,
ask_p_2 numeric,
ask_q_2 numeric,
ask_p_3 numeric,
ask_q_3 numeric,
ask_p_4 numeric,
ask_q_4 numeric,
ask_p_5 numeric,
ask_q_5 numeric,
ask_p_6 numeric,
ask_q_6 numeric,
ask_p_7 numeric,
ask_q_7 numeric,
ask_p_8 numeric,
ask_q_8 numeric,
ask_p_9 numeric,
ask_q_9 numeric,
ibid_p_0 numeric,
ibid_q_0 numeric,
ibid_p_1 numeric,
ibid_q_1 numeric,
iask_p_0 numeric,
iask_q_0 numeric,
iask_p_1 numeric,
iask_q_1 numeric
);
drop table securities;
CREATE TABLE securities(
sid numeric,
@@ -224,9 +224,9 @@ int compress(std::string const &file, std::string const &out_file) {
while (reader.read_next())
;
auto feed_map = guesser.result;
auto feed_map = guesser.result;
if (!feed_map) {
LOG(Spartan,Fatal, "Could not guess feed ip/ports in {}. Aborting",
LOG(Spartan, Fatal, "Could not guess feed ip/ports in {}. Aborting",
file);
return -1;
}
@@ -314,7 +314,7 @@ struct PacketBuffers {
mdp3::MDP3Packet n;
};
int to_json(std::string const&pcap) {
int to_json(std::string const &pcap) {
auto reader = PCapReader{pcap};
JSON_Writer writer;
reader.on_packet() = [&](auto const &, auto const &payload) {
@@ -367,7 +367,6 @@ void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
} ctrl;
std::array<PacketBuffers, 2> buffers;
Guesser guesser;
auto reader = PCapReader{file};
reader.on_packet() = [&](auto const &info, auto const &payload) {
@@ -382,9 +381,9 @@ void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
while (reader.read_next())
;
auto feed_map = guesser.result;
auto feed_map = guesser.result;
if (!feed_map) {
LOG(Spartan,Fatal, "Could not guess feed ip/ports in {}. Aborting",
LOG(Spartan, Fatal, "Could not guess feed ip/ports in {}. Aborting",
file);
bts_assert(false);
}
@@ -411,149 +410,177 @@ void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
;
}
int to_csv(std::string const &file, std::string const &out_dir) {
using namespace bts::spartan::mdp3;
struct CSVObs : IChannelObserver {
struct Obs : IChannelObserver {
CSVObs(std::string const &out_dir)
: events(out_dir + "/events.csv"),
event_items(out_dir + "/event_items.csv"),
securities(out_dir + "/securities.csv"),
books(out_dir + "/books.csv") {
Obs(std::string const &out_dir)
: events(out_dir + "/events.csv"),
event_items(out_dir + "/event_items.csv"),
securities(out_dir + "/securities.csv") {
events << "eid, t_time,first_sn,last_sn,"
"first_st,last_st"
<< std::endl;
events << "eid, t_time,first_sn,last_sn,"
"first_st,last_st"
<< std::endl;
event_items << "eid, t_time,sid,status,type,"
"change,ok,side,"
"pri,price,qty,tid,agg"
<< std::endl;
event_items << "eid, t_time,sid,status,type,"
"change,ok,side,"
"pri,price,qty,tid,agg"
<< std::endl;
securities << "sid,symbol,group,asset,last_t" << std::endl;
}
securities << "sid,symbol,group,asset,last_t" << std::endl;
std::ofstream events;
std::ofstream event_items;
std::ofstream orders;
std::ofstream securities;
std::ofstream books;
size_t event_id = 0;
std::unordered_set<MDP3SecurityID> known_securities;
template <class B> void print_book(B const &b) {
for (auto level : b) {
books << ",";
books << level.price.price() << "," << level.quantity.value();
}
}
std::ofstream events;
std::ofstream event_items;
std::ofstream orders;
std::ofstream securities;
size_t event_id = 0;
std::unordered_set<MDP3SecurityID> known_securities;
void on_recovery_complete(
std::vector<InstrumentSnapshot const *> const &secs) override {
for (auto s : secs) {
void on_recovery_complete(
std::vector<InstrumentSnapshot const *> const &secs) override {
for (auto s : secs) {
securities << s->security_id().value() // security id
<< "," << s->symbol() << "," // symbol
<< s->definition().security_group() << "," // group
<< s->definition().asset() << "," // asset
<< s->definition().last_update_time() // last update time
<< std::endl;
known_securities.insert(s->security_id());
books << 0 << "," // event_id
<< 0 << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << ","; // trading status
print_book(s->book(api::SIDE_BUY));
print_book(s->book(api::SIDE_SELL));
print_book(s->implied_book(api::SIDE_BUY));
print_book(s->implied_book(api::SIDE_SELL));
books << std::endl;
for (auto const &[id, order] : s->orders())
event_items << 0 << "," // event_id
<< 0 << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "order_change," // type
<< "init," // change
<< id << "," // order key
<< api::Side_Name(order.side) << "," // side
<< order.priority.value() << "," // priority
<< order.price.price() << "," // price
<< order.quantity.value() << "," // quantity
<< "," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
}
}
void on_real_time_event(MDP3Event const &ev) override {
event_id++;
events << event_id << "," // event id
<< ev.transact_time << "," // transact time
<< ev.first_packet_sn << "," // first packet_sn
<< ev.last_packet_sn << "," // last_packet_sn
<< ev.first_packet_send_time << "," // first packet send time
<< ev.last_packet_send_time // last packet send time
<< std::endl;
for (auto s : ev.affected_securities) {
if (known_securities.count(s->security_id()) == 0) {
known_securities.insert(s->security_id());
securities << s->security_id().value() // security id
<< "," << s->symbol() << "," // symbol
<< s->definition().security_group() << "," // group
<< s->definition().asset() << "," // asset
<< s->definition().last_update_time() // last update time
<< std::endl;
}
known_securities.insert(s->security_id());
for (auto const &[id, order] : s->orders())
event_items << 0 << "," // event_id
<< 0 << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "order_change," // type
<< "init," // change
<< id << "," // order key
<< api::Side_Name(order.side) << "," // side
<< order.priority.value() << "," // priority
<< order.price.price() << "," // price
<< order.quantity.value() << "," // quantity
<< "," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
books << event_id << "," // event id
<< ev.transact_time.value() << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status(); // trading status
print_book(s->book(api::SIDE_BUY));
print_book(s->book(api::SIDE_SELL));
print_book(s->implied_book(api::SIDE_BUY));
print_book(s->implied_book(api::SIDE_SELL));
books << std::endl;
for (auto const &change : s->order_changes())
event_items << event_id << "," // event id
<< ev.transact_time.value() // transact time
<< "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "order_change," // type
<< change_name(change.change) << "," // change type
<< change.order.key.value() << "," // order key
<< api::Side_Name(change.order.side) << "," // side
<< change.order.priority.value() << "," // priority
<< change.order.price.price() << "," // price
<< change.order.quantity.value() << "," // quantity
<< "," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
for (auto const &f : s->fills())
event_items << event_id << "," // event id
<< ev.transact_time.value() << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "fill," // type
<< "," // null
<< f.key.value() << "," // order key
<< "," // side
<< "," // priority
<< f.price << "," // price
<< f.quantity << "," // quantity
<< f.trade_entry_id << "," // trade_entry_id
<< aggressor_name(f.aggressor) // aggressor
<< std::endl;
auto deduced_order = s->deduced_order_change();
if (deduced_order) {
event_items << event_id << "," // event id
<< ev.transact_time.value() << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "deduced_order," // type
<< change_name(deduced_order->change) << ","
<< deduced_order->order.key.value() << "," // order key
<< api::Side_Name(deduced_order->order.side)
<< "," // side
<< "," // priority
<< deduced_order->order.price.price() << "," // price
<< deduced_order->order.quantity.value()
<< "," // quantity
<< "," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
}
}
}
void on_real_time_event(MDP3Event const &ev) override {
event_id++;
events << event_id << "," // event id
<< ev.transact_time << "," // transact time
<< ev.first_packet_sn << "," // first packet_sn
<< ev.last_packet_sn << "," // last_packet_sn
<< ev.first_packet_send_time << "," // first packet send time
<< ev.last_packet_send_time // last packet send time
<< std::endl;
for (auto s : ev.affected_securities) {
if (known_securities.count(s->security_id()) == 0) {
known_securities.insert(s->security_id());
securities << s->security_id().value() // security id
<< "," << s->symbol() << "," // symbol
<< s->definition().security_group() << "," // group
<< s->definition().asset() << "," // asset
<< s->definition().last_update_time() // last update time
<< std::endl;
}
for (auto const &change : s->order_changes())
event_items << event_id << "," // event id
<< ev.transact_time.value() // transact time
<< "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "order_change," // type
<< change_name(change.change) << "," // change type
<< change.order.key.value() << "," // order key
<< api::Side_Name(change.order.side) << "," // side
<< change.order.priority.value() << "," // priority
<< change.order.price.price() << "," // price
<< change.order.quantity.value() << "," // quantity
<< "," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
for (auto const &f : s->fills())
event_items << event_id << "," // event id
<< ev.transact_time.value() << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "fill," // type
<< "," // null
<< f.key.value() << "," // order key
<< "," // side
<< "," // priority
<< f.price << "," // price
<< f.quantity << "," // quantity
<< f.trade_entry_id << "," // trade_entry_id
<< aggressor_name(f.aggressor) // aggressor
<< std::endl;
auto deduced_order = s->deduced_order_change();
if (deduced_order) {
event_items << event_id << "," // event id
<< ev.transact_time.value() << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "deduced_order," // type
<< change_name(deduced_order->change) << ","
<< deduced_order->order.key.value()
<< "," // order key
<< api::Side_Name(deduced_order->order.side)
<< "," // side
<< "," // priority
<< deduced_order->order.price.price() << "," // price
<< deduced_order->order.quantity.value()
<< "," // quantity
<< "," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
}
}
}
void on_real_time_channel_loss() override {
LOG(Spartan,Error,"Detected loss in feed. Aborting");
bts_assert(false);
}
} obs(out_dir);
void on_real_time_channel_loss() override {
LOG(Spartan, Error, "Detected loss in feed. Aborting");
bts_assert(false);
}
};
int to_csv(std::string const &file, std::string const &out_dir) {
using namespace bts::spartan::mdp3;
CSVObs obs{out_dir};
observe_channel(file, obs);
return 0;
}