Added books to CSV output.
This commit is contained in:
1 parent
056e0f8aa7
commit
291699fb9b
3 files changed
+240
-148
No files matched your search
@@ -29,32 +29,17 @@ class InstrumentSnapshot {
|
||||
~InstrumentSnapshot();
|
||||
|
||||
// Queries ------------------------------------------------------------
|
||||
// Security
|
||||
MDP3SecurityID security_id() const;
|
||||
InstrumentDefinition definition() const;
|
||||
const char *symbol() const { return definition().symbol(); }
|
||||
::mdp3::SecurityTradingStatus::Value trading_status() const;
|
||||
|
||||
// Last packet incorporated in this snapshot
|
||||
size_t last_processed_sn() const;
|
||||
|
||||
// Books
|
||||
Book const &book(api::Side) const;
|
||||
size_t last_processed_sn() const;
|
||||
Book const &book(api::Side) const;
|
||||
ImpliedBook const &implied_book(api::Side) const;
|
||||
|
||||
// Orders
|
||||
Orders const &orders() const;
|
||||
// Last round of order changes
|
||||
OrderChanges const &order_changes() const;
|
||||
// Deduced order change, that is the order change that
|
||||
// would explain the last event processed in this snapshot as
|
||||
// described by order_changes() and fills()
|
||||
MaybeOrderChange deduced_order_change() const;
|
||||
|
||||
// Last round of fills
|
||||
Fills const &fills() const;
|
||||
|
||||
// Stats
|
||||
md::IntPrice high_limit_price() const;
|
||||
md::IntPrice low_limit_price() const;
|
||||
md::IntPrice max_price_variation() const;
|
||||
|
||||
@@ -23,6 +23,86 @@ CREATE TABLE event_items(
|
||||
tid numeric,
|
||||
agg TEXT
|
||||
);
|
||||
drop table books;
|
||||
CREATE TABLE books(
|
||||
eid numeric,
|
||||
t_time numeric,
|
||||
sid numeric,
|
||||
status TEXT,
|
||||
|
||||
bid_p_0 numeric,
|
||||
bid_q_0 numeric,
|
||||
|
||||
bid_p_1 numeric,
|
||||
bid_q_1 numeric,
|
||||
|
||||
bid_p_2 numeric,
|
||||
bid_q_2 numeric,
|
||||
|
||||
bid_p_3 numeric,
|
||||
bid_q_3 numeric,
|
||||
|
||||
bid_p_4 numeric,
|
||||
bid_q_4 numeric,
|
||||
|
||||
bid_p_5 numeric,
|
||||
bid_q_5 numeric,
|
||||
|
||||
bid_p_6 numeric,
|
||||
bid_q_6 numeric,
|
||||
|
||||
bid_p_7 numeric,
|
||||
bid_q_7 numeric,
|
||||
|
||||
bid_p_8 numeric,
|
||||
bid_q_8 numeric,
|
||||
|
||||
bid_p_9 numeric,
|
||||
bid_q_9 numeric,
|
||||
|
||||
ask_p_0 numeric,
|
||||
ask_q_0 numeric,
|
||||
|
||||
ask_p_1 numeric,
|
||||
ask_q_1 numeric,
|
||||
|
||||
ask_p_2 numeric,
|
||||
ask_q_2 numeric,
|
||||
|
||||
ask_p_3 numeric,
|
||||
ask_q_3 numeric,
|
||||
|
||||
ask_p_4 numeric,
|
||||
ask_q_4 numeric,
|
||||
|
||||
ask_p_5 numeric,
|
||||
ask_q_5 numeric,
|
||||
|
||||
ask_p_6 numeric,
|
||||
ask_q_6 numeric,
|
||||
|
||||
ask_p_7 numeric,
|
||||
ask_q_7 numeric,
|
||||
|
||||
ask_p_8 numeric,
|
||||
ask_q_8 numeric,
|
||||
|
||||
ask_p_9 numeric,
|
||||
ask_q_9 numeric,
|
||||
|
||||
|
||||
ibid_p_0 numeric,
|
||||
ibid_q_0 numeric,
|
||||
|
||||
ibid_p_1 numeric,
|
||||
ibid_q_1 numeric,
|
||||
|
||||
iask_p_0 numeric,
|
||||
iask_q_0 numeric,
|
||||
|
||||
iask_p_1 numeric,
|
||||
iask_q_1 numeric
|
||||
);
|
||||
drop table securities;
|
||||
CREATE TABLE securities(
|
||||
sid numeric,
|
||||
|
||||
@@ -224,9 +224,9 @@ int compress(std::string const &file, std::string const &out_file) {
|
||||
while (reader.read_next())
|
||||
;
|
||||
|
||||
auto feed_map = guesser.result;
|
||||
auto feed_map = guesser.result;
|
||||
if (!feed_map) {
|
||||
LOG(Spartan,Fatal, "Could not guess feed ip/ports in {}. Aborting",
|
||||
LOG(Spartan, Fatal, "Could not guess feed ip/ports in {}. Aborting",
|
||||
file);
|
||||
return -1;
|
||||
}
|
||||
@@ -314,7 +314,7 @@ struct PacketBuffers {
|
||||
mdp3::MDP3Packet n;
|
||||
};
|
||||
|
||||
int to_json(std::string const&pcap) {
|
||||
int to_json(std::string const &pcap) {
|
||||
auto reader = PCapReader{pcap};
|
||||
JSON_Writer writer;
|
||||
reader.on_packet() = [&](auto const &, auto const &payload) {
|
||||
@@ -367,7 +367,6 @@ void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
|
||||
} ctrl;
|
||||
std::array<PacketBuffers, 2> buffers;
|
||||
|
||||
|
||||
Guesser guesser;
|
||||
auto reader = PCapReader{file};
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
@@ -382,9 +381,9 @@ void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
|
||||
while (reader.read_next())
|
||||
;
|
||||
|
||||
auto feed_map = guesser.result;
|
||||
auto feed_map = guesser.result;
|
||||
if (!feed_map) {
|
||||
LOG(Spartan,Fatal, "Could not guess feed ip/ports in {}. Aborting",
|
||||
LOG(Spartan, Fatal, "Could not guess feed ip/ports in {}. Aborting",
|
||||
file);
|
||||
bts_assert(false);
|
||||
}
|
||||
@@ -411,149 +410,177 @@ void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
|
||||
;
|
||||
}
|
||||
|
||||
int to_csv(std::string const &file, std::string const &out_dir) {
|
||||
using namespace bts::spartan::mdp3;
|
||||
struct CSVObs : IChannelObserver {
|
||||
|
||||
struct Obs : IChannelObserver {
|
||||
CSVObs(std::string const &out_dir)
|
||||
: events(out_dir + "/events.csv"),
|
||||
event_items(out_dir + "/event_items.csv"),
|
||||
securities(out_dir + "/securities.csv"),
|
||||
books(out_dir + "/books.csv") {
|
||||
|
||||
Obs(std::string const &out_dir)
|
||||
: events(out_dir + "/events.csv"),
|
||||
event_items(out_dir + "/event_items.csv"),
|
||||
securities(out_dir + "/securities.csv") {
|
||||
events << "eid, t_time,first_sn,last_sn,"
|
||||
"first_st,last_st"
|
||||
<< std::endl;
|
||||
|
||||
events << "eid, t_time,first_sn,last_sn,"
|
||||
"first_st,last_st"
|
||||
<< std::endl;
|
||||
event_items << "eid, t_time,sid,status,type,"
|
||||
"change,ok,side,"
|
||||
"pri,price,qty,tid,agg"
|
||||
<< std::endl;
|
||||
|
||||
event_items << "eid, t_time,sid,status,type,"
|
||||
"change,ok,side,"
|
||||
"pri,price,qty,tid,agg"
|
||||
<< std::endl;
|
||||
securities << "sid,symbol,group,asset,last_t" << std::endl;
|
||||
}
|
||||
|
||||
securities << "sid,symbol,group,asset,last_t" << std::endl;
|
||||
std::ofstream events;
|
||||
std::ofstream event_items;
|
||||
std::ofstream orders;
|
||||
std::ofstream securities;
|
||||
std::ofstream books;
|
||||
size_t event_id = 0;
|
||||
std::unordered_set<MDP3SecurityID> known_securities;
|
||||
|
||||
template <class B> void print_book(B const &b) {
|
||||
for (auto level : b) {
|
||||
books << ",";
|
||||
books << level.price.price() << "," << level.quantity.value();
|
||||
}
|
||||
}
|
||||
|
||||
std::ofstream events;
|
||||
std::ofstream event_items;
|
||||
std::ofstream orders;
|
||||
std::ofstream securities;
|
||||
size_t event_id = 0;
|
||||
std::unordered_set<MDP3SecurityID> known_securities;
|
||||
|
||||
void on_recovery_complete(
|
||||
std::vector<InstrumentSnapshot const *> const &secs) override {
|
||||
for (auto s : secs) {
|
||||
void on_recovery_complete(
|
||||
std::vector<InstrumentSnapshot const *> const &secs) override {
|
||||
for (auto s : secs) {
|
||||
securities << s->security_id().value() // security id
|
||||
<< "," << s->symbol() << "," // symbol
|
||||
<< s->definition().security_group() << "," // group
|
||||
<< s->definition().asset() << "," // asset
|
||||
<< s->definition().last_update_time() // last update time
|
||||
<< std::endl;
|
||||
|
||||
known_securities.insert(s->security_id());
|
||||
|
||||
books << 0 << "," // event_id
|
||||
<< 0 << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << ","; // trading status
|
||||
print_book(s->book(api::SIDE_BUY));
|
||||
print_book(s->book(api::SIDE_SELL));
|
||||
print_book(s->implied_book(api::SIDE_BUY));
|
||||
print_book(s->implied_book(api::SIDE_SELL));
|
||||
books << std::endl;
|
||||
|
||||
for (auto const &[id, order] : s->orders())
|
||||
event_items << 0 << "," // event_id
|
||||
<< 0 << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "order_change," // type
|
||||
<< "init," // change
|
||||
<< id << "," // order key
|
||||
<< api::Side_Name(order.side) << "," // side
|
||||
<< order.priority.value() << "," // priority
|
||||
<< order.price.price() << "," // price
|
||||
<< order.quantity.value() << "," // quantity
|
||||
<< "," // trade_entry_id
|
||||
<< "n/a" // aggressor
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
void on_real_time_event(MDP3Event const &ev) override {
|
||||
event_id++;
|
||||
events << event_id << "," // event id
|
||||
<< ev.transact_time << "," // transact time
|
||||
<< ev.first_packet_sn << "," // first packet_sn
|
||||
<< ev.last_packet_sn << "," // last_packet_sn
|
||||
<< ev.first_packet_send_time << "," // first packet send time
|
||||
<< ev.last_packet_send_time // last packet send time
|
||||
<< std::endl;
|
||||
|
||||
for (auto s : ev.affected_securities) {
|
||||
|
||||
if (known_securities.count(s->security_id()) == 0) {
|
||||
known_securities.insert(s->security_id());
|
||||
securities << s->security_id().value() // security id
|
||||
<< "," << s->symbol() << "," // symbol
|
||||
<< s->definition().security_group() << "," // group
|
||||
<< s->definition().asset() << "," // asset
|
||||
<< s->definition().last_update_time() // last update time
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
known_securities.insert(s->security_id());
|
||||
|
||||
for (auto const &[id, order] : s->orders())
|
||||
event_items << 0 << "," // event_id
|
||||
<< 0 << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "order_change," // type
|
||||
<< "init," // change
|
||||
<< id << "," // order key
|
||||
<< api::Side_Name(order.side) << "," // side
|
||||
<< order.priority.value() << "," // priority
|
||||
<< order.price.price() << "," // price
|
||||
<< order.quantity.value() << "," // quantity
|
||||
<< "," // trade_entry_id
|
||||
<< "n/a" // aggressor
|
||||
<< std::endl;
|
||||
books << event_id << "," // event id
|
||||
<< ev.transact_time.value() << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status(); // trading status
|
||||
print_book(s->book(api::SIDE_BUY));
|
||||
print_book(s->book(api::SIDE_SELL));
|
||||
print_book(s->implied_book(api::SIDE_BUY));
|
||||
print_book(s->implied_book(api::SIDE_SELL));
|
||||
books << std::endl;
|
||||
|
||||
for (auto const &change : s->order_changes())
|
||||
event_items << event_id << "," // event id
|
||||
<< ev.transact_time.value() // transact time
|
||||
<< "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "order_change," // type
|
||||
<< change_name(change.change) << "," // change type
|
||||
<< change.order.key.value() << "," // order key
|
||||
<< api::Side_Name(change.order.side) << "," // side
|
||||
<< change.order.priority.value() << "," // priority
|
||||
<< change.order.price.price() << "," // price
|
||||
<< change.order.quantity.value() << "," // quantity
|
||||
<< "," // trade_entry_id
|
||||
<< "n/a" // aggressor
|
||||
<< std::endl;
|
||||
|
||||
for (auto const &f : s->fills())
|
||||
event_items << event_id << "," // event id
|
||||
<< ev.transact_time.value() << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "fill," // type
|
||||
<< "," // null
|
||||
<< f.key.value() << "," // order key
|
||||
<< "," // side
|
||||
<< "," // priority
|
||||
<< f.price << "," // price
|
||||
<< f.quantity << "," // quantity
|
||||
<< f.trade_entry_id << "," // trade_entry_id
|
||||
<< aggressor_name(f.aggressor) // aggressor
|
||||
<< std::endl;
|
||||
|
||||
auto deduced_order = s->deduced_order_change();
|
||||
if (deduced_order) {
|
||||
event_items << event_id << "," // event id
|
||||
<< ev.transact_time.value() << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "deduced_order," // type
|
||||
<< change_name(deduced_order->change) << ","
|
||||
<< deduced_order->order.key.value() << "," // order key
|
||||
<< api::Side_Name(deduced_order->order.side)
|
||||
<< "," // side
|
||||
<< "," // priority
|
||||
<< deduced_order->order.price.price() << "," // price
|
||||
<< deduced_order->order.quantity.value()
|
||||
<< "," // quantity
|
||||
<< "," // trade_entry_id
|
||||
<< "n/a" // aggressor
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void on_real_time_event(MDP3Event const &ev) override {
|
||||
event_id++;
|
||||
events << event_id << "," // event id
|
||||
<< ev.transact_time << "," // transact time
|
||||
<< ev.first_packet_sn << "," // first packet_sn
|
||||
<< ev.last_packet_sn << "," // last_packet_sn
|
||||
<< ev.first_packet_send_time << "," // first packet send time
|
||||
<< ev.last_packet_send_time // last packet send time
|
||||
<< std::endl;
|
||||
|
||||
for (auto s : ev.affected_securities) {
|
||||
|
||||
if (known_securities.count(s->security_id()) == 0) {
|
||||
known_securities.insert(s->security_id());
|
||||
securities << s->security_id().value() // security id
|
||||
<< "," << s->symbol() << "," // symbol
|
||||
<< s->definition().security_group() << "," // group
|
||||
<< s->definition().asset() << "," // asset
|
||||
<< s->definition().last_update_time() // last update time
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
for (auto const &change : s->order_changes())
|
||||
event_items << event_id << "," // event id
|
||||
<< ev.transact_time.value() // transact time
|
||||
<< "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "order_change," // type
|
||||
<< change_name(change.change) << "," // change type
|
||||
<< change.order.key.value() << "," // order key
|
||||
<< api::Side_Name(change.order.side) << "," // side
|
||||
<< change.order.priority.value() << "," // priority
|
||||
<< change.order.price.price() << "," // price
|
||||
<< change.order.quantity.value() << "," // quantity
|
||||
<< "," // trade_entry_id
|
||||
<< "n/a" // aggressor
|
||||
<< std::endl;
|
||||
|
||||
for (auto const &f : s->fills())
|
||||
event_items << event_id << "," // event id
|
||||
<< ev.transact_time.value() << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "fill," // type
|
||||
<< "," // null
|
||||
<< f.key.value() << "," // order key
|
||||
<< "," // side
|
||||
<< "," // priority
|
||||
<< f.price << "," // price
|
||||
<< f.quantity << "," // quantity
|
||||
<< f.trade_entry_id << "," // trade_entry_id
|
||||
<< aggressor_name(f.aggressor) // aggressor
|
||||
<< std::endl;
|
||||
|
||||
auto deduced_order = s->deduced_order_change();
|
||||
if (deduced_order) {
|
||||
event_items << event_id << "," // event id
|
||||
<< ev.transact_time.value() << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "deduced_order," // type
|
||||
<< change_name(deduced_order->change) << ","
|
||||
<< deduced_order->order.key.value()
|
||||
<< "," // order key
|
||||
<< api::Side_Name(deduced_order->order.side)
|
||||
<< "," // side
|
||||
<< "," // priority
|
||||
<< deduced_order->order.price.price() << "," // price
|
||||
<< deduced_order->order.quantity.value()
|
||||
<< "," // quantity
|
||||
<< "," // trade_entry_id
|
||||
<< "n/a" // aggressor
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void on_real_time_channel_loss() override {
|
||||
LOG(Spartan,Error,"Detected loss in feed. Aborting");
|
||||
bts_assert(false);
|
||||
}
|
||||
} obs(out_dir);
|
||||
void on_real_time_channel_loss() override {
|
||||
LOG(Spartan, Error, "Detected loss in feed. Aborting");
|
||||
bts_assert(false);
|
||||
}
|
||||
};
|
||||
|
||||
int to_csv(std::string const &file, std::string const &out_dir) {
|
||||
using namespace bts::spartan::mdp3;
|
||||
CSVObs obs{out_dir};
|
||||
observe_channel(file, obs);
|
||||
return 0;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user