diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h index 7aa0583f3..28ddc636d 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h @@ -29,32 +29,17 @@ class InstrumentSnapshot { ~InstrumentSnapshot(); // Queries ------------------------------------------------------------ - // Security MDP3SecurityID security_id() const; InstrumentDefinition definition() const; const char *symbol() const { return definition().symbol(); } ::mdp3::SecurityTradingStatus::Value trading_status() const; - - // Last packet incorporated in this snapshot - size_t last_processed_sn() const; - - // Books - Book const &book(api::Side) const; + size_t last_processed_sn() const; + Book const &book(api::Side) const; ImpliedBook const &implied_book(api::Side) const; - - // Orders Orders const &orders() const; - // Last round of order changes OrderChanges const &order_changes() const; - // Deduced order change, that is the order change that - // would explain the last event processed in this snapshot as - // described by order_changes() and fills() MaybeOrderChange deduced_order_change() const; - - // Last round of fills Fills const &fills() const; - - // Stats md::IntPrice high_limit_price() const; md::IntPrice low_limit_price() const; md::IntPrice max_price_variation() const; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/sql/md.sql b/src/server/sparkd/cmds/spartan/md/mdp3/sql/md.sql index c555a4537..3ef07a9c2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/sql/md.sql +++ b/src/server/sparkd/cmds/spartan/md/mdp3/sql/md.sql @@ -23,6 +23,86 @@ CREATE TABLE event_items( tid numeric, agg TEXT ); +drop table books; +CREATE TABLE books( + eid numeric, + t_time numeric, + sid numeric, + status TEXT, + + bid_p_0 numeric, + bid_q_0 numeric, + + bid_p_1 numeric, + bid_q_1 numeric, + + bid_p_2 numeric, + bid_q_2 numeric, + + bid_p_3 numeric, + bid_q_3 numeric, + + bid_p_4 numeric, + bid_q_4 numeric, + + bid_p_5 numeric, + bid_q_5 numeric, + + bid_p_6 numeric, + bid_q_6 numeric, + + bid_p_7 numeric, + bid_q_7 numeric, + + bid_p_8 numeric, + bid_q_8 numeric, + + bid_p_9 numeric, + bid_q_9 numeric, + + ask_p_0 numeric, + ask_q_0 numeric, + + ask_p_1 numeric, + ask_q_1 numeric, + + ask_p_2 numeric, + ask_q_2 numeric, + + ask_p_3 numeric, + ask_q_3 numeric, + + ask_p_4 numeric, + ask_q_4 numeric, + + ask_p_5 numeric, + ask_q_5 numeric, + + ask_p_6 numeric, + ask_q_6 numeric, + + ask_p_7 numeric, + ask_q_7 numeric, + + ask_p_8 numeric, + ask_q_8 numeric, + + ask_p_9 numeric, + ask_q_9 numeric, + + + ibid_p_0 numeric, + ibid_q_0 numeric, + + ibid_p_1 numeric, + ibid_q_1 numeric, + + iask_p_0 numeric, + iask_q_0 numeric, + + iask_p_1 numeric, + iask_q_1 numeric +); drop table securities; CREATE TABLE securities( sid numeric, diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp index 0d89c40b6..7cf9740db 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp @@ -224,9 +224,9 @@ int compress(std::string const &file, std::string const &out_file) { while (reader.read_next()) ; - auto feed_map = guesser.result; + auto feed_map = guesser.result; if (!feed_map) { - LOG(Spartan,Fatal, "Could not guess feed ip/ports in {}. Aborting", + LOG(Spartan, Fatal, "Could not guess feed ip/ports in {}. Aborting", file); return -1; } @@ -314,7 +314,7 @@ struct PacketBuffers { mdp3::MDP3Packet n; }; -int to_json(std::string const&pcap) { +int to_json(std::string const &pcap) { auto reader = PCapReader{pcap}; JSON_Writer writer; reader.on_packet() = [&](auto const &, auto const &payload) { @@ -367,7 +367,6 @@ void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) { } ctrl; std::array buffers; - Guesser guesser; auto reader = PCapReader{file}; reader.on_packet() = [&](auto const &info, auto const &payload) { @@ -382,9 +381,9 @@ void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) { while (reader.read_next()) ; - auto feed_map = guesser.result; + auto feed_map = guesser.result; if (!feed_map) { - LOG(Spartan,Fatal, "Could not guess feed ip/ports in {}. Aborting", + LOG(Spartan, Fatal, "Could not guess feed ip/ports in {}. Aborting", file); bts_assert(false); } @@ -411,149 +410,177 @@ void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) { ; } -int to_csv(std::string const &file, std::string const &out_dir) { - using namespace bts::spartan::mdp3; +struct CSVObs : IChannelObserver { - struct Obs : IChannelObserver { + CSVObs(std::string const &out_dir) + : events(out_dir + "/events.csv"), + event_items(out_dir + "/event_items.csv"), + securities(out_dir + "/securities.csv"), + books(out_dir + "/books.csv") { - Obs(std::string const &out_dir) - : events(out_dir + "/events.csv"), - event_items(out_dir + "/event_items.csv"), - securities(out_dir + "/securities.csv") { + events << "eid, t_time,first_sn,last_sn," + "first_st,last_st" + << std::endl; - events << "eid, t_time,first_sn,last_sn," - "first_st,last_st" - << std::endl; + event_items << "eid, t_time,sid,status,type," + "change,ok,side," + "pri,price,qty,tid,agg" + << std::endl; - event_items << "eid, t_time,sid,status,type," - "change,ok,side," - "pri,price,qty,tid,agg" - << std::endl; + securities << "sid,symbol,group,asset,last_t" << std::endl; + } - securities << "sid,symbol,group,asset,last_t" << std::endl; + std::ofstream events; + std::ofstream event_items; + std::ofstream orders; + std::ofstream securities; + std::ofstream books; + size_t event_id = 0; + std::unordered_set known_securities; + + template void print_book(B const &b) { + for (auto level : b) { + books << ","; + books << level.price.price() << "," << level.quantity.value(); } + } - std::ofstream events; - std::ofstream event_items; - std::ofstream orders; - std::ofstream securities; - size_t event_id = 0; - std::unordered_set known_securities; - - void on_recovery_complete( - std::vector const &secs) override { - for (auto s : secs) { + void on_recovery_complete( + std::vector const &secs) override { + for (auto s : secs) { + securities << s->security_id().value() // security id + << "," << s->symbol() << "," // symbol + << s->definition().security_group() << "," // group + << s->definition().asset() << "," // asset + << s->definition().last_update_time() // last update time + << std::endl; + + known_securities.insert(s->security_id()); + + books << 0 << "," // event_id + << 0 << "," // transact time + << s->security_id().value() << "," // security id + << s->trading_status() << ","; // trading status + print_book(s->book(api::SIDE_BUY)); + print_book(s->book(api::SIDE_SELL)); + print_book(s->implied_book(api::SIDE_BUY)); + print_book(s->implied_book(api::SIDE_SELL)); + books << std::endl; + + for (auto const &[id, order] : s->orders()) + event_items << 0 << "," // event_id + << 0 << "," // transact time + << s->security_id().value() << "," // security id + << s->trading_status() << "," // trading status + << "order_change," // type + << "init," // change + << id << "," // order key + << api::Side_Name(order.side) << "," // side + << order.priority.value() << "," // priority + << order.price.price() << "," // price + << order.quantity.value() << "," // quantity + << "," // trade_entry_id + << "n/a" // aggressor + << std::endl; + } + } + + void on_real_time_event(MDP3Event const &ev) override { + event_id++; + events << event_id << "," // event id + << ev.transact_time << "," // transact time + << ev.first_packet_sn << "," // first packet_sn + << ev.last_packet_sn << "," // last_packet_sn + << ev.first_packet_send_time << "," // first packet send time + << ev.last_packet_send_time // last packet send time + << std::endl; + + for (auto s : ev.affected_securities) { + + if (known_securities.count(s->security_id()) == 0) { + known_securities.insert(s->security_id()); securities << s->security_id().value() // security id << "," << s->symbol() << "," // symbol << s->definition().security_group() << "," // group << s->definition().asset() << "," // asset << s->definition().last_update_time() // last update time << std::endl; + } - known_securities.insert(s->security_id()); - - for (auto const &[id, order] : s->orders()) - event_items << 0 << "," // event_id - << 0 << "," // transact time - << s->security_id().value() << "," // security id - << s->trading_status() << "," // trading status - << "order_change," // type - << "init," // change - << id << "," // order key - << api::Side_Name(order.side) << "," // side - << order.priority.value() << "," // priority - << order.price.price() << "," // price - << order.quantity.value() << "," // quantity - << "," // trade_entry_id - << "n/a" // aggressor - << std::endl; + books << event_id << "," // event id + << ev.transact_time.value() << "," // transact time + << s->security_id().value() << "," // security id + << s->trading_status(); // trading status + print_book(s->book(api::SIDE_BUY)); + print_book(s->book(api::SIDE_SELL)); + print_book(s->implied_book(api::SIDE_BUY)); + print_book(s->implied_book(api::SIDE_SELL)); + books << std::endl; + + for (auto const &change : s->order_changes()) + event_items << event_id << "," // event id + << ev.transact_time.value() // transact time + << "," // transact time + << s->security_id().value() << "," // security id + << s->trading_status() << "," // trading status + << "order_change," // type + << change_name(change.change) << "," // change type + << change.order.key.value() << "," // order key + << api::Side_Name(change.order.side) << "," // side + << change.order.priority.value() << "," // priority + << change.order.price.price() << "," // price + << change.order.quantity.value() << "," // quantity + << "," // trade_entry_id + << "n/a" // aggressor + << std::endl; + + for (auto const &f : s->fills()) + event_items << event_id << "," // event id + << ev.transact_time.value() << "," // transact time + << s->security_id().value() << "," // security id + << s->trading_status() << "," // trading status + << "fill," // type + << "," // null + << f.key.value() << "," // order key + << "," // side + << "," // priority + << f.price << "," // price + << f.quantity << "," // quantity + << f.trade_entry_id << "," // trade_entry_id + << aggressor_name(f.aggressor) // aggressor + << std::endl; + + auto deduced_order = s->deduced_order_change(); + if (deduced_order) { + event_items << event_id << "," // event id + << ev.transact_time.value() << "," // transact time + << s->security_id().value() << "," // security id + << s->trading_status() << "," // trading status + << "deduced_order," // type + << change_name(deduced_order->change) << "," + << deduced_order->order.key.value() << "," // order key + << api::Side_Name(deduced_order->order.side) + << "," // side + << "," // priority + << deduced_order->order.price.price() << "," // price + << deduced_order->order.quantity.value() + << "," // quantity + << "," // trade_entry_id + << "n/a" // aggressor + << std::endl; } } + } - void on_real_time_event(MDP3Event const &ev) override { - event_id++; - events << event_id << "," // event id - << ev.transact_time << "," // transact time - << ev.first_packet_sn << "," // first packet_sn - << ev.last_packet_sn << "," // last_packet_sn - << ev.first_packet_send_time << "," // first packet send time - << ev.last_packet_send_time // last packet send time - << std::endl; - - for (auto s : ev.affected_securities) { - - if (known_securities.count(s->security_id()) == 0) { - known_securities.insert(s->security_id()); - securities << s->security_id().value() // security id - << "," << s->symbol() << "," // symbol - << s->definition().security_group() << "," // group - << s->definition().asset() << "," // asset - << s->definition().last_update_time() // last update time - << std::endl; - } - - for (auto const &change : s->order_changes()) - event_items << event_id << "," // event id - << ev.transact_time.value() // transact time - << "," // transact time - << s->security_id().value() << "," // security id - << s->trading_status() << "," // trading status - << "order_change," // type - << change_name(change.change) << "," // change type - << change.order.key.value() << "," // order key - << api::Side_Name(change.order.side) << "," // side - << change.order.priority.value() << "," // priority - << change.order.price.price() << "," // price - << change.order.quantity.value() << "," // quantity - << "," // trade_entry_id - << "n/a" // aggressor - << std::endl; - - for (auto const &f : s->fills()) - event_items << event_id << "," // event id - << ev.transact_time.value() << "," // transact time - << s->security_id().value() << "," // security id - << s->trading_status() << "," // trading status - << "fill," // type - << "," // null - << f.key.value() << "," // order key - << "," // side - << "," // priority - << f.price << "," // price - << f.quantity << "," // quantity - << f.trade_entry_id << "," // trade_entry_id - << aggressor_name(f.aggressor) // aggressor - << std::endl; - - auto deduced_order = s->deduced_order_change(); - if (deduced_order) { - event_items << event_id << "," // event id - << ev.transact_time.value() << "," // transact time - << s->security_id().value() << "," // security id - << s->trading_status() << "," // trading status - << "deduced_order," // type - << change_name(deduced_order->change) << "," - << deduced_order->order.key.value() - << "," // order key - << api::Side_Name(deduced_order->order.side) - << "," // side - << "," // priority - << deduced_order->order.price.price() << "," // price - << deduced_order->order.quantity.value() - << "," // quantity - << "," // trade_entry_id - << "n/a" // aggressor - << std::endl; - } - } - } - - void on_real_time_channel_loss() override { - LOG(Spartan,Error,"Detected loss in feed. Aborting"); - bts_assert(false); - } - } obs(out_dir); + void on_real_time_channel_loss() override { + LOG(Spartan, Error, "Detected loss in feed. Aborting"); + bts_assert(false); + } +}; +int to_csv(std::string const &file, std::string const &out_dir) { + using namespace bts::spartan::mdp3; + CSVObs obs{out_dir}; observe_channel(file, obs); return 0; }