From 78bc14348d1b69d7eec970ae50fb8261cd5af507 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 10 Aug 2020 13:46:59 -0400 Subject: [PATCH 1/6] Miscelaenous improvements. --- src/server/sparkd/cmds/QuickTest.cpp | 25 ++++++++----------- .../md/mdp3/common/InstrumentSnapshot.cpp | 14 ++++++++--- .../md/mdp3/common/InstrumentSnapshot.h | 3 +++ .../cmds/spartan/md/mdp3/common/mdp3_common.h | 23 ++++++++++++++--- .../md/mdp3/recovery/RecoveryHandler.cpp | 23 ++++++++++++++--- 5 files changed, 63 insertions(+), 25 deletions(-) diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index e364ca239..b2dc71354 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -64,26 +64,23 @@ void dump_snapshots() { } int QuickTest::execute(std::vector const &) { - //dump_snapshots(); - + // dump_snapshots(); + using namespace bts::spartan::mdp3; struct Obs : IChannelObserver { void on_recovery_complete( - std::vector const &snaps) override { - std::cout << "Initial Snapshot\n"; - for (auto is : snaps) { - if (is->security_id() == MDP3SecurityID{12181}) - std::cout << *is << "\n"; - } - } + std::vector const &) override {} void on_real_time_event(MDP3Event const &ev) override { - for (auto is : ev.affected_securities) { - if (is->security_id() == MDP3SecurityID{12181}) { - std::cout << "Seq numbers: " << ev.first_packet_sn << " to " - << ev.last_packet_sn << "\n"; - std::cout << *is << "\n"; + for (auto s : ev.affected_securities) { + if (s->order_changes().size() || s->fills().size()) { + std::cout << "symbol: " << s->definition().symbol() << "\n" + << "seq_num: " << ev.first_packet_sn << " to " + << ev.last_packet_sn << "\n" + << "changes:\n" + << s->order_changes() << "Fills:\n" + << s->fills(); } } } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp index dc55158de..18186924f 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp @@ -249,20 +249,28 @@ void InstrumentSnapshot::clear_order_changes() { me->order_changes.clear(); } void InstrumentSnapshot::insert_order(MDP3Order const &order) { bts_assert_d(me->orders.count(order.key) == 0); me->orders.insert({order.key, order}); - me->order_changes.emplace_back(OrderChange{order.key, OrderChange::added}); + me->order_changes.emplace_back(OrderChange{order.key, + order.price, + order.quantity, + OrderChange::added}); } void InstrumentSnapshot::update_order(MDP3Order const &order) { auto it = me->orders.find(order.key); bts_assert_d(it != me->orders.end()); it->second = order; - me->order_changes.emplace_back(OrderChange{order.key, OrderChange::updated}); + me->order_changes.emplace_back(OrderChange{order.key, + order.price, + order.quantity, + OrderChange::updated}); } void InstrumentSnapshot::remove_order(MDP3OrderKey id) { bts_assert_d(me->orders.count(id)); me->orders.erase(id); - me->order_changes.emplace_back(OrderChange{id, OrderChange::removed}); + me->order_changes.emplace_back(OrderChange{id, + {}, {}, + OrderChange::removed}); } Book InstrumentSnapshot::build_book_from_orders (api::Side side) const diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h index 32c04b5b9..7ebe26bf6 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h @@ -30,6 +30,9 @@ class InstrumentSnapshot { InstrumentDefinition definition() const; void set_definition(InstrumentDefinition const&); + // Symbol (via definition) + const char* symbol() const {return definition().symbol();} + // Last packet that affected this security size_t last_processed_sn() const ; void set_last_msg_processed_sn(size_t); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h index e1601e023..51c126903 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h @@ -69,15 +69,18 @@ struct MDP3Order { struct OrderChange { MDP3OrderKey key; + md::IntPrice price; + MDP3Quantity quantity; enum ChangeType { added, removed, updated } change; }; inline std::ostream &operator<<(std::ostream &s, OrderChange const &ch) { s << " ok: " << ch.key.value() << " " - << (ch.change == OrderChange::added - ? "create" - : (ch.change == OrderChange::updated ? "update" : "delete")); - + << " price: " << ch.price.price() + << " qty: " << ch.quantity.value() + << " " << (ch.change == OrderChange::added + ? "create" + : (ch.change == OrderChange::updated ? "update" : "delete")); return s; } @@ -93,6 +96,12 @@ inline std::ostream &operator<<(std::ostream &s, MDP3Order const &order) { return s; } +inline std::ostream &operator<<(std::ostream &s, OrderChanges const &changes) { + for (auto c : changes) + s << " " << c << "\n"; + return s; +} + using Orders = std::unordered_map; using MDP3TradeEntryID = utils::ValueWrapper; @@ -103,6 +112,7 @@ struct MDP3Fill { MDP3TradeEntryID trade_entry_id; enum Aggressor { THIS, OTHER, IMPLIED } aggressor; }; + inline std::ostream &operator<<(std::ostream &s, MDP3Fill const &fill) { s << " ok: " << fill.key.value() << " " << "price: " << fill.price.price() << " " @@ -113,6 +123,11 @@ inline std::ostream &operator<<(std::ostream &s, MDP3Fill const &fill) { } using Fills = std::vector; +inline std::ostream &operator<<(std::ostream &s, Fills const &fills) { + for (auto f : fills) + s << " " << f << "\n"; + return s; +} const auto MDP3_MAX_BOOK_DEPTH = 10; const auto MDP3_MAX_IMPLIED_BOOK_DEPTH = 2; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp index 97490b220..a24ec9c51 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp @@ -89,13 +89,13 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { // Does the queue extend beyond the newest event of each snapshot? If // if does not then we could run the risk of applying an event to a // snapshot that already has it. - if (me->book_snapshot.newest_snapshot_seq_num() >= + if (me->book_snapshot.newest_snapshot_seq_num() > me->incremental_queue.newest_packet_seq_num()) { // Accumulate a few more LOG(Spartan, Warn, - "Most recent packet in incremental queue ({}) is not newer than most " - "recent last processed packet in snapshot ({}). Will accumulate a " - "few more packets in incremental queue", + "Most recent packet in incremental queue ({}) is older than most " + "recent last processed packet in snapshot ({}). Will acumulate a " + "few more packets in incremental queue.", me->incremental_queue.newest_packet_seq_num(), me->book_snapshot.newest_snapshot_seq_num()); return {}; @@ -156,6 +156,7 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { RealTimeHandler rt_handler(null_handler); auto pending_packets = me->incremental_queue.get_queued_packets_and_reset(); auto current_snapshot = sortable_snapshots.begin(); + uint32_t last_applied_packet = 0; for (auto &p : pending_packets) { auto pv = p.view(); @@ -180,7 +181,21 @@ RecoveryHandler::MaybeSnapshot RecoveryHandler::try_build_snapshot_and_reset() { LOG(Spartan, Info, "Appplying queued packet with sequence number {}", pv.seq_num()); parse_packet(pv, rt_handler); + last_applied_packet = pv.seq_num(); } + + for (; current_snapshot != sortable_snapshots.end(); current_snapshot++) { + auto &this_snapshot = *current_snapshot; + bts_assert(this_snapshot.last_processed_sn == last_applied_packet); + LOG(Spartan, Info, + "Added snapshot with last update time {} for security {} with " + "last processed sn {}", + this_snapshot.snapshot->definition().last_update_time().value(), + this_snapshot.snapshot->security_id().value(), + this_snapshot.last_processed_sn); + rt_handler.add_snapshot(std::move(this_snapshot.snapshot)); + } + bts_assert(current_snapshot == sortable_snapshots.end()); // At this point we are caught up and all the instrument snapshots From 7ae0142dff190da55ae2d934075df28dfda71100 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Wed, 12 Aug 2020 11:56:08 -0400 Subject: [PATCH 2/6] Fixed reset of sequence numbers problem --- src/data_files/CMakeDeps.cmake | 2 +- src/lib/orderman/actors/ilink3/IL3GW.h | 2 +- src/lib/orderman/actors/ilink3/SessionStore.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/data_files/CMakeDeps.cmake b/src/data_files/CMakeDeps.cmake index de9a0184e..b4c513198 100644 --- a/src/data_files/CMakeDeps.cmake +++ b/src/data_files/CMakeDeps.cmake @@ -1 +1 @@ -set(sources files/cert_secure_logon.txt;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js) \ No newline at end of file +set(sources files/cert_secure_logon.txt;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js) \ No newline at end of file diff --git a/src/lib/orderman/actors/ilink3/IL3GW.h b/src/lib/orderman/actors/ilink3/IL3GW.h index d8832369c..a5fd50472 100644 --- a/src/lib/orderman/actors/ilink3/IL3GW.h +++ b/src/lib/orderman/actors/ilink3/IL3GW.h @@ -33,7 +33,7 @@ struct Session { // cme->bts uint32_t from_cme; - operator bool const() { return from_cme; } + operator bool const() { return to_cme || from_cme; } }; struct Config { diff --git a/src/lib/orderman/actors/ilink3/SessionStore.cpp b/src/lib/orderman/actors/ilink3/SessionStore.cpp index c0dc8e327..0b5aaf1b2 100644 --- a/src/lib/orderman/actors/ilink3/SessionStore.cpp +++ b/src/lib/orderman/actors/ilink3/SessionStore.cpp @@ -108,7 +108,7 @@ void save_session(std::string const &path, Session const &session) { } Session load_session(jetfix::SessionName const &sn, std::string const &path) { - auto result = Session{sn.as_uint64_t, 1, 0}; + auto result = Session{sn.as_uint64_t, 0, 0}; if (boost::filesystem::exists(path)) { std::ifstream f(path); uint64_t dummy; From c2f8e92b105b29dfae6d3846eda3069e6780acba Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Thu, 13 Aug 2020 08:42:43 -0400 Subject: [PATCH 3/6] Hopefully fixed the ilink3 filled qty reporting problem. This was crashing the spreader. --- src/lib/orderman/actors/OrderStore.cpp | 4 ++++ src/lib/orderman/orders/GLBXNativeOrder.cpp | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib/orderman/actors/OrderStore.cpp b/src/lib/orderman/actors/OrderStore.cpp index a5f350e7c..bedaf78cb 100644 --- a/src/lib/orderman/actors/OrderStore.cpp +++ b/src/lib/orderman/actors/OrderStore.cpp @@ -101,6 +101,8 @@ void OrderStore::on_order_state_update( for (auto const &ev : *u.events) { auto &rec_info = me->known_orders[ev.key]; + LOG(iLink3,Info, "Received order state update for {}", + ev.key.value()); auto &rec = rec_info.rec; bts_assert(rec.has_def()); rec.set_key(ev.key.value()); @@ -123,6 +125,8 @@ void OrderStore::on_order_definition_update( std::set modified_orders; if (u.events) for (auto const &ev : *u.events) { + LOG(iLink3,Info, "Received order definition update for {}", + ev.key.value()); auto &rec_info = me->known_orders[ev.key]; auto &rec = rec_info.rec; rec.set_key(ev.key.value()); diff --git a/src/lib/orderman/orders/GLBXNativeOrder.cpp b/src/lib/orderman/orders/GLBXNativeOrder.cpp index 36d297b96..5a1dbc2b1 100644 --- a/src/lib/orderman/orders/GLBXNativeOrder.cpp +++ b/src/lib/orderman/orders/GLBXNativeOrder.cpp @@ -861,7 +861,7 @@ void GLBXNativeOrder::on_exe_report(const NewExecutionReportEvent &er) { auto external_status = map_external_status(er.data->order_status); // filled quantity is reset back to 0 when the order is canceled - if (external_status != ORDER_STATUS_CANCELED) { + if (external_status != ORDER_STATUS_CANCELED ) { if (er.data->exec_type == api::orderman::EXEC_TYPE_PARTIAL_FILL || er.data->exec_type == api::orderman::EXEC_TYPE_FILL) { @@ -876,9 +876,9 @@ void GLBXNativeOrder::on_exe_report(const NewExecutionReportEvent &er) { (previous_sz + fill_sz); state.set_avg_price(new_avg); + state.set_filled_quantity(er.data->filled_quantity.value()); + state.set_last_price(er.data->last_price.value()); } - state.set_filled_quantity(er.data->filled_quantity.value()); - state.set_last_price(er.data->last_price.value()); } state.set_last_msg(er.data->msg); From 7489d70c95aa046567f8439eacf7f1eeec03380d Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 18 Aug 2020 07:51:16 -0400 Subject: [PATCH 4/6] Added some sql tools, added order deduction --- src/data_files/CMakeDeps.cmake | 2 +- src/lib/domain_model/Security.h | 19 ++ src/server/sparkd/cmds/QuickTest.cpp | 82 ++++--- src/server/sparkd/cmds/Spartan.cpp | 208 +++++++++++++++++- .../cmds/spartan/analysis/RunAnalysis.cpp | 17 +- .../md/mdp3/common/InstrumentSnapshot.cpp | 97 +++++--- .../md/mdp3/common/InstrumentSnapshot.h | 4 + .../cmds/spartan/md/mdp3/common/mdp3_common.h | 84 +++++-- .../md/mdp3/realtime/RealTimeHandler.cpp | 55 +++-- .../sparkd/cmds/spartan/sql_tools/msg.sql | 32 --- 10 files changed, 463 insertions(+), 137 deletions(-) delete mode 100644 src/server/sparkd/cmds/spartan/sql_tools/msg.sql diff --git a/src/data_files/CMakeDeps.cmake b/src/data_files/CMakeDeps.cmake index bddfdd268..e71f89cf5 100644 --- a/src/data_files/CMakeDeps.cmake +++ b/src/data_files/CMakeDeps.cmake @@ -1 +1 @@ -set(sources files/cert_secure_logon.txt;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js;files/small_es.pcap.xz) \ No newline at end of file +set(sources files/cert_secure_logon.txt;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js;files/small_es.pcap.xz) \ No newline at end of file diff --git a/src/lib/domain_model/Security.h b/src/lib/domain_model/Security.h index eafe97380..50205f815 100644 --- a/src/lib/domain_model/Security.h +++ b/src/lib/domain_model/Security.h @@ -115,6 +115,25 @@ inline Price better_price(api::Side side, Price p1, Price p2) { return is_equal_or_better(side, p1, p2) ? p1 : p2; } +// Returns the worst possible price representable in the Price data type for the side in question +inline Price worst_price(api::Side side) { + switch (side) { + case api::SIDE_BUY: + return Price {std::numeric_limits::min()}; + case api::SIDE_SELL: + return Price {std::numeric_limits::max()}; + default: + bts_assert(false); + break; + } + bts_assert(false); + return Price(0); +} + +inline Price best_price(api::Side side) { + return worst_price(other_side(side)); +} + inline Price decode_exchange_price(api::Security const &sec, Price in) { return Price(in.value() * sec.contract_spec().price_multiplier()); } diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index b2dc71354..29c2d103a 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -6,6 +6,7 @@ #include "lib/tracing/PerfLog.h" #include "lib/utils/datetime.h" +#include "lib/config/protos/logging.pb.h" #include "lib/domain_model/sod.h" #include "lib/logging/logging.h" #include "lib/utils/datetime.h" @@ -30,55 +31,80 @@ namespace bts { namespace cmd { using namespace bts::orderman; using namespace spartan::mdp3; +using namespace spartan; -void dump_snapshots() { - using namespace bts::spartan::mdp3; - using namespace bts::spartan; +struct TimeWriter { + mdp3::MDP3PacketView packet; + void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; } + void on_end_packet() { + std::cout << packet.sending_time() << std::endl; + } - auto file = "/home/ppinto/es_md/es_all.pcap"; - auto feed_map = guess_feed_map(file); + template void operator()(M const &) { + } +}; +void print_times(std::string const &file) { + struct PacketBuffers { + spartan::mdp3::MDP3Packet i; + spartan::mdp3::MDP3Packet n; + }; + auto feed_map = spartan::mdp3::guess_feed_map(file); bts_assert(feed_map); auto reader = PCapReader{file}; - struct JSON_Writer { - mdp3::MDP3PacketView packet; - void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; } - void on_end_packet() {} + mdp3::AB_Arbitrageur ab; - void operator()(::mdp3::SnapshotFullRefresh52 const &m) { - if (m.securityID() == 15234) - std::cout << "{\"SeqNum\": " << packet.seq_num() << ", " - << " \"SendTime\": " << packet.sending_time() << ", " - << " \"Size\": " << packet.size() << "," - << " \"Msg\": " << m << "}\n"; + reader.on_packet() = [&](auto const &info, auto const &payload) { + if ((info.dst_ip == feed_map.incremental[0].ip && + info.dst_port == feed_map.incremental[0].port)) { + TimeWriter tw; + parse_packet(MDP3PacketView {payload},tw); } - }; - - JSON_Writer writer; - reader.on_packet() = [&](auto const &, auto const &payload) { - parse_packet(mdp3::MDP3PacketView{payload}, writer); + return true; }; while (reader.read_next()) ; } -int QuickTest::execute(std::vector const &) { - // dump_snapshots(); +int QuickTest::execute(std::vector const &args) { + auto cfg = config::get(); + cfg.mutable_console()->set_default_level("Fatal"); + log::configure_logging(cfg); + + auto file = args[0]; + print_times(file); + return 0; + using namespace bts::spartan::mdp3; struct Obs : IChannelObserver { void on_recovery_complete( - std::vector const &) override {} + std::vector const &secs) override { + + for (auto s : secs) { + std::cout << "Bootstrap -----------------------------------------" + "------------------------------" + << std::endl; + std::cout << "Symbol: " << s->definition().symbol() << std::endl + << "Status: " << s->trading_status() << std::endl + << "Orders:\n" + << s->orders(); + } + } void on_real_time_event(MDP3Event const &ev) override { + for (auto s : ev.affected_securities) { if (s->order_changes().size() || s->fills().size()) { - std::cout << "symbol: " << s->definition().symbol() << "\n" - << "seq_num: " << ev.first_packet_sn << " to " - << ev.last_packet_sn << "\n" - << "changes:\n" + std::cout << "Event " << ev.transact_time + << " -----------------------------" << std::endl; + std::cout << "Symbol: " << s->definition().symbol() << std::endl + << "Status: " << s->trading_status() << std::endl + << "Seq Num: " << ev.first_packet_sn << " to " + << ev.last_packet_sn << std::endl + << "Orders:\n" << s->order_changes() << "Fills:\n" << s->fills(); } @@ -89,7 +115,7 @@ int QuickTest::execute(std::vector const &) { } } obs; - spartan::observe_channel("/home/ppinto/es_md/es_all.pcap", obs); + spartan::observe_channel(file, obs); return 0; } diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index 391cbe1ff..e98e84def 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -1,20 +1,216 @@ #include "Spartan.h" #include "server/sparkd/cmds/spartan/algo/RunTAS.h" #include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h" #include +#include #include namespace bts { namespace cmd { +int export_tables(std::string const &file, std::string const &out_dir) { + using namespace bts::spartan::mdp3; + + struct Obs : IChannelObserver { + + Obs(std::string const &out_dir) + : events(out_dir + "/events.csv"), + event_items(out_dir + "/event_items.csv"), + securities(out_dir + "/securities.csv") { + + events << "event_id, transact_time,first_packet_sn,last_packet_sn," + "first_packet_send_time,last_packet_send_time" + << std::endl; + + event_items + << "event_id, transact_time,security_id,trading_status,type," + "change,order_key,side," + "priority,price,quantity,trade_entry_id,aggressor" + << std::endl; + + securities + << "security_id,symbol,security_group,asset,last_update_time" + << std::endl; + } + + std::ofstream events; + std::ofstream event_items; + std::ofstream orders; + std::ofstream securities; + size_t event_id = 0; + + void on_recovery_complete( + std::vector const &secs) override { + for (auto s : secs) { + securities << s->security_id().value() << "," << s->symbol() << "," + << s->definition().security_group() << "," + << s->definition().asset() << "," + << s->definition().last_update_time() << std::endl; + + for (auto const &[id, order] : s->orders()) + event_items << 0 << "," // event_id + + << 0 << "," // transact time + + << s->security_id().value() << "," // security id + + << s->trading_status() << "," // trading status + + << "deduced_order," // type + + << "init," // change + + << id << "," // order key + + << api::Side_Name(order.side) << "," // side + + << order.priority.value() << "," // priority + + << order.price.price() << "," // price + + << order.quantity.value() << "," // quantity + + << "," // trade_entry_id + + << "n/a" // aggressor + + << std::endl; + } + } + + void on_real_time_event(MDP3Event const &ev) override { + event_id++; + + events << event_id << "," + + << ev.transact_time << "," + + << ev.first_packet_sn << "," + + << ev.last_packet_sn << "," + + << ev.first_packet_send_time << "," + + << ev.last_packet_send_time << std::endl; + + for (auto s : ev.affected_securities) { + for (auto const &change : s->order_changes()) + event_items << event_id << "," + + << ev.transact_time.value() << "," // transact time + + << s->security_id().value() << "," // security id + + << s->trading_status() << "," // trading status + + << "order_change," // type + + << change_name(change.change) << "," + + << change.order.key.value() << "," // order key + + << api::Side_Name(change.order.side) << "," // side + + << change.order.priority.value() << "," // priority + + << change.order.price.price() << "," // price + + << change.order.quantity.value() << "," // quantity + + << "," // trade_entry_id + + << "n/a" // aggressor + + << std::endl; + + for (auto const &f : s->fills()) + event_items << event_id << "," + + << ev.transact_time.value() << "," // transact time + + << s->security_id().value() << "," // security id + + << s->trading_status() << "," // trading status + + << "fill," // type + + << "," + + << f.key.value() << "," // order key + + << "," // side + + << "," // priority + + << f.price << "," // price + + << f.quantity << "," // quantity + + << f.trade_entry_id << "," // trade_entry_id + + << aggressor_name(f.aggressor) // aggressor + + << std::endl; + + if (s->security_id().value() == 12181 && event_id == 1065) + std::cout << "There is" << std::endl; + auto deduced_order = s->deduced_order_change(); + if (deduced_order) { + event_items << event_id << "," + + << ev.transact_time.value() << "," // transact time + + << s->security_id().value() << "," // security id + + << s->trading_status() << "," // trading status + + << "order," // type + + << change_name(deduced_order->change) << "," + + << deduced_order->order.key.value() + << "," // order key + + << api::Side_Name(deduced_order->order.side) + << "," // side + + << "," // priority + + << deduced_order->order.price.price() << "," // price + + << deduced_order->order.quantity.value() + << "," // quantity + + << "," // trade_entry_id + + << "n/a" // aggressor + + << std::endl; + } + } + } + + void on_real_time_channel_loss() override { + std::cout << "Observed loss, resetting\n"; + } + } obs(out_dir); + + spartan::observe_channel(file, obs); + return 0; +} + int Spartan::execute(std::vector const &args) { auto error = 1; if (args.size() > 0) { - if (args[0] == "pcap_to_csv") + if (args[0] == "pcap_to_msgs") error = spartan::convert_pcap_to_csv_summary(args); else if (args[0] == "pcap_to_json") error = spartan::convert_pcap_to_json(args); + else if (args[0] == "pcap_to_tables") + error = export_tables(args[1], args[2]); } if (error) std::cout << help(); @@ -22,15 +218,9 @@ int Spartan::execute(std::vector const &args) { } std::string Spartan::help() { - return fmt::format("\tpcap_to_csv pcap_file\n" - "\tfat_pcap_to_csv pcap_file\n" - "\tfat_pcap_to_json pcap_file\n" + return fmt::format("\tpcap_to_msgs pcap_file\n" "\tpcap_to_json pcap_file\n" - "\trun_tas_from_pcap pcap_file\n" - "\tdetect_gaps pcap_file\n" - "\tfat_pcap_to_snapshot_csv_summary\n" - "\ttest\n" - "\trun_tas_from_mcast\n"); + "\tpcap_to_tables pcap_file output_dir\n"); } } // namespace cmd diff --git a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp index 533fd61cd..ac6993f82 100644 --- a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp +++ b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp @@ -44,15 +44,24 @@ void cme_ts_to_str(uint64_t cme_ts, std::stringstream *out) { } struct JSON_Writer { + + std::stringstream json_buffer; + nlohmann::json json_obj; mdp3::MDP3PacketView packet; + void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; } void on_end_packet() {} template void operator()(M const &m) { - std::cout << "{\"SeqNum\": " << packet.seq_num() << ", " - << " \"SendTime\": " << packet.sending_time() << ", " - << " \"Size\": " << packet.size() << "," - << " \"Msg\": " << m << "}\n"; + json_obj["SeqNum"] = packet.seq_num(); + json_obj["SendTime"] = packet.sending_time() ; + json_obj["Size"] = packet.size(); + + json_buffer.str(""); + json_buffer.clear(); + json_buffer << m; + json_obj["Msg"] = nlohmann::json::parse(json_buffer.str()); + std::cout << json_obj.dump(4) << std::endl; } }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp index 18186924f..f5a72d139 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp @@ -249,32 +249,25 @@ void InstrumentSnapshot::clear_order_changes() { me->order_changes.clear(); } void InstrumentSnapshot::insert_order(MDP3Order const &order) { bts_assert_d(me->orders.count(order.key) == 0); me->orders.insert({order.key, order}); - me->order_changes.emplace_back(OrderChange{order.key, - order.price, - order.quantity, - OrderChange::added}); + me->order_changes.emplace_back(OrderChange{order, OrderChange::added}); } void InstrumentSnapshot::update_order(MDP3Order const &order) { auto it = me->orders.find(order.key); bts_assert_d(it != me->orders.end()); it->second = order; - me->order_changes.emplace_back(OrderChange{order.key, - order.price, - order.quantity, - OrderChange::updated}); + me->order_changes.emplace_back(OrderChange{order, OrderChange::updated}); } void InstrumentSnapshot::remove_order(MDP3OrderKey id) { - bts_assert_d(me->orders.count(id)); - me->orders.erase(id); - me->order_changes.emplace_back(OrderChange{id, - {}, {}, - OrderChange::removed}); + auto it = me->orders.find(id); + bts_assert(it != me->orders.end()); + me->order_changes.emplace_back( + OrderChange{it->second, OrderChange::removed}); + me->orders.erase(it); } -Book InstrumentSnapshot::build_book_from_orders (api::Side side) const -{ +Book InstrumentSnapshot::build_book_from_orders(api::Side side) const { auto invert = side == api::SIDE_BUY ? -1 : 1; auto book_idx = std::map(); for (auto &o : orders()) @@ -294,7 +287,7 @@ Book InstrumentSnapshot::build_book_from_orders (api::Side side) const return mbo_book; } -void InstrumentSnapshot::set_books_from_orders() { +void InstrumentSnapshot::set_books_from_orders() { me->books[0] = build_book_from_orders(api::SIDE_BUY); me->books[1] = build_book_from_orders(api::SIDE_SELL); } @@ -337,24 +330,58 @@ void InstrumentSnapshot::remove_fill(MDP3TradeEntryID tid, MDP3OrderKey oid) { void InstrumentSnapshot::clear_fills() { me->fills.clear(); } -auto get_orders(InstrumentSnapshot const &snap, api::Side side) { - std::vector result; - for (auto &o : snap.orders()) { - auto &order = o.second; - if (order.side == side) - result.push_back(order); +MaybeOrderChange InstrumentSnapshot::deduced_order_change() const { + if (fills().empty()) + return {}; + + Fills aggressor_fills; + std::copy_if(fills().begin(), fills().end(), + std::back_inserter(aggressor_fills), [](auto const &f) { + return f.aggressor == MDP3Fill::THIS_BUY || + f.aggressor == MDP3Fill::THIS_SELL; + }); + + if (aggressor_fills.empty()) + return {}; + + OrderChange result; + result.order.key = aggressor_fills[0].key; + result.order.security = security_id(); + result.order.side = aggressor_fills[0].aggressor == MDP3Fill::THIS_BUY + ? api::SIDE_BUY + : api::SIDE_SELL; + result.order.priority = {}; + result.order.price = aggressor_fills[0].price; + result.order.quantity = MDP3Quantity{0}; + + for (auto &f : aggressor_fills) { + if (securities::is_better(result.order.side, + Price(result.order.price.price()), + Price(f.price.price()))) + result.order.price = f.price; + result.order.quantity.value() += result.order.quantity.value(); } - std::sort(result.begin(), result.end(), - [side](auto const &o1, auto const &o2) { - if (o1.price.price() != o2.price.price()) { - if (side == api::SIDE_BUY) - return o2.price.price() < o1.price.price(); - else - return o1.price.price() < o2.price.price(); - } else - return o1.priority.value() < o2.priority.value(); - }); - return result; + + // Lets find out if this is a new or existing order. If it was an + // existing order then we must have a delete or replace in our list + // of order changes. If we do not then this must be a new order. + result.change = OrderChange::added; + for (auto &o : order_changes()) + if (o.order.key == result.order.key) { + result.change = OrderChange::updated; + break; + } + + // Now lets find out if this was a full fill or a partial fill. A + // partial fill would have left the order in the books. In that + // case we can use the information in the book to correct our + // estimated quantity and prices from above. + auto it = orders().find(result.order.key); + if (it != orders().end()) { + result.order.quantity.value() += it->second.quantity.value(); + result.order.price = it->second.price; + } + return {result}; } std::ostream &operator<<(std::ostream &out, InstrumentSnapshot const &snap) { @@ -374,11 +401,11 @@ std::ostream &operator<<(std::ostream &out, InstrumentSnapshot const &snap) { out << "Order Changes -----------------------------------\n"; for (auto &change : snap.order_changes()) out << change << "\n"; - auto orders = get_orders(snap, api::SIDE_BUY); + auto orders = sort_orders(snap.orders(), api::SIDE_BUY); out << "Buy Orders -----------------------------------\n"; for (auto &order : orders) out << order << "\n"; - orders = get_orders(snap, api::SIDE_SELL); + orders = sort_orders(snap.orders(), api::SIDE_SELL); out << "Sell Orders ----------------------------------\n"; for (auto &order : orders) out << order << "\n"; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h index 7ebe26bf6..ca136b899 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h @@ -62,6 +62,10 @@ class InstrumentSnapshot { void remove_order(MDP3OrderKey); Book build_book_from_orders (api::Side ) const; void set_books_from_orders() ; + // Deduced order changes, that is the implied order change that + // would explain the last event processed in this snapshot as + // described by order_changes() and fills() + MaybeOrderChange deduced_order_change() const; // Reset books in response to a channel reset message void reset_books(); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h index 51c126903..2a36532ca 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h @@ -52,8 +52,8 @@ class BookLevel { }; inline std::ostream &operator<<(std::ostream &s, BookLevel const &l) { - s << " " << l.quantity.value() << " @ " << l.price.price() << " (" << l.n_orders - << " orders)"; + s << " " << l.quantity.value() << " @ " << l.price.price() << " (" + << l.n_orders << " orders)"; return s; } @@ -67,23 +67,26 @@ struct MDP3Order { MDP3Quantity quantity; }; + struct OrderChange { - MDP3OrderKey key; - md::IntPrice price; - MDP3Quantity quantity; + MDP3Order order; enum ChangeType { added, removed, updated } change; }; +using MaybeOrderChange = std::optional; -inline std::ostream &operator<<(std::ostream &s, OrderChange const &ch) { - s << " ok: " << ch.key.value() << " " - << " price: " << ch.price.price() - << " qty: " << ch.quantity.value() - << " " << (ch.change == OrderChange::added - ? "create" - : (ch.change == OrderChange::updated ? "update" : "delete")); - return s; +inline const char*change_name (OrderChange::ChangeType ct) { + switch(ct) { + case OrderChange::added: + return "added"; + case OrderChange::removed: + return "removed"; + case OrderChange::updated: + return "updated"; + } + return ""; } + using OrderChanges = std::vector; inline std::ostream &operator<<(std::ostream &s, MDP3Order const &order) { @@ -96,6 +99,15 @@ inline std::ostream &operator<<(std::ostream &s, MDP3Order const &order) { return s; } +inline std::ostream &operator<<(std::ostream &s, OrderChange const &ch) { + s << " " + << (ch.change == OrderChange::added + ? "create" + : (ch.change == OrderChange::updated ? "update" : "delete")) + << ch.order; + return s; +} + inline std::ostream &operator<<(std::ostream &s, OrderChanges const &changes) { for (auto c : changes) s << " " << c << "\n"; @@ -103,16 +115,60 @@ inline std::ostream &operator<<(std::ostream &s, OrderChanges const &changes) { } using Orders = std::unordered_map; + + using MDP3TradeEntryID = utils::ValueWrapper; struct MDP3Fill { MDP3OrderKey key; md::IntPrice price; MDP3Quantity quantity; + MDP3Quantity summary_quantity; MDP3TradeEntryID trade_entry_id; - enum Aggressor { THIS, OTHER, IMPLIED } aggressor; + enum Aggressor {THIS_BUY, THIS_SELL, OTHER, IMPLIED } aggressor; }; +inline const char*aggressor_name (MDP3Fill::Aggressor ag) { + switch(ag) { + case MDP3Fill::THIS_BUY: + return "yes_buy"; + case MDP3Fill::THIS_SELL: + return "yes_sell"; + case MDP3Fill::OTHER: + return "no"; + case MDP3Fill::IMPLIED: + return "implied"; + } + return ""; +} + +inline std::vector sort_orders(Orders const &orders, api::Side side) { + std::vector result; + for (auto &o : orders) { + auto &order = o.second; + if (order.side == side) + result.push_back(order); + } + std::sort(result.begin(), result.end(), + [side](auto const &o1, auto const &o2) { + if (o1.price.price() != o2.price.price()) { + if (side == api::SIDE_BUY) + return o2.price.price() < o1.price.price(); + else + return o1.price.price() < o2.price.price(); + } else + return o1.priority.value() < o2.priority.value(); + }); + return result; +} + +inline std::ostream &operator<<(std::ostream &out, Orders const &orders) { + auto sorted = sort_orders(orders, api::SIDE_BUY); + for (auto &o : sorted) + out << o << std::endl; + return out; +} + inline std::ostream &operator<<(std::ostream &s, MDP3Fill const &fill) { s << " ok: " << fill.key.value() << " " << "price: " << fill.price.price() << " " diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp index e448b7d54..8eb37f8e2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp @@ -12,6 +12,7 @@ struct RealTimeHandler::TradeSummary { ::mdp3::MDUpdateAction::Value update_action; InstrumentSnapshot *security; md::IntPrice price; + MDP3Quantity quantity; size_t n_orders; ::mdp3::AggressorSide::Value aggressor; MDP3TradeEntryID trade_entry_id; @@ -27,7 +28,7 @@ struct RealTimeHandler::PImpl { PImpl(IRealTimeObserver &o) : instruments(), current_packet_send_time(MDP3TimeStamp{0}), current_packet_sn(0), current_event(), observer(o), pending_ts(), - pending_order_detail() {} + pending_trade_order_detail() {} std::unordered_map> instruments; @@ -37,7 +38,7 @@ struct RealTimeHandler::PImpl { MDP3Event current_event; IRealTimeObserver &observer; std::vector pending_ts; - std::vector pending_order_detail; + std::vector pending_trade_order_detail; }; struct RealTimeHandler::EventGuard { @@ -93,7 +94,7 @@ struct RealTimeHandler::EventGuard { owner.on_end_of_event(); info.reset(); owner.me->pending_ts.clear(); - owner.me->pending_order_detail.clear(); + owner.me->pending_trade_order_detail.clear(); } } @@ -434,6 +435,7 @@ void RealTimeHandler::operator()( summary.security = iptr; summary.price = md::IntPrice{md_entry.mDEntryPx().mantissa(), md_entry.mDEntryPx().exponent()}; + summary.quantity = MDP3Quantity{md_entry.mDEntrySize()}; summary.n_orders = md_entry.numberOfOrders(); summary.aggressor = md_entry.aggressorSide(); summary.trade_entry_id = MDP3TradeEntryID{md_entry.mDTradeEntryID()}; @@ -444,7 +446,7 @@ void RealTimeHandler::operator()( TradeOrderDetail detail; detail.order_id = MDP3OrderKey{order_id_entry.orderID()}; detail.quantity = MDP3Quantity{order_id_entry.lastQty()}; - me->pending_order_detail.emplace_back(detail); + me->pending_trade_order_detail.emplace_back(detail); }); } @@ -552,25 +554,50 @@ void RealTimeHandler::operator()(::mdp3::ChannelReset4 &r) { } void RealTimeHandler::on_end_of_event() { - // trade summary messages have to be processed together once all of + // Trade summary messages have to be processed together once all of // them have been received. We do this here, because we know the // event is over and therefore so are the trade summaries size_t current_order_detail = 0; for (auto &summary : me->pending_ts) { for (size_t i = 0; i < summary.n_orders; i++) { - auto &detail = me->pending_order_detail[current_order_detail++]; + auto &detail = me->pending_trade_order_detail[current_order_detail++]; MDP3Fill f; f.key = detail.order_id; f.price = summary.price; f.quantity = detail.quantity; + f.summary_quantity = summary.quantity; f.trade_entry_id = summary.trade_entry_id; - if (summary.aggressor != ::mdp3::AggressorSide::NoAggressor) { - if (i == 0) - f.aggressor = MDP3Fill::THIS; - else + if (i == 0) { + switch (summary.aggressor) { + case ::mdp3::AggressorSide::NoAggressor: + f.aggressor = MDP3Fill::IMPLIED; + break; + case ::mdp3::AggressorSide::Buy: + f.aggressor = MDP3Fill::THIS_BUY; + break; + case ::mdp3::AggressorSide::Sell: + f.aggressor = MDP3Fill::THIS_SELL; + break; + case ::mdp3::AggressorSide::NULL_VALUE: + bts_assert(false); + break; + } + } + else { + switch (summary.aggressor) { + case ::mdp3::AggressorSide::NoAggressor: + f.aggressor = MDP3Fill::IMPLIED; + break; + case ::mdp3::AggressorSide::Buy: + case ::mdp3::AggressorSide::Sell: f.aggressor = MDP3Fill::OTHER; - } else - f.aggressor = MDP3Fill::IMPLIED; + break; + case ::mdp3::AggressorSide::NULL_VALUE: + bts_assert(false); + break; + } + } + switch (summary.update_action) { case ::mdp3::MDUpdateAction::New: summary.security->add_fill(f); @@ -589,8 +616,8 @@ void RealTimeHandler::on_end_of_event() { me->observer.on_real_time_event(me->current_event); for (auto s : me->current_event.affected_securities) { - const_cast(s)->clear_fills(); - const_cast(s)->clear_order_changes(); + const_cast(s)->clear_fills(); + const_cast(s)->clear_order_changes(); } } diff --git a/src/server/sparkd/cmds/spartan/sql_tools/msg.sql b/src/server/sparkd/cmds/spartan/sql_tools/msg.sql deleted file mode 100644 index 294a56a87..000000000 --- a/src/server/sparkd/cmds/spartan/sql_tools/msg.sql +++ /dev/null @@ -1,32 +0,0 @@ -drop table msg; -create table if not exists -msg ( - "HumanTransactTime" TEXT, - "HumanSendingTime" TEXT, - "Latency" numeric, - "TransactTime" numeric, - "SendingTime" numeric, - "SeqNum" numeric, - "MsgType" TEXT, - "TimeInQueue" numeric, - "MatchEventIndicator" numeric -); - -.mode csv -.import "/home/ppinto/es_md/es_i.csv" msg - -drop view event_len; -create view event_len as -select transacttime, count(*) len from msg group by transacttime; - -drop view summary_event_len; -create view summary_event_len as -select - count(*) n_events, - msg_count len -from - (select transacttime, count(*) msg_count from msg group by transacttime) t -group by - len - - From ec3d642ceec4a48d86e4503bc0b0562f8920e7fe Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 18 Aug 2020 10:58:42 -0400 Subject: [PATCH 5/6] Better export to CSV. --- src/server/sparkd/cmds/Spartan.cpp | 16 ++-- .../md/mdp3/common/InstrumentSnapshot.cpp | 4 +- .../sparkd/cmds/spartan/sql_tools/md.sql | 88 +++++++++---------- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index e98e84def..2889c2d1f 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -21,18 +21,18 @@ int export_tables(std::string const &file, std::string const &out_dir) { event_items(out_dir + "/event_items.csv"), securities(out_dir + "/securities.csv") { - events << "event_id, transact_time,first_packet_sn,last_packet_sn," - "first_packet_send_time,last_packet_send_time" + events << "eid, t_time,first_sn,last_sn," + "first_st,last_st" << std::endl; event_items - << "event_id, transact_time,security_id,trading_status,type," - "change,order_key,side," - "priority,price,quantity,trade_entry_id,aggressor" + << "eid, t_time,sid,status,type," + "change,ok,side," + "pri,price,qty,tid,agg" << std::endl; securities - << "security_id,symbol,security_group,asset,last_update_time" + << "sid,symbol,group,asset,last_t" << std::endl; } @@ -59,7 +59,7 @@ int export_tables(std::string const &file, std::string const &out_dir) { << s->trading_status() << "," // trading status - << "deduced_order," // type + << "order_change," // type << "init," // change @@ -167,7 +167,7 @@ int export_tables(std::string const &file, std::string const &out_dir) { << s->trading_status() << "," // trading status - << "order," // type + << "deduced_order," // type << change_name(deduced_order->change) << "," diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp index f5a72d139..b7333794b 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp @@ -331,7 +331,7 @@ void InstrumentSnapshot::remove_fill(MDP3TradeEntryID tid, MDP3OrderKey oid) { void InstrumentSnapshot::clear_fills() { me->fills.clear(); } MaybeOrderChange InstrumentSnapshot::deduced_order_change() const { - if (fills().empty()) + if (fills().empty() || trading_status() != ::mdp3::SecurityTradingStatus::ReadyToTrade ) return {}; Fills aggressor_fills; @@ -359,7 +359,7 @@ MaybeOrderChange InstrumentSnapshot::deduced_order_change() const { Price(result.order.price.price()), Price(f.price.price()))) result.order.price = f.price; - result.order.quantity.value() += result.order.quantity.value(); + result.order.quantity.value() += f.quantity.value(); } // Lets find out if this is a new or existing order. If it was an diff --git a/src/server/sparkd/cmds/spartan/sql_tools/md.sql b/src/server/sparkd/cmds/spartan/sql_tools/md.sql index 9aba06d33..ae517c93c 100644 --- a/src/server/sparkd/cmds/spartan/sql_tools/md.sql +++ b/src/server/sparkd/cmds/spartan/sql_tools/md.sql @@ -25,38 +25,38 @@ -- (select transacttime, count(*) msg_count from msg group by transacttime) t -- group by -- len -drop table event_items; -CREATE TABLE event_items( - event_id numeric, - "transact_time" numeric, - "security_id" numeric, - "trading_status" TEXT, - "type" TEXT, - "change" TEXT, - "order_key" numeric, - "side" TEXT, - "priority" numeric, - "price" numeric, - "quantity" numeric, - "trade_entry_id" numeric, - "aggressor" TEXT -); drop table events; CREATE TABLE events( - event_id numeric, - "transact_time" numeric, - "first_packet_sn" numeric, - "last_packet_sn" numeric, - "first_packet_send_time" numeric, - "last_packet_send_time" numeric + eid numeric, + "t_time" numeric, + "first_sn" numeric, + "last_sn" numeric, + "first_st" numeric, + "last_st" numeric +); +drop table event_items; +CREATE TABLE event_items( + eid numeric, + "t_time" numeric, + "sid" numeric, + "status" TEXT, + "type" TEXT, + "change" TEXT, + "ok" numeric, + "side" TEXT, + "pri" numeric, + "price" numeric, + "qty" numeric, + "tid" numeric, + "agg" TEXT ); drop table securities; CREATE TABLE securities( - "security_id" numeric, + "sid" numeric, "symbol" TEXT, - "security_group" TEXT, + "group" TEXT, "asset" TEXT, - "last_update_time" numeric + "last_t" numeric ); .mode csv .import event_items.csv event_items @@ -71,47 +71,47 @@ select *, (case when type='order' then 1 else 0 end) is_order, (case when type='fill' then 1 else 0 end) is_fill, - (case when aggressor='1' then 1 else 0 end) is_aggressor + (case when agg='1' then 1 else 0 end) is_agg from event_items where - trading_status='ReadyToTrade' + status='ReadyToTrade' order by - event_id; + eid; drop view orders_fills; create view orders_fills as select - transact_time, - event_id, + t_time, + eid, sum(is_order) order_count, sum(is_fill) fill_count, - sum(is_aggressor) aggressor_count + sum(is_agg) agg_count from event_items_aux where - trading_status='ReadyToTrade' + status='ReadyToTrade' group by - transact_time,event_id ; + t_time,eid ; drop view simple_fills; create view simple_fills as -select * from event_items where event_id in (select event_id from orders_Fills where fill_count=2) ; +select * from event_items where eid in (select eid from orders_Fills where fill_count=2) ; -drop view aggressor_fills; -create view aggressor_fills as -select * from event_items where aggressor='yes' ; +drop view agg_fills; +create view agg_fills as +select * from event_items where agg='yes' ; -drop view aggressor_orders; -create view aggressor_orders as +drop view agg_orders; +create view agg_orders as select - event_items.event_id, - event_items.order_key aggressor_ok + event_items.eid, + event_items.ok agg_ok from - event_items, aggressor_fills + event_items, agg_fills where event_items.type='order' and - event_items.event_id=aggressor_fills.event_id and - event_items.order_key=aggressor_fills.order_key; + event_items.eid=agg_fills.eid and + event_items.ok=agg_fills.ok; From 86ca092f18f3f8aaffd06dc134c0e3e4dc180181 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 18 Aug 2020 13:06:58 -0400 Subject: [PATCH 6/6] Fixed bug that was preventing import of oil data. --- .../sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp index 8eb37f8e2..0ea3c6dd1 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp @@ -178,8 +178,10 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) { m.noMDEntries().forEach([&ev, this, &cache, &cache_idx](auto &md_entry) { auto sid = MDP3SecurityID{md_entry.securityID()}; auto iptr = find_instrument(sid); - if (!iptr) + if (!iptr) { + cache_idx++; return; + } ev.record_security_change(iptr); auto side = convert_side(md_entry.mDEntryType());