diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index 7998d2477..afe9c05d9 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/analysis/ActionableEventAnalyzer.cpp;cmds/spartan/analysis/RunAnalysis.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index ca518c550..a90b4bbfe 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -10,7 +10,6 @@ #include "lib/domain_model/sod.h" #include "lib/logging/logging.h" #include "lib/utils/datetime.h" -#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" @@ -29,40 +28,6 @@ namespace bts { namespace cmd { -using namespace bts::orderman; -using namespace spartan::mdp3; -using namespace spartan; - -struct TimeWriter { - mdp3::MDP3PacketView packet; - void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; } - void on_end_packet() { std::cout << packet.sending_time() << std::endl; } - - template void operator()(M const &) {} -}; -void print_times(std::string const &file) { - struct PacketBuffers { - spartan::mdp3::MDP3Packet i; - spartan::mdp3::MDP3Packet n; - }; - auto feed_map = spartan::mdp3::guess_feed_map(file); - bts_assert(feed_map); - auto reader = PCapReader{file}; - - mdp3::AB_Arbitrageur ab; - - reader.on_packet() = [&](auto const &info, auto const &payload) { - if ((info.dst_ip == feed_map.incremental[0].ip && - info.dst_port == feed_map.incremental[0].port)) { - TimeWriter tw; - parse_packet(MDP3PacketView{payload}, tw); - } - - return true; - }; - while (reader.read_next()) - ; -} int QuickTest::execute(std::vector const &args) { @@ -128,7 +93,7 @@ int QuickTest::execute(std::vector const &args) { } } obs; - spartan::observe_channel(file, obs); + spartan::mdp3::observe_channel(file, obs); return 0; } diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index af4cf607d..04e33c57b 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -1,6 +1,5 @@ #include "Spartan.h" #include "server/sparkd/cmds/spartan/algo/RunTAS.h" -#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" #include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" @@ -18,153 +17,15 @@ namespace bts { namespace cmd { -int slim_pcap(std::string const &file, std::string const &out_dir) { - return !spartan::mdp3::slim_down_pcap(file, out_dir); -} - -int export_tables(std::string const &file, std::string const &out_dir) { - using namespace bts::spartan::mdp3; - - struct Obs : IChannelObserver { - - Obs(std::string const &out_dir) - : events(out_dir + "/events.csv"), - event_items(out_dir + "/event_items.csv"), - securities(out_dir + "/securities.csv") { - - events << "eid, t_time,first_sn,last_sn," - "first_st,last_st" - << std::endl; - - event_items << "eid, t_time,sid,status,type," - "change,ok,side," - "pri,price,qty,tid,agg" - << std::endl; - - securities << "sid,symbol,group,asset,last_t" << std::endl; - } - - std::ofstream events; - std::ofstream event_items; - std::ofstream orders; - std::ofstream securities; - size_t event_id = 0; - - void on_recovery_complete( - std::vector const &secs) override { - for (auto s : secs) { - securities << s->security_id().value() // security id - << "," << s->symbol() << "," // symbol - << s->definition().security_group() << "," // group - << s->definition().asset() << "," // asset - << s->definition().last_update_time() // last update time - << std::endl; - - for (auto const &[id, order] : s->orders()) - event_items << 0 << "," // event_id - << 0 << "," // transact time - << s->security_id().value() << "," // security id - << s->trading_status() << "," // trading status - << "order_change," // type - << "init," // change - << id << "," // order key - << api::Side_Name(order.side) << "," // side - << order.priority.value() << "," // priority - << order.price.price() << "," // price - << order.quantity.value() << "," // quantity - << "," // trade_entry_id - << "n/a" // aggressor - << std::endl; - } - } - - void on_real_time_event(MDP3Event const &ev) override { - event_id++; - events << event_id << "," // event id - << ev.transact_time << "," // transactr time - << ev.first_packet_sn << "," // first packet_sn - << ev.last_packet_sn << "," // last_packet_sn - << ev.first_packet_send_time << "," // first packet send time - << ev.last_packet_send_time // last packet send time - << std::endl; - - for (auto s : ev.affected_securities) { - for (auto const &change : s->order_changes()) - event_items << event_id << "," // event id - << ev.transact_time.value() // transact time - << "," // transact time - << s->security_id().value() << "," // security id - << s->trading_status() << "," // trading status - << "order_change," // type - << change_name(change.change) << "," // change type - << change.order.key.value() << "," // order key - << api::Side_Name(change.order.side) << "," // side - << change.order.priority.value() << "," // priority - << change.order.price.price() << "," // price - << change.order.quantity.value() << "," // quantity - << "," // trade_entry_id - << "n/a" // aggressor - << std::endl; - - for (auto const &f : s->fills()) - event_items << event_id << "," // event id - << ev.transact_time.value() << "," // transact time - << s->security_id().value() << "," // security id - << s->trading_status() << "," // trading status - << "fill," // type - << "," // null - << f.key.value() << "," // order key - << "," // side - << "," // priority - << f.price << "," // price - << f.quantity << "," // quantity - << f.trade_entry_id << "," // trade_entry_id - << aggressor_name(f.aggressor) // aggressor - << std::endl; - - auto deduced_order = s->deduced_order_change(); - if (deduced_order) { - event_items << event_id << "," // event id - << ev.transact_time.value() << "," // transact time - << s->security_id().value() << "," // security id - << s->trading_status() << "," // trading status - << "deduced_order," // type - << change_name(deduced_order->change) << "," - << deduced_order->order.key.value() - << "," // order key - << api::Side_Name(deduced_order->order.side) - << "," // side - << "," // priority - << deduced_order->order.price.price() << "," // price - << deduced_order->order.quantity.value() - << "," // quantity - << "," // trade_entry_id - << "n/a" // aggressor - << std::endl; - } - } - } - - void on_real_time_channel_loss() override { - std::cout << "Observed loss, resetting\n"; - } - } obs(out_dir); - - spartan::observe_channel(file, obs); - return 0; -} - int Spartan::execute(std::vector const &args) { auto error = 1; if (args.size() > 0) { - if (args[0] == "pcap_to_msgs") - error = spartan::convert_pcap_to_csv_summary(args); - else if (args[0] == "pcap_to_json") - error = spartan::convert_pcap_to_json(args); - else if (args[0] == "pcap_to_tables") - error = export_tables(args[1], args[2]); - else if (args[0] == "slim_pcap") - error = slim_pcap(args[1], args[2]); + if (args[0] == "json" && args.size() == 2) + error = spartan::mdp3::to_json(args[1]); + else if (args[0] == "compress" && args.size() == 3) + error = spartan::mdp3::compress(args[1],args[2]); + else if (args[0] == "csv" && args.size() == 3) + error = spartan::mdp3::to_csv(args[1], args[2]); } if (error) std::cout << help(); @@ -172,10 +33,9 @@ int Spartan::execute(std::vector const &args) { } std::string Spartan::help() { - return fmt::format("\tpcap_to_msgs pcap_file\n" - "\tpcap_to_json pcap_file\n" - "\tslim_pcap in_pcap out_pcap\n" - "\tpcap_to_tables pcap_file output_dir\n"); + return fmt::format("\tjson \n" + "\tcompress \n" + "\tcsv "); } } // namespace cmd diff --git a/src/server/sparkd/cmds/spartan/analysis/ActionableEventAnalyzer.cpp b/src/server/sparkd/cmds/spartan/analysis/ActionableEventAnalyzer.cpp deleted file mode 100644 index 3c32c39d8..000000000 --- a/src/server/sparkd/cmds/spartan/analysis/ActionableEventAnalyzer.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "ActionableEventAnalyzer.h" diff --git a/src/server/sparkd/cmds/spartan/analysis/ActionableEventAnalyzer.h b/src/server/sparkd/cmds/spartan/analysis/ActionableEventAnalyzer.h deleted file mode 100644 index a2d70e437..000000000 --- a/src/server/sparkd/cmds/spartan/analysis/ActionableEventAnalyzer.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once -#include "../md/mdp3/parsing/MDP3Parser.h" -#include "lib/utils/stats.h" -#include -#include - -namespace bts { -namespace spartan { - -struct ActionableEventAnalyzer { - - struct Event { - uint64_t transact_time; - uint64_t send_time; - uint64_t actionable_window; - uint64_t exchange_latency; - uint64_t time_from_last_transaction; - }; - std::vector events; - template void operator()(mdp3::MDP3PacketHeader const&ph,mdp3::MDP3MsgHeader const &, const M &m) { - Event e{m.transactTime(), ph.sending_time, 0, - ph.sending_time - m.transactTime(), 0}; - if (events.size()) { - auto &previous = events.back(); - previous.actionable_window = - std::max(0l, (int64_t)(previous.send_time - m.transactTime())); - previous.time_from_last_transaction = m.transactTime() - previous.transact_time; - } - events.push_back(e); - } - - std::string single_stat(uint64_t Event::*field) { - std::ostringstream str; - str.precision(0); - str << std::fixed << "\tmin: " - << stats::min(events, [field](auto const &ev) { return ev.*field; }) - << "\t\tmed: " - << stats::med(events, [field](auto const &ev) { return ev.*field; }) - << "\t\tmax: " - << stats::max(events, [field](auto const &ev) { return ev.*field; }); - return str.str(); - } - - ~ActionableEventAnalyzer() { - std::cout << "Data points:\n \t\t" << events.size() - - << "\nExchange Latency:\n" - << single_stat(&Event::exchange_latency) - - << "\nTransaction Gap:\n" - << single_stat(&Event::time_from_last_transaction) - - << "\nActionable Window:\n" - << single_stat(&Event::actionable_window) << "\n"; - } -}; - -} // namespace spartan -} // namespace bts diff --git a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp deleted file mode 100644 index 68fffc8c6..000000000 --- a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.cpp +++ /dev/null @@ -1,251 +0,0 @@ -#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h" -#include "../md/mdp3/parsing/MDP3Parser.h" -#include "../util/PCapReader.h" -#include "lib/utils/bts_chrono.h" -#include "lib/utils/overloaded.h" -#include "lib/utils/stats.h" -#include "lib/utils/type_name.h" -#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" -#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" -#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h" -#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h" -#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" -#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" -#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" - -#include -#include -#include -#include - -namespace bts { -namespace spartan { - -struct PacketBuffers { - mdp3::MDP3Packet i; - mdp3::MDP3Packet n; -}; - -mdp3::AB_Arbitrageur ab; - -struct JSON_Writer { - - std::stringstream json_buffer; - nlohmann::json json_obj; - mdp3::MDP3PacketView packet; - - void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; } - void on_end_packet() {} - - template void operator()(M const &m) { - json_obj["SeqNum"] = packet.seq_num(); - json_obj["SendTime"] = packet.sending_time(); - json_obj["Size"] = packet.size(); - - json_buffer.str(""); - json_buffer.clear(); - json_buffer << m; - json_obj["Msg"] = nlohmann::json::parse(json_buffer.str()); - std::cout << json_obj.dump(4) << std::endl; - } -}; - -int convert_pcap_to_json(std::vector const &args) { - std::array buffers; - - auto reader = PCapReader{args[1]}; - JSON_Writer writer; - - reader.on_packet() = [&](auto const &, auto const &payload) { - parse_packet(mdp3::MDP3PacketView{payload}, writer); - return true; - }; - while (reader.read_next()) - ; - return 0; -} - -template transactTime()) = 0> -uint64_t get_transact_time(M &m) { - return m.transactTime(); -} - -template lastUpdateTime()) = 0> -uint64_t get_transact_time(M &m) { - return m.lastUpdateTime(); -} - -struct CSVPrinter { - - std::stringstream stime_buffer; - std::stringstream ttime_buffer; - size_t previous_t_time = 0; - size_t previous_s_time = 0; - mdp3::MDP3PacketView packet; - - CSVPrinter() { - std::cout - << "HumanTransactTime,HumanSendingTime," - "Latency,TransactTime,SendingTime,SeqNum,MsgType,TimeInQueue," - "MatchEventindicator" - << std::endl; - } - - void on_begin_packet(mdp3::MDP3PacketView const &v) { packet = v; } - void on_end_packet() {} - - template void print(M &m) { - stime_buffer.str(""); - stime_buffer.clear(); - mdp3::cme_ts_to_date_str(packet.sending_time(), &stime_buffer); - ttime_buffer.str(""); - ttime_buffer.clear(); - mdp3::cme_ts_to_date_str(get_transact_time(m), &ttime_buffer); - - auto t = get_transact_time(m); - std::cout << ttime_buffer.str() << "," << stime_buffer.str() << "," - << (packet.sending_time() - t) / 1000 << "," << t << "," - << packet.sending_time() << "," << packet.seq_num() << "," - << utils::type_name() << "," - << (t < previous_s_time ? ((previous_s_time - t) / 1000) : 0) - << "," << m.matchEventIndicator().endOfEvent() << std::endl; - previous_s_time = packet.sending_time(); - previous_t_time = t; - } - - void operator()(::mdp3::MDIncrementalRefreshOrderBook43 &m) { print(m); } - void operator()(::mdp3::MDIncrementalRefreshOrderBook47 &m) { print(m); } - void operator()(::mdp3::MDIncrementalRefreshTradeSummary42 &m) { print(m); } - void operator()(::mdp3::MDIncrementalRefreshTradeSummary48 &m) { print(m); } - void operator()(::mdp3::MDIncrementalRefreshVolume37 &m) { print(m); } - void operator()(::mdp3::MDIncrementalRefreshDailyStatistics33 &m) { - print(m); - } - void operator()(::mdp3::MDIncrementalRefreshDailyStatistics49 &m) { - print(m); - } - void operator()(::mdp3::MDIncrementalRefreshLimitsBanding34 &m) { print(m); } - void operator()(::mdp3::MDIncrementalRefreshLimitsBanding50 &m) { print(m); } - void operator()(::mdp3::MDIncrementalRefreshBook46 &m) { print(m); } - void operator()(::mdp3::MDInstrumentDefinitionFuture54 &m) { print(m); } - void operator()(::mdp3::MDIncrementalRefreshSessionStatistics35 &m) { - print(m); - } - void operator()(::mdp3::MDIncrementalRefreshSessionStatistics51 &m) { - print(m); - } - void operator()(::mdp3::MDInstrumentDefinitionFuture27 &m) { print(m); } - void operator()(::mdp3::ChannelReset4 &m) { print(m); } - void operator()(::mdp3::MDIncrementalRefreshBook32 &m) { print(m); } - void operator()(::mdp3::MDInstrumentDefinitionOption41 &m) { print(m); } - void operator()(::mdp3::MDInstrumentDefinitionOption55 &m) { print(m); } - void operator()(::mdp3::MDInstrumentDefinitionSpread29 &m) { print(m); } - void operator()(::mdp3::MDInstrumentDefinitionSpread56 &m) { print(m); } - void operator()(::mdp3::QuoteRequest39 &m) { print(m); } - void operator()(::mdp3::SecurityStatus30 &m) { print(m); } -}; - -int convert_pcap_to_csv_summary(std::vector const &args) { - std::array buffers; - - auto feed_map = mdp3::guess_feed_map(args[1]); - bts_assert(feed_map); - auto reader = PCapReader{args[1]}; - - mdp3::AB_Arbitrageur ab; - CSVPrinter csv_printer; - - reader.on_packet() = [&](auto const &info, auto const &payload) { - if ((info.dst_ip == feed_map.incremental[0].ip && - info.dst_port == feed_map.incremental[0].port)) { - buffers[0].i = mdp3::MDP3Packet{payload}; - } else if ((info.dst_ip == feed_map.incremental[0].ip && - info.dst_port == feed_map.incremental[0].port)) { - buffers[1].i = mdp3::MDP3Packet{payload}; - } else - return true; - auto action = ab(buffers[0].i.view(), buffers[1].i.view()); - if (action.action == mdp3::AB_Arbitrageur::APPLY) - parse_packet(action.packet, csv_printer); - else if (action.action == mdp3::AB_Arbitrageur::RECOVER) { - std::cerr << "Unrecoverable loss detecte at sequence number: " - << ab.expected_next_sn() - 1 << std::endl; - return false; - } - return true; - }; - while (reader.read_next()) - ; - return 0; -} - -void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) { - struct ObsWrapper : mdp3::IChannelObserver { - ObsWrapper(mdp3::IChannelObserver &obs) : real_obs(obs) {} - IChannelObserver &real_obs; - size_t previous_packet = 0; - size_t previous_send_ts = 0; - TimeStamp last_print = Timer::now(); - - void on_recovery_complete( - std::vector const &s) override { - real_obs.on_recovery_complete(s); - } - void on_real_time_event(mdp3::MDP3Event const &ev) override { - if (ms_diff(Timer::now(), last_print) >= 1000) { - last_print = Timer::now(); - std::stringstream str; - mdp3::cme_ts_to_time_str(ev.transact_time.value(), &str); - LOG(Spartan, Info, "Packet time: {} ({} pkt/sec {}x real time)", - str.str(), (ev.first_packet_sn - previous_packet), - (ev.first_packet_send_time.value() - previous_send_ts) / - 1000000000); - previous_packet = ev.first_packet_sn; - previous_send_ts = ev.first_packet_send_time.value(); - } - real_obs.on_real_time_event(ev); - } - void on_real_time_channel_loss() override { - real_obs.on_real_time_channel_loss(); - } - } obs_wrapper(obs); - - struct Ctrl : mdp3::IFeedController { - void start_instrument_feed() override{}; - void stop_instrument_feed() override{}; - void start_snapshot_feed() override{}; - void stop_snapshot_feed() override{}; - void start_incremental_feed() override{}; - void stop_incremental_feed() override{}; - } ctrl; - std::array buffers; - - auto feed_map = mdp3::guess_feed_map(file); - bts_assert(feed_map); - auto reader = PCapReader{file}; - mdp3::ChannelHandler handler(obs_wrapper, ctrl); - - reader.on_packet() = [&](auto const &info, auto const &payload) { - if ((info.dst_ip == feed_map.incremental[0].ip && - info.dst_port == feed_map.incremental[0].port)) { - buffers[0].i.set_bytes(payload); - handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view()); - } else if ((info.dst_ip == feed_map.incremental[1].ip && - info.dst_port == feed_map.incremental[1].port)) { - buffers[1].i.set_bytes(payload); - handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view()); - } else if ((info.dst_ip == feed_map.instrument.ip && - info.dst_port == feed_map.instrument.port)) { - handler.on_instrument_feed(mdp3::MDP3PacketView{payload}); - } else if ((info.dst_ip == feed_map.snapshot.ip && - info.dst_port == feed_map.snapshot.port)) { - handler.on_snapshot_feed(mdp3::MDP3PacketView{payload}); - } - return true; - }; - while (reader.read_next()) - ; -} - -} // namespace spartan -} // namespace bts diff --git a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.h b/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.h deleted file mode 100644 index ef34cafeb..000000000 --- a/src/server/sparkd/cmds/spartan/analysis/RunAnalysis.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include -#include - -namespace bts::spartan::mdp3 { -class IChannelObserver; -} -namespace bts::spartan { -int convert_pcap_to_csv_summary(std::vector const &args); -int convert_pcap_to_json(std::vector const &args ); - -void observe_channel(std::string const &, mdp3::IChannelObserver &) ; -} diff --git a/src/server/sparkd/cmds/spartan/docs/recorder.org b/src/server/sparkd/cmds/spartan/docs/recorder.org deleted file mode 100644 index cb53066c2..000000000 --- a/src/server/sparkd/cmds/spartan/docs/recorder.org +++ /dev/null @@ -1,29 +0,0 @@ -* Arbitration -on_msg (Msg* a, Msg *b) { - if (a && a->seq_num == seq_num) - seq_n++; - deliver(*a); - return; - } - if (b && b->seq_num == seq_num) - seq_n++; - deliver(*b); - return; - } - on_loss(seq_num); -} -* On startup -** In recovery -- join instrument feed -- join a, b -- join sambo - -- queued events is not 0 and its an unbroken sequence -- joined the snapshot feed, and it either has a seq n < the the top of the queue (so the snapshot is useless) or we are missing some messages in the snapshot -- we are either joined with the instrument feed but still not gone through a full cycle or we have a full set of securities - - - begin recovery - - b - - - diff --git a/src/server/sparkd/cmds/spartan/instruments/InstrumentStore.h b/src/server/sparkd/cmds/spartan/instruments/InstrumentStore.h deleted file mode 100644 index 164ffe83c..000000000 --- a/src/server/sparkd/cmds/spartan/instruments/InstrumentStore.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include - -class mdp3::MDInstrumentDefinitionFuture27; -class mdp3::MDInstrumentDefinitionFuture54; -class mdp3::MDInstrumentDefinitionOption41; -class mdp3::MDInstrumentDefinitionOption55; -class mdp3::MDInstrumentDefinitionSpread29; -class mdp3::MDInstrumentDefinitionSpread56; - -namespace bts { -namespace spartan { - -class InstrumentStore { - InstrumentStore(std::string const&base_path); - void store( const mdp3::MDInstrumentDefinitionFuture27&); - void store( const mdp3::MDInstrumentDefinitionFuture54&); - void store( const mdp3::MDInstrumentDefinitionOption41&); - void store( const mdp3::MDInstrumentDefinitionOption55&); - void store( const mdp3::MDInstrumentDefinitionSpread29&); - void store( const mdp3::MDInstrumentDefinitionSpread56&); - -private: - struct PImpl; - std::unique_ptr me; -}; - -}} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h index ca136b899..7aa0583f3 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h @@ -1,7 +1,7 @@ #pragma once -#include "mdp3_common.h" #include "InstrumentDefinition.h" +#include "mdp3_common.h" #include "lib/domain_model/Security.pb.h" #include @@ -17,103 +17,103 @@ namespace bts::spartan::mdp3 { // - The instrument's definition // - Price books (implied and customer) // - Resting orders -// - Fills observed so far +// - Fills generated by the last event +// - Order changes generated in the last event +// - The deduced order for the last event // - Market statistics + class InstrumentSnapshot { public: - InstrumentSnapshot(InstrumentDefinition const&); + InstrumentSnapshot(InstrumentDefinition const &); InstrumentSnapshot(InstrumentDefinition &&); ~InstrumentSnapshot(); - // Definition + // Queries ------------------------------------------------------------ + // Security MDP3SecurityID security_id() const; InstrumentDefinition definition() const; - void set_definition(InstrumentDefinition const&); - - // Symbol (via definition) - const char* symbol() const {return definition().symbol();} - - // Last packet that affected this security - size_t last_processed_sn() const ; - void set_last_msg_processed_sn(size_t); - - // Trading status + const char *symbol() const { return definition().symbol(); } ::mdp3::SecurityTradingStatus::Value trading_status() const; - void set_trading_status(::mdp3::SecurityTradingStatus::Value) const; - // Book + // Last packet incorporated in this snapshot + size_t last_processed_sn() const; + + // Books Book const &book(api::Side) const; - void insert_book_level(api::Side, size_t idx,BookLevel const&); - void update_book_level(api::Side, size_t idx,BookLevel const&); - void remove_book_level(api::Side, size_t idx); - - // Implied Book ImpliedBook const &implied_book(api::Side) const; - void insert_implied_book_level(api::Side, size_t idx, BookLevel const&); - void update_implied_book_level(api::Side, size_t idx, BookLevel const&); - void remove_implied_book_level(api::Side, size_t idx); // Orders Orders const &orders() const; + // Last round of order changes OrderChanges const &order_changes() const; - void clear_order_changes() ; - void insert_order(MDP3Order const&); - void update_order(MDP3Order const&); - void remove_order(MDP3OrderKey); - Book build_book_from_orders (api::Side ) const; - void set_books_from_orders() ; - // Deduced order changes, that is the implied order change that + // Deduced order change, that is the order change that // would explain the last event processed in this snapshot as // described by order_changes() and fills() MaybeOrderChange deduced_order_change() const; - - // Reset books in response to a channel reset message - void reset_books(); - - // Fills - Fills const&fills() const; - void add_fill(MDP3Fill const&); - void update_fill(MDP3Fill const&); - void remove_fill(MDP3TradeEntryID, MDP3OrderKey ); - void clear_fills(); - - // Stats + + // Last round of fills + Fills const &fills() const; + + // Stats md::IntPrice high_limit_price() const; - void set_high_limit_price(md::IntPrice); - md::IntPrice low_limit_price() const; - void set_low_limit_price(md::IntPrice); md::IntPrice max_price_variation() const; - void set_max_price_variation(md::IntPrice); - MDP3Quantity cleared_volume() const; - void set_cleared_volume(MDP3Quantity); MDP3Quantity electronic_volume() const; - void set_electronic_volume(MDP3Quantity); + MDP3Quantity cleared_volume() const; md::IntPrice fixing_price() const; - void set_fixing_price(md::IntPrice); MDP3Quantity open_interest() const; - void set_open_interest(MDP3Quantity); md::IntPrice open_price() const; - void set_open_price(md::IntPrice ) ; md::IntPrice high_bid() const; - void set_high_bid(md::IntPrice ) ; md::IntPrice low_offer() const; - void set_low_offer(md::IntPrice ) ; md::IntPrice settlement_price() const; - void set_settlement_price(md::IntPrice ) ; md::IntPrice high_trade_price() const; - void set_high_trade_price(md::IntPrice ) ; md::IntPrice low_trade_price() const; - void set_low_trade_price(md::IntPrice ) ; + + // Mutators ------------------------------------------------------------ + void set_definition(InstrumentDefinition const &); + void set_last_msg_processed_sn(size_t); + void set_trading_status(::mdp3::SecurityTradingStatus::Value) const; + void insert_book_level(api::Side, size_t idx, BookLevel const &); + void update_book_level(api::Side, size_t idx, BookLevel const &); + void remove_book_level(api::Side, size_t idx); + void insert_implied_book_level(api::Side, size_t idx, BookLevel const &); + void update_implied_book_level(api::Side, size_t idx, BookLevel const &); + void remove_implied_book_level(api::Side, size_t idx); + void clear_order_changes(); + void insert_order(MDP3Order const &); + void update_order(MDP3Order const &); + void remove_order(MDP3OrderKey); + Book build_book_from_orders(api::Side) const; + void set_books_from_orders(); + void reset_books(); + void add_fill(MDP3Fill const &); + void update_fill(MDP3Fill const &); + void remove_fill(MDP3TradeEntryID, MDP3OrderKey); + void clear_fills(); + void set_high_limit_price(md::IntPrice); + void set_low_limit_price(md::IntPrice); + void set_max_price_variation(md::IntPrice); + void set_cleared_volume(MDP3Quantity); + void set_electronic_volume(MDP3Quantity); + void set_fixing_price(md::IntPrice); + void set_open_interest(MDP3Quantity); + void set_open_price(md::IntPrice); + void set_high_bid(md::IntPrice); + void set_low_offer(md::IntPrice); + void set_settlement_price(md::IntPrice); + void set_high_trade_price(md::IntPrice); + void set_low_trade_price(md::IntPrice); private: - template void insert(api::Side,std::array &, size_t idx,BookLevel const&); - template void update(api::Side,std::array &, size_t idx, BookLevel const&); - template void remove(api::Side,std::array &, size_t idx); + template + void insert(api::Side, std::array &, size_t idx, BookLevel const &); + template + void update(api::Side, std::array &, size_t idx, BookLevel const &); + template void remove(api::Side, std::array &, size_t idx); struct PImpl; std::unique_ptr me; -}; +}; -std::ostream& operator<<(std::ostream&, InstrumentSnapshot const&); +std::ostream &operator<<(std::ostream &, InstrumentSnapshot const &); } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/sql_tools/md.sql b/src/server/sparkd/cmds/spartan/md/mdp3/sql/md.sql similarity index 55% rename from src/server/sparkd/cmds/spartan/sql_tools/md.sql rename to src/server/sparkd/cmds/spartan/md/mdp3/sql/md.sql index ae517c93c..c555a4537 100644 --- a/src/server/sparkd/cmds/spartan/sql_tools/md.sql +++ b/src/server/sparkd/cmds/spartan/md/mdp3/sql/md.sql @@ -1,62 +1,35 @@ --- drop table msg; --- create table if not exists --- msg ( --- "HumanTransactTime" TEXT, --- "HumanSendingTime" TEXT, --- "Latency" numeric, --- "TransactTime" numeric, --- "SendingTime" numeric, --- "SeqNum" numeric, --- "MsgType" TEXT, --- "TimeInQueue" numeric, --- "MatchEventIndicator" numeric --- ); - --- drop view event_len; --- create view event_len as --- select transacttime, count(*) len from msg group by transacttime; - --- drop view summary_event_len; --- create view summary_event_len as --- select --- count(*) n_events, --- msg_count len --- from --- (select transacttime, count(*) msg_count from msg group by transacttime) t --- group by --- len drop table events; CREATE TABLE events( eid numeric, - "t_time" numeric, - "first_sn" numeric, - "last_sn" numeric, - "first_st" numeric, - "last_st" numeric + t_time numeric, + first_sn numeric, + last_sn numeric, + first_st numeric, + last_st numeric ); drop table event_items; CREATE TABLE event_items( eid numeric, - "t_time" numeric, - "sid" numeric, - "status" TEXT, - "type" TEXT, - "change" TEXT, - "ok" numeric, - "side" TEXT, - "pri" numeric, - "price" numeric, - "qty" numeric, - "tid" numeric, - "agg" TEXT + t_time numeric, + sid numeric, + status TEXT, + type TEXT, + change TEXT, + ok numeric, + side TEXT, + pri numeric, + price numeric, + qty numeric, + tid numeric, + agg TEXT ); drop table securities; CREATE TABLE securities( - "sid" numeric, - "symbol" TEXT, - "group" TEXT, - "asset" TEXT, - "last_t" numeric + sid numeric, + symbol TEXT, + group TEXT, + asset TEXT, + last_t numeric ); .mode csv .import event_items.csv event_items diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp index aabb7e0be..fe82df22c 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp @@ -30,7 +30,6 @@ struct PacketBuffers { }; TEST_CASE("spartan_md", "[spartan_md]") { - //auto file = "/home/ppinto/es_md/es_all.pcap"; auto file = os::sparkd_wd() + "/data_files/small_es.pcap"; auto feed_map = guess_feed_map(file); auto reader = PCapReader{file}; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp index cd0e14eaa..58bd995ff 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.cpp @@ -22,9 +22,41 @@ #include #include #include +#include namespace bts::spartan::mdp3 { +void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out) { + auto one_billion = 1000000000; + time_t unix_ts = cme_ts / one_billion; + auto boost_ts = boost::posix_time::from_time_t(unix_ts); + int hours = boost_ts.time_of_day().hours(); + int minutes = boost_ts.time_of_day().minutes(); + int seconds = boost_ts.time_of_day().seconds(); + *out << std::setw(2) << std::setfill('0') << hours << ":" << std::setw(2) + << std::setfill('0') << minutes << ":" << std::setw(2) + << std::setfill('0') << seconds << "." << std::setw(9) + << cme_ts - (unix_ts * one_billion); +} + +void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out) { + auto one_billion = 1000000000; + time_t unix_ts = cme_ts / one_billion; + auto boost_ts = boost::posix_time::from_time_t(unix_ts); + auto year = boost_ts.date().year(); + auto month = boost_ts.date().month().as_number(); + auto day = boost_ts.date().day().as_number(); + int hours = boost_ts.time_of_day().hours(); + int minutes = boost_ts.time_of_day().minutes(); + int seconds = boost_ts.time_of_day().seconds(); + *out << year << "-" << std::setw(2) << std::setfill('0') << month << "-" + << std::setw(2) << std::setw(2) << std::setfill('0') << day << "." + << std::setw(2) << std::setw(2) << std::setfill('0') << hours << ":" + << std::setw(2) << std::setfill('0') << minutes << ":" << std::setw(2) + << std::setfill('0') << seconds << "." << std::setw(9) + << cme_ts - (unix_ts * one_billion); +} + struct Guesser { FeedMap result; PCapReader::PacketInfo current_packet; @@ -121,7 +153,7 @@ std::ostream &operator<<(std::ostream &o, FeedMap const &fm) { return o; } -bool slim_down_pcap(std::string const &file, std::string const &out_file) { +int compress(std::string const &file, std::string const &out_file) { using namespace bts::spartan; using namespace bts::spartan::mdp3; @@ -141,7 +173,7 @@ bool slim_down_pcap(std::string const &file, std::string const &out_file) { struct Obs : IChannelObserver { bool observed_loss = false; bool recovery_complete = false; - + size_t previous_packet = 0; size_t previous_send_ts = 0; TimeStamp last_print; @@ -156,11 +188,10 @@ bool slim_down_pcap(std::string const &file, std::string const &out_file) { last_print = Timer::now(); std::stringstream str; cme_ts_to_time_str(ev.transact_time.value(), &str); - LOG(Spartan, Info, - "Time: {} -> {} pkt/sec at {}x real time", - str.str(), - (ev.first_packet_sn - previous_packet), - (ev.first_packet_send_time.value() - previous_send_ts) / 1000000000); + LOG(Spartan, Info, "Time: {} -> {} pkt/sec at {}x real time", + str.str(), (ev.first_packet_sn - previous_packet), + (ev.first_packet_send_time.value() - previous_send_ts) / + 1000000000); previous_packet = ev.first_packet_sn; previous_send_ts = ev.first_packet_send_time.value(); } @@ -232,41 +263,262 @@ bool slim_down_pcap(std::string const &file, std::string const &out_file) { while (reader.read_next()) if (obs.observed_loss) { - LOG(Spartan, Error, "Observed loss in {}. Cannot slim it down", file); - return false; + LOG(Spartan, Error, "Observed loss in {}. Cannot compress.", file); + return 1; }; - return true; + return 0; } -void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out) { - auto one_billion = 1000000000; - time_t unix_ts = cme_ts / one_billion; - auto boost_ts = boost::posix_time::from_time_t(unix_ts); - int hours = boost_ts.time_of_day().hours(); - int minutes = boost_ts.time_of_day().minutes(); - int seconds = boost_ts.time_of_day().seconds(); - *out << std::setw(2) << std::setfill('0') << hours << ":" << std::setw(2) - << std::setfill('0') << minutes << ":" << std::setw(2) - << std::setfill('0') << seconds << "." << std::setw(9) - << cme_ts - (unix_ts * one_billion); +struct JSON_Writer { + std::stringstream json_buffer; + nlohmann::json json_obj; + mdp3::MDP3PacketView packet; + + void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; } + void on_end_packet() {} + + template void operator()(M const &m) { + json_obj["SeqNum"] = packet.seq_num(); + json_obj["SendTime"] = packet.sending_time(); + json_obj["Size"] = packet.size(); + + json_buffer.str(""); + json_buffer.clear(); + json_buffer << m; + json_obj["Msg"] = nlohmann::json::parse(json_buffer.str()); + std::cout << json_obj.dump(4) << std::endl; + } +}; + +struct PacketBuffers { + mdp3::MDP3Packet i; + mdp3::MDP3Packet n; +}; + +int to_json(std::string const&pcap) { + auto reader = PCapReader{pcap}; + JSON_Writer writer; + reader.on_packet() = [&](auto const &, auto const &payload) { + parse_packet(mdp3::MDP3PacketView{payload}, writer); + return true; + }; + while (reader.read_next()) + ; + return 0; } -void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out) { - auto one_billion = 1000000000; - time_t unix_ts = cme_ts / one_billion; - auto boost_ts = boost::posix_time::from_time_t(unix_ts); - auto year = boost_ts.date().year(); - auto month = boost_ts.date().month().as_number(); - auto day = boost_ts.date().day().as_number(); - int hours = boost_ts.time_of_day().hours(); - int minutes = boost_ts.time_of_day().minutes(); - int seconds = boost_ts.time_of_day().seconds(); - *out << year << "-" << std::setw(2) << std::setfill('0') << month << "-" - << std::setw(2) << std::setw(2) << std::setfill('0') << day << "." - << std::setw(2) << std::setw(2) << std::setfill('0') << hours << ":" - << std::setw(2) << std::setfill('0') << minutes << ":" << std::setw(2) - << std::setfill('0') << seconds << "." << std::setw(9) - << cme_ts - (unix_ts * one_billion); +void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) { + struct ObsWrapper : mdp3::IChannelObserver { + ObsWrapper(mdp3::IChannelObserver &obs) : real_obs(obs) {} + IChannelObserver &real_obs; + size_t previous_packet = 0; + size_t previous_send_ts = 0; + TimeStamp last_print = Timer::now(); + + void on_recovery_complete( + std::vector const &s) override { + real_obs.on_recovery_complete(s); + } + void on_real_time_event(mdp3::MDP3Event const &ev) override { + if (ms_diff(Timer::now(), last_print) >= 1000) { + last_print = Timer::now(); + std::stringstream str; + mdp3::cme_ts_to_time_str(ev.transact_time.value(), &str); + LOG(Spartan, Info, "Packet time: {} ({} pkt/sec {}x real time)", + str.str(), (ev.first_packet_sn - previous_packet), + (ev.first_packet_send_time.value() - previous_send_ts) / + 1000000000); + previous_packet = ev.first_packet_sn; + previous_send_ts = ev.first_packet_send_time.value(); + } + real_obs.on_real_time_event(ev); + } + void on_real_time_channel_loss() override { + real_obs.on_real_time_channel_loss(); + } + } obs_wrapper(obs); + + struct Ctrl : mdp3::IFeedController { + void start_instrument_feed() override{}; + void stop_instrument_feed() override{}; + void start_snapshot_feed() override{}; + void stop_snapshot_feed() override{}; + void start_incremental_feed() override{}; + void stop_incremental_feed() override{}; + } ctrl; + std::array buffers; + + auto feed_map = mdp3::guess_feed_map(file); + bts_assert(feed_map); + auto reader = PCapReader{file}; + mdp3::ChannelHandler handler(obs_wrapper, ctrl); + + reader.on_packet() = [&](auto const &info, auto const &payload) { + if ((info.dst_ip == feed_map.incremental[0].ip && + info.dst_port == feed_map.incremental[0].port)) { + buffers[0].i.set_bytes(payload); + handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view()); + } else if ((info.dst_ip == feed_map.incremental[1].ip && + info.dst_port == feed_map.incremental[1].port)) { + buffers[1].i.set_bytes(payload); + handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view()); + } else if ((info.dst_ip == feed_map.instrument.ip && + info.dst_port == feed_map.instrument.port)) { + handler.on_instrument_feed(mdp3::MDP3PacketView{payload}); + } else if ((info.dst_ip == feed_map.snapshot.ip && + info.dst_port == feed_map.snapshot.port)) { + handler.on_snapshot_feed(mdp3::MDP3PacketView{payload}); + } + return true; + }; + while (reader.read_next()) + ; +} +int to_csv(std::string const &file, std::string const &out_dir) { + using namespace bts::spartan::mdp3; + + struct Obs : IChannelObserver { + + Obs(std::string const &out_dir) + : events(out_dir + "/events.csv"), + event_items(out_dir + "/event_items.csv"), + securities(out_dir + "/securities.csv") { + + events << "eid, t_time,first_sn,last_sn," + "first_st,last_st" + << std::endl; + + event_items << "eid, t_time,sid,status,type," + "change,ok,side," + "pri,price,qty,tid,agg" + << std::endl; + + securities << "sid,symbol,group,asset,last_t" << std::endl; + } + + std::ofstream events; + std::ofstream event_items; + std::ofstream orders; + std::ofstream securities; + size_t event_id = 0; + std::unordered_set known_securities; + + void on_recovery_complete( + std::vector const &secs) override { + for (auto s : secs) { + securities << s->security_id().value() // security id + << "," << s->symbol() << "," // symbol + << s->definition().security_group() << "," // group + << s->definition().asset() << "," // asset + << s->definition().last_update_time() // last update time + << std::endl; + + known_securities.insert(s->security_id()); + + for (auto const &[id, order] : s->orders()) + event_items << 0 << "," // event_id + << 0 << "," // transact time + << s->security_id().value() << "," // security id + << s->trading_status() << "," // trading status + << "order_change," // type + << "init," // change + << id << "," // order key + << api::Side_Name(order.side) << "," // side + << order.priority.value() << "," // priority + << order.price.price() << "," // price + << order.quantity.value() << "," // quantity + << "," // trade_entry_id + << "n/a" // aggressor + << std::endl; + } + } + + void on_real_time_event(MDP3Event const &ev) override { + event_id++; + events << event_id << "," // event id + << ev.transact_time << "," // transact time + << ev.first_packet_sn << "," // first packet_sn + << ev.last_packet_sn << "," // last_packet_sn + << ev.first_packet_send_time << "," // first packet send time + << ev.last_packet_send_time // last packet send time + << std::endl; + + for (auto s : ev.affected_securities) { + + if (known_securities.count(s->security_id()) == 0) { + known_securities.insert(s->security_id()); + securities << s->security_id().value() // security id + << "," << s->symbol() << "," // symbol + << s->definition().security_group() << "," // group + << s->definition().asset() << "," // asset + << s->definition().last_update_time() // last update time + << std::endl; + } + + for (auto const &change : s->order_changes()) + event_items << event_id << "," // event id + << ev.transact_time.value() // transact time + << "," // transact time + << s->security_id().value() << "," // security id + << s->trading_status() << "," // trading status + << "order_change," // type + << change_name(change.change) << "," // change type + << change.order.key.value() << "," // order key + << api::Side_Name(change.order.side) << "," // side + << change.order.priority.value() << "," // priority + << change.order.price.price() << "," // price + << change.order.quantity.value() << "," // quantity + << "," // trade_entry_id + << "n/a" // aggressor + << std::endl; + + for (auto const &f : s->fills()) + event_items << event_id << "," // event id + << ev.transact_time.value() << "," // transact time + << s->security_id().value() << "," // security id + << s->trading_status() << "," // trading status + << "fill," // type + << "," // null + << f.key.value() << "," // order key + << "," // side + << "," // priority + << f.price << "," // price + << f.quantity << "," // quantity + << f.trade_entry_id << "," // trade_entry_id + << aggressor_name(f.aggressor) // aggressor + << std::endl; + + auto deduced_order = s->deduced_order_change(); + if (deduced_order) { + event_items << event_id << "," // event id + << ev.transact_time.value() << "," // transact time + << s->security_id().value() << "," // security id + << s->trading_status() << "," // trading status + << "deduced_order," // type + << change_name(deduced_order->change) << "," + << deduced_order->order.key.value() + << "," // order key + << api::Side_Name(deduced_order->order.side) + << "," // side + << "," // priority + << deduced_order->order.price.price() << "," // price + << deduced_order->order.quantity.value() + << "," // quantity + << "," // trade_entry_id + << "n/a" // aggressor + << std::endl; + } + } + } + + void on_real_time_channel_loss() override { + LOG(Spartan,Error,"Detected loss in feed. Aborting"); + bts_assert(false); + } + } obs(out_dir); + + observe_channel(file, obs); + return 0; } } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h index 3888a81ca..5fb24f39c 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h @@ -3,9 +3,14 @@ #include #include #include +#include namespace bts::spartan::mdp3 { +class IChannelObserver; + +// Details the ip/port used for each of the incremental a/b, snapshot +// and instrument feeds in a pcap. struct FeedMap { struct Feed { std::string ip = ""; @@ -16,19 +21,39 @@ struct FeedMap { Feed snapshot; Feed instrument; operator bool() const noexcept { - return - incremental[0].port && - incremental[1].port && - instrument.port && - snapshot.port; + return incremental[0].port && incremental[1].port && instrument.port && + snapshot.port; } }; std::ostream &operator<<(std::ostream &o, FeedMap const &fm); -FeedMap guess_feed_map(std::string const &pcap_path); -bool slim_down_pcap(std::string const&in, std::string const &out); -void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out); -void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out); +// Examine the passed pcap and determine, if possible, the ip/port +// associated with each of the following feeds: +// - Incremental A +// - Incremental B +// - Snapshot +// - Instrument +FeedMap guess_feed_map(std::string const &pcap) ; +// Examine the passed pcap and throw away redundant packets. Including: +// - Instrument and Snapshoft feed packets after the initial recovery +// - Feed A/Feed B redundant packets +int compress(std::string const&pcap_in, std::string const &pcap_out); + +// Convert the pcap passed to json and output it to standard out (this +// is typically only usefull for debugging) +int to_json(std::string const&pcap); + +// Export the passed pcap to a set of tables in CSV format. The +// following tables are generated: +// - events.csv : one row per event in the pcap +// - event_items.csv : one row per item in an event. Event items +// include: order changes, fills and deduced order +// - securities.csv : one row per security present in the pcap +int to_csv(std::string const &pcap, std::string const &out_dir); + +// Observe the pcap specified, invoking the observer for each event on +// it +void observe_channel(std::string const &pcap, IChannelObserver &obs) ; } // namespace bts::spartan::mdp3