Cleaned up code.
This commit is contained in:
1 parent
6aec729b6a
commit
b871cc4e09
14 files changed
+419
-728
No files matched your search
@@ -1 +1 @@
|
||||
set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/analysis/ActionableEventAnalyzer.cpp;cmds/spartan/analysis/RunAnalysis.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
|
||||
set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
|
||||
@@ -10,7 +10,6 @@
|
||||
#include "lib/domain_model/sod.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/utils/datetime.h"
|
||||
#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
|
||||
@@ -29,40 +28,6 @@
|
||||
|
||||
namespace bts {
|
||||
namespace cmd {
|
||||
using namespace bts::orderman;
|
||||
using namespace spartan::mdp3;
|
||||
using namespace spartan;
|
||||
|
||||
struct TimeWriter {
|
||||
mdp3::MDP3PacketView packet;
|
||||
void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; }
|
||||
void on_end_packet() { std::cout << packet.sending_time() << std::endl; }
|
||||
|
||||
template <class M> void operator()(M const &) {}
|
||||
};
|
||||
void print_times(std::string const &file) {
|
||||
struct PacketBuffers {
|
||||
spartan::mdp3::MDP3Packet i;
|
||||
spartan::mdp3::MDP3Packet n;
|
||||
};
|
||||
auto feed_map = spartan::mdp3::guess_feed_map(file);
|
||||
bts_assert(feed_map);
|
||||
auto reader = PCapReader{file};
|
||||
|
||||
mdp3::AB_Arbitrageur ab;
|
||||
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
if ((info.dst_ip == feed_map.incremental[0].ip &&
|
||||
info.dst_port == feed_map.incremental[0].port)) {
|
||||
TimeWriter tw;
|
||||
parse_packet(MDP3PacketView{payload}, tw);
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
while (reader.read_next())
|
||||
;
|
||||
}
|
||||
|
||||
int QuickTest::execute(std::vector<std::string> const &args) {
|
||||
|
||||
@@ -128,7 +93,7 @@ int QuickTest::execute(std::vector<std::string> const &args) {
|
||||
}
|
||||
} obs;
|
||||
|
||||
spartan::observe_channel(file, obs);
|
||||
spartan::mdp3::observe_channel(file, obs);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
#include "Spartan.h"
|
||||
#include "server/sparkd/cmds/spartan/algo/RunTAS.h"
|
||||
#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h"
|
||||
@@ -18,153 +17,15 @@
|
||||
namespace bts {
|
||||
namespace cmd {
|
||||
|
||||
int slim_pcap(std::string const &file, std::string const &out_dir) {
|
||||
return !spartan::mdp3::slim_down_pcap(file, out_dir);
|
||||
}
|
||||
|
||||
int export_tables(std::string const &file, std::string const &out_dir) {
|
||||
using namespace bts::spartan::mdp3;
|
||||
|
||||
struct Obs : IChannelObserver {
|
||||
|
||||
Obs(std::string const &out_dir)
|
||||
: events(out_dir + "/events.csv"),
|
||||
event_items(out_dir + "/event_items.csv"),
|
||||
securities(out_dir + "/securities.csv") {
|
||||
|
||||
events << "eid, t_time,first_sn,last_sn,"
|
||||
"first_st,last_st"
|
||||
<< std::endl;
|
||||
|
||||
event_items << "eid, t_time,sid,status,type,"
|
||||
"change,ok,side,"
|
||||
"pri,price,qty,tid,agg"
|
||||
<< std::endl;
|
||||
|
||||
securities << "sid,symbol,group,asset,last_t" << std::endl;
|
||||
}
|
||||
|
||||
std::ofstream events;
|
||||
std::ofstream event_items;
|
||||
std::ofstream orders;
|
||||
std::ofstream securities;
|
||||
size_t event_id = 0;
|
||||
|
||||
void on_recovery_complete(
|
||||
std::vector<InstrumentSnapshot const *> const &secs) override {
|
||||
for (auto s : secs) {
|
||||
securities << s->security_id().value() // security id
|
||||
<< "," << s->symbol() << "," // symbol
|
||||
<< s->definition().security_group() << "," // group
|
||||
<< s->definition().asset() << "," // asset
|
||||
<< s->definition().last_update_time() // last update time
|
||||
<< std::endl;
|
||||
|
||||
for (auto const &[id, order] : s->orders())
|
||||
event_items << 0 << "," // event_id
|
||||
<< 0 << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "order_change," // type
|
||||
<< "init," // change
|
||||
<< id << "," // order key
|
||||
<< api::Side_Name(order.side) << "," // side
|
||||
<< order.priority.value() << "," // priority
|
||||
<< order.price.price() << "," // price
|
||||
<< order.quantity.value() << "," // quantity
|
||||
<< "," // trade_entry_id
|
||||
<< "n/a" // aggressor
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
void on_real_time_event(MDP3Event const &ev) override {
|
||||
event_id++;
|
||||
events << event_id << "," // event id
|
||||
<< ev.transact_time << "," // transactr time
|
||||
<< ev.first_packet_sn << "," // first packet_sn
|
||||
<< ev.last_packet_sn << "," // last_packet_sn
|
||||
<< ev.first_packet_send_time << "," // first packet send time
|
||||
<< ev.last_packet_send_time // last packet send time
|
||||
<< std::endl;
|
||||
|
||||
for (auto s : ev.affected_securities) {
|
||||
for (auto const &change : s->order_changes())
|
||||
event_items << event_id << "," // event id
|
||||
<< ev.transact_time.value() // transact time
|
||||
<< "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "order_change," // type
|
||||
<< change_name(change.change) << "," // change type
|
||||
<< change.order.key.value() << "," // order key
|
||||
<< api::Side_Name(change.order.side) << "," // side
|
||||
<< change.order.priority.value() << "," // priority
|
||||
<< change.order.price.price() << "," // price
|
||||
<< change.order.quantity.value() << "," // quantity
|
||||
<< "," // trade_entry_id
|
||||
<< "n/a" // aggressor
|
||||
<< std::endl;
|
||||
|
||||
for (auto const &f : s->fills())
|
||||
event_items << event_id << "," // event id
|
||||
<< ev.transact_time.value() << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "fill," // type
|
||||
<< "," // null
|
||||
<< f.key.value() << "," // order key
|
||||
<< "," // side
|
||||
<< "," // priority
|
||||
<< f.price << "," // price
|
||||
<< f.quantity << "," // quantity
|
||||
<< f.trade_entry_id << "," // trade_entry_id
|
||||
<< aggressor_name(f.aggressor) // aggressor
|
||||
<< std::endl;
|
||||
|
||||
auto deduced_order = s->deduced_order_change();
|
||||
if (deduced_order) {
|
||||
event_items << event_id << "," // event id
|
||||
<< ev.transact_time.value() << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "deduced_order," // type
|
||||
<< change_name(deduced_order->change) << ","
|
||||
<< deduced_order->order.key.value()
|
||||
<< "," // order key
|
||||
<< api::Side_Name(deduced_order->order.side)
|
||||
<< "," // side
|
||||
<< "," // priority
|
||||
<< deduced_order->order.price.price() << "," // price
|
||||
<< deduced_order->order.quantity.value()
|
||||
<< "," // quantity
|
||||
<< "," // trade_entry_id
|
||||
<< "n/a" // aggressor
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void on_real_time_channel_loss() override {
|
||||
std::cout << "Observed loss, resetting\n";
|
||||
}
|
||||
} obs(out_dir);
|
||||
|
||||
spartan::observe_channel(file, obs);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int Spartan::execute(std::vector<std::string> const &args) {
|
||||
auto error = 1;
|
||||
if (args.size() > 0) {
|
||||
if (args[0] == "pcap_to_msgs")
|
||||
error = spartan::convert_pcap_to_csv_summary(args);
|
||||
else if (args[0] == "pcap_to_json")
|
||||
error = spartan::convert_pcap_to_json(args);
|
||||
else if (args[0] == "pcap_to_tables")
|
||||
error = export_tables(args[1], args[2]);
|
||||
else if (args[0] == "slim_pcap")
|
||||
error = slim_pcap(args[1], args[2]);
|
||||
if (args[0] == "json" && args.size() == 2)
|
||||
error = spartan::mdp3::to_json(args[1]);
|
||||
else if (args[0] == "compress" && args.size() == 3)
|
||||
error = spartan::mdp3::compress(args[1],args[2]);
|
||||
else if (args[0] == "csv" && args.size() == 3)
|
||||
error = spartan::mdp3::to_csv(args[1], args[2]);
|
||||
}
|
||||
if (error)
|
||||
std::cout << help();
|
||||
@@ -172,10 +33,9 @@ int Spartan::execute(std::vector<std::string> const &args) {
|
||||
}
|
||||
|
||||
std::string Spartan::help() {
|
||||
return fmt::format("\tpcap_to_msgs pcap_file\n"
|
||||
"\tpcap_to_json pcap_file\n"
|
||||
"\tslim_pcap in_pcap out_pcap\n"
|
||||
"\tpcap_to_tables pcap_file output_dir\n");
|
||||
return fmt::format("\tjson <pcap>\n"
|
||||
"\tcompress <pcap> <out-pcap>\n"
|
||||
"\tcsv <pcap> <out-dir>");
|
||||
}
|
||||
|
||||
} // namespace cmd
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
#include "ActionableEventAnalyzer.h"
|
||||
@@ -1,59 +0,0 @@
|
||||
#pragma once
|
||||
#include "../md/mdp3/parsing/MDP3Parser.h"
|
||||
#include "lib/utils/stats.h"
|
||||
#include <stdint.h>
|
||||
#include <vector>
|
||||
|
||||
namespace bts {
|
||||
namespace spartan {
|
||||
|
||||
struct ActionableEventAnalyzer {
|
||||
|
||||
struct Event {
|
||||
uint64_t transact_time;
|
||||
uint64_t send_time;
|
||||
uint64_t actionable_window;
|
||||
uint64_t exchange_latency;
|
||||
uint64_t time_from_last_transaction;
|
||||
};
|
||||
std::vector<Event> events;
|
||||
template <class M> void operator()(mdp3::MDP3PacketHeader const&ph,mdp3::MDP3MsgHeader const &, const M &m) {
|
||||
Event e{m.transactTime(), ph.sending_time, 0,
|
||||
ph.sending_time - m.transactTime(), 0};
|
||||
if (events.size()) {
|
||||
auto &previous = events.back();
|
||||
previous.actionable_window =
|
||||
std::max(0l, (int64_t)(previous.send_time - m.transactTime()));
|
||||
previous.time_from_last_transaction = m.transactTime() - previous.transact_time;
|
||||
}
|
||||
events.push_back(e);
|
||||
}
|
||||
|
||||
std::string single_stat(uint64_t Event::*field) {
|
||||
std::ostringstream str;
|
||||
str.precision(0);
|
||||
str << std::fixed << "\tmin: "
|
||||
<< stats::min(events, [field](auto const &ev) { return ev.*field; })
|
||||
<< "\t\tmed: "
|
||||
<< stats::med(events, [field](auto const &ev) { return ev.*field; })
|
||||
<< "\t\tmax: "
|
||||
<< stats::max(events, [field](auto const &ev) { return ev.*field; });
|
||||
return str.str();
|
||||
}
|
||||
|
||||
~ActionableEventAnalyzer() {
|
||||
std::cout << "Data points:\n \t\t" << events.size()
|
||||
|
||||
<< "\nExchange Latency:\n"
|
||||
<< single_stat(&Event::exchange_latency)
|
||||
|
||||
<< "\nTransaction Gap:\n"
|
||||
<< single_stat(&Event::time_from_last_transaction)
|
||||
|
||||
<< "\nActionable Window:\n"
|
||||
<< single_stat(&Event::actionable_window) << "\n";
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace spartan
|
||||
} // namespace bts
|
||||
@@ -1,251 +0,0 @@
|
||||
#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h"
|
||||
#include "../md/mdp3/parsing/MDP3Parser.h"
|
||||
#include "../util/PCapReader.h"
|
||||
#include "lib/utils/bts_chrono.h"
|
||||
#include "lib/utils/overloaded.h"
|
||||
#include "lib/utils/stats.h"
|
||||
#include "lib/utils/type_name.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
|
||||
|
||||
#include <boost/date_time/posix_time/conversion.hpp>
|
||||
#include <boost/date_time/posix_time/ptime.hpp>
|
||||
#include <iostream>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace bts {
|
||||
namespace spartan {
|
||||
|
||||
struct PacketBuffers {
|
||||
mdp3::MDP3Packet i;
|
||||
mdp3::MDP3Packet n;
|
||||
};
|
||||
|
||||
mdp3::AB_Arbitrageur ab;
|
||||
|
||||
struct JSON_Writer {
|
||||
|
||||
std::stringstream json_buffer;
|
||||
nlohmann::json json_obj;
|
||||
mdp3::MDP3PacketView packet;
|
||||
|
||||
void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; }
|
||||
void on_end_packet() {}
|
||||
|
||||
template <class M> void operator()(M const &m) {
|
||||
json_obj["SeqNum"] = packet.seq_num();
|
||||
json_obj["SendTime"] = packet.sending_time();
|
||||
json_obj["Size"] = packet.size();
|
||||
|
||||
json_buffer.str("");
|
||||
json_buffer.clear();
|
||||
json_buffer << m;
|
||||
json_obj["Msg"] = nlohmann::json::parse(json_buffer.str());
|
||||
std::cout << json_obj.dump(4) << std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
int convert_pcap_to_json(std::vector<std::string> const &args) {
|
||||
std::array<PacketBuffers, 2> buffers;
|
||||
|
||||
auto reader = PCapReader{args[1]};
|
||||
JSON_Writer writer;
|
||||
|
||||
reader.on_packet() = [&](auto const &, auto const &payload) {
|
||||
parse_packet(mdp3::MDP3PacketView{payload}, writer);
|
||||
return true;
|
||||
};
|
||||
while (reader.read_next())
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <class M, decltype(((M *)nullptr)->transactTime()) = 0>
|
||||
uint64_t get_transact_time(M &m) {
|
||||
return m.transactTime();
|
||||
}
|
||||
|
||||
template <class M, decltype(((M *)nullptr)->lastUpdateTime()) = 0>
|
||||
uint64_t get_transact_time(M &m) {
|
||||
return m.lastUpdateTime();
|
||||
}
|
||||
|
||||
struct CSVPrinter {
|
||||
|
||||
std::stringstream stime_buffer;
|
||||
std::stringstream ttime_buffer;
|
||||
size_t previous_t_time = 0;
|
||||
size_t previous_s_time = 0;
|
||||
mdp3::MDP3PacketView packet;
|
||||
|
||||
CSVPrinter() {
|
||||
std::cout
|
||||
<< "HumanTransactTime,HumanSendingTime,"
|
||||
"Latency,TransactTime,SendingTime,SeqNum,MsgType,TimeInQueue,"
|
||||
"MatchEventindicator"
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
void on_begin_packet(mdp3::MDP3PacketView const &v) { packet = v; }
|
||||
void on_end_packet() {}
|
||||
|
||||
template <class M> void print(M &m) {
|
||||
stime_buffer.str("");
|
||||
stime_buffer.clear();
|
||||
mdp3::cme_ts_to_date_str(packet.sending_time(), &stime_buffer);
|
||||
ttime_buffer.str("");
|
||||
ttime_buffer.clear();
|
||||
mdp3::cme_ts_to_date_str(get_transact_time(m), &ttime_buffer);
|
||||
|
||||
auto t = get_transact_time(m);
|
||||
std::cout << ttime_buffer.str() << "," << stime_buffer.str() << ","
|
||||
<< (packet.sending_time() - t) / 1000 << "," << t << ","
|
||||
<< packet.sending_time() << "," << packet.seq_num() << ","
|
||||
<< utils::type_name<M>() << ","
|
||||
<< (t < previous_s_time ? ((previous_s_time - t) / 1000) : 0)
|
||||
<< "," << m.matchEventIndicator().endOfEvent() << std::endl;
|
||||
previous_s_time = packet.sending_time();
|
||||
previous_t_time = t;
|
||||
}
|
||||
|
||||
void operator()(::mdp3::MDIncrementalRefreshOrderBook43 &m) { print(m); }
|
||||
void operator()(::mdp3::MDIncrementalRefreshOrderBook47 &m) { print(m); }
|
||||
void operator()(::mdp3::MDIncrementalRefreshTradeSummary42 &m) { print(m); }
|
||||
void operator()(::mdp3::MDIncrementalRefreshTradeSummary48 &m) { print(m); }
|
||||
void operator()(::mdp3::MDIncrementalRefreshVolume37 &m) { print(m); }
|
||||
void operator()(::mdp3::MDIncrementalRefreshDailyStatistics33 &m) {
|
||||
print(m);
|
||||
}
|
||||
void operator()(::mdp3::MDIncrementalRefreshDailyStatistics49 &m) {
|
||||
print(m);
|
||||
}
|
||||
void operator()(::mdp3::MDIncrementalRefreshLimitsBanding34 &m) { print(m); }
|
||||
void operator()(::mdp3::MDIncrementalRefreshLimitsBanding50 &m) { print(m); }
|
||||
void operator()(::mdp3::MDIncrementalRefreshBook46 &m) { print(m); }
|
||||
void operator()(::mdp3::MDInstrumentDefinitionFuture54 &m) { print(m); }
|
||||
void operator()(::mdp3::MDIncrementalRefreshSessionStatistics35 &m) {
|
||||
print(m);
|
||||
}
|
||||
void operator()(::mdp3::MDIncrementalRefreshSessionStatistics51 &m) {
|
||||
print(m);
|
||||
}
|
||||
void operator()(::mdp3::MDInstrumentDefinitionFuture27 &m) { print(m); }
|
||||
void operator()(::mdp3::ChannelReset4 &m) { print(m); }
|
||||
void operator()(::mdp3::MDIncrementalRefreshBook32 &m) { print(m); }
|
||||
void operator()(::mdp3::MDInstrumentDefinitionOption41 &m) { print(m); }
|
||||
void operator()(::mdp3::MDInstrumentDefinitionOption55 &m) { print(m); }
|
||||
void operator()(::mdp3::MDInstrumentDefinitionSpread29 &m) { print(m); }
|
||||
void operator()(::mdp3::MDInstrumentDefinitionSpread56 &m) { print(m); }
|
||||
void operator()(::mdp3::QuoteRequest39 &m) { print(m); }
|
||||
void operator()(::mdp3::SecurityStatus30 &m) { print(m); }
|
||||
};
|
||||
|
||||
int convert_pcap_to_csv_summary(std::vector<std::string> const &args) {
|
||||
std::array<PacketBuffers, 2> buffers;
|
||||
|
||||
auto feed_map = mdp3::guess_feed_map(args[1]);
|
||||
bts_assert(feed_map);
|
||||
auto reader = PCapReader{args[1]};
|
||||
|
||||
mdp3::AB_Arbitrageur ab;
|
||||
CSVPrinter csv_printer;
|
||||
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
if ((info.dst_ip == feed_map.incremental[0].ip &&
|
||||
info.dst_port == feed_map.incremental[0].port)) {
|
||||
buffers[0].i = mdp3::MDP3Packet{payload};
|
||||
} else if ((info.dst_ip == feed_map.incremental[0].ip &&
|
||||
info.dst_port == feed_map.incremental[0].port)) {
|
||||
buffers[1].i = mdp3::MDP3Packet{payload};
|
||||
} else
|
||||
return true;
|
||||
auto action = ab(buffers[0].i.view(), buffers[1].i.view());
|
||||
if (action.action == mdp3::AB_Arbitrageur::APPLY)
|
||||
parse_packet(action.packet, csv_printer);
|
||||
else if (action.action == mdp3::AB_Arbitrageur::RECOVER) {
|
||||
std::cerr << "Unrecoverable loss detecte at sequence number: "
|
||||
<< ab.expected_next_sn() - 1 << std::endl;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
while (reader.read_next())
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
|
||||
struct ObsWrapper : mdp3::IChannelObserver {
|
||||
ObsWrapper(mdp3::IChannelObserver &obs) : real_obs(obs) {}
|
||||
IChannelObserver &real_obs;
|
||||
size_t previous_packet = 0;
|
||||
size_t previous_send_ts = 0;
|
||||
TimeStamp last_print = Timer::now();
|
||||
|
||||
void on_recovery_complete(
|
||||
std::vector<mdp3::InstrumentSnapshot const *> const &s) override {
|
||||
real_obs.on_recovery_complete(s);
|
||||
}
|
||||
void on_real_time_event(mdp3::MDP3Event const &ev) override {
|
||||
if (ms_diff(Timer::now(), last_print) >= 1000) {
|
||||
last_print = Timer::now();
|
||||
std::stringstream str;
|
||||
mdp3::cme_ts_to_time_str(ev.transact_time.value(), &str);
|
||||
LOG(Spartan, Info, "Packet time: {} ({} pkt/sec {}x real time)",
|
||||
str.str(), (ev.first_packet_sn - previous_packet),
|
||||
(ev.first_packet_send_time.value() - previous_send_ts) /
|
||||
1000000000);
|
||||
previous_packet = ev.first_packet_sn;
|
||||
previous_send_ts = ev.first_packet_send_time.value();
|
||||
}
|
||||
real_obs.on_real_time_event(ev);
|
||||
}
|
||||
void on_real_time_channel_loss() override {
|
||||
real_obs.on_real_time_channel_loss();
|
||||
}
|
||||
} obs_wrapper(obs);
|
||||
|
||||
struct Ctrl : mdp3::IFeedController {
|
||||
void start_instrument_feed() override{};
|
||||
void stop_instrument_feed() override{};
|
||||
void start_snapshot_feed() override{};
|
||||
void stop_snapshot_feed() override{};
|
||||
void start_incremental_feed() override{};
|
||||
void stop_incremental_feed() override{};
|
||||
} ctrl;
|
||||
std::array<PacketBuffers, 2> buffers;
|
||||
|
||||
auto feed_map = mdp3::guess_feed_map(file);
|
||||
bts_assert(feed_map);
|
||||
auto reader = PCapReader{file};
|
||||
mdp3::ChannelHandler handler(obs_wrapper, ctrl);
|
||||
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
if ((info.dst_ip == feed_map.incremental[0].ip &&
|
||||
info.dst_port == feed_map.incremental[0].port)) {
|
||||
buffers[0].i.set_bytes(payload);
|
||||
handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view());
|
||||
} else if ((info.dst_ip == feed_map.incremental[1].ip &&
|
||||
info.dst_port == feed_map.incremental[1].port)) {
|
||||
buffers[1].i.set_bytes(payload);
|
||||
handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view());
|
||||
} else if ((info.dst_ip == feed_map.instrument.ip &&
|
||||
info.dst_port == feed_map.instrument.port)) {
|
||||
handler.on_instrument_feed(mdp3::MDP3PacketView{payload});
|
||||
} else if ((info.dst_ip == feed_map.snapshot.ip &&
|
||||
info.dst_port == feed_map.snapshot.port)) {
|
||||
handler.on_snapshot_feed(mdp3::MDP3PacketView{payload});
|
||||
}
|
||||
return true;
|
||||
};
|
||||
while (reader.read_next())
|
||||
;
|
||||
}
|
||||
|
||||
} // namespace spartan
|
||||
} // namespace bts
|
||||
@@ -1,14 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
namespace bts::spartan::mdp3 {
|
||||
class IChannelObserver;
|
||||
}
|
||||
namespace bts::spartan {
|
||||
int convert_pcap_to_csv_summary(std::vector<std::string> const &args);
|
||||
int convert_pcap_to_json(std::vector<std::string> const &args );
|
||||
|
||||
void observe_channel(std::string const &, mdp3::IChannelObserver &) ;
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
* Arbitration
|
||||
on_msg (Msg* a, Msg *b) {
|
||||
if (a && a->seq_num == seq_num)
|
||||
seq_n++;
|
||||
deliver(*a);
|
||||
return;
|
||||
}
|
||||
if (b && b->seq_num == seq_num)
|
||||
seq_n++;
|
||||
deliver(*b);
|
||||
return;
|
||||
}
|
||||
on_loss(seq_num);
|
||||
}
|
||||
* On startup
|
||||
** In recovery
|
||||
- join instrument feed
|
||||
- join a, b
|
||||
- join sambo
|
||||
|
||||
- queued events is not 0 and its an unbroken sequence
|
||||
- joined the snapshot feed, and it either has a seq n < the the top of the queue (so the snapshot is useless) or we are missing some messages in the snapshot
|
||||
- we are either joined with the instrument feed but still not gone through a full cycle or we have a full set of securities
|
||||
|
||||
- begin recovery
|
||||
- b
|
||||
|
||||
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
class mdp3::MDInstrumentDefinitionFuture27;
|
||||
class mdp3::MDInstrumentDefinitionFuture54;
|
||||
class mdp3::MDInstrumentDefinitionOption41;
|
||||
class mdp3::MDInstrumentDefinitionOption55;
|
||||
class mdp3::MDInstrumentDefinitionSpread29;
|
||||
class mdp3::MDInstrumentDefinitionSpread56;
|
||||
|
||||
namespace bts {
|
||||
namespace spartan {
|
||||
|
||||
class InstrumentStore {
|
||||
InstrumentStore(std::string const&base_path);
|
||||
void store( const mdp3::MDInstrumentDefinitionFuture27&);
|
||||
void store( const mdp3::MDInstrumentDefinitionFuture54&);
|
||||
void store( const mdp3::MDInstrumentDefinitionOption41&);
|
||||
void store( const mdp3::MDInstrumentDefinitionOption55&);
|
||||
void store( const mdp3::MDInstrumentDefinitionSpread29&);
|
||||
void store( const mdp3::MDInstrumentDefinitionSpread56&);
|
||||
|
||||
private:
|
||||
struct PImpl;
|
||||
std::unique_ptr<PImpl> me;
|
||||
};
|
||||
|
||||
}}
|
||||
@@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "mdp3_common.h"
|
||||
#include "InstrumentDefinition.h"
|
||||
#include "mdp3_common.h"
|
||||
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include <memory>
|
||||
@@ -17,103 +17,103 @@ namespace bts::spartan::mdp3 {
|
||||
// - The instrument's definition
|
||||
// - Price books (implied and customer)
|
||||
// - Resting orders
|
||||
// - Fills observed so far
|
||||
// - Fills generated by the last event
|
||||
// - Order changes generated in the last event
|
||||
// - The deduced order for the last event
|
||||
// - Market statistics
|
||||
|
||||
class InstrumentSnapshot {
|
||||
public:
|
||||
InstrumentSnapshot(InstrumentDefinition const&);
|
||||
InstrumentSnapshot(InstrumentDefinition const &);
|
||||
InstrumentSnapshot(InstrumentDefinition &&);
|
||||
~InstrumentSnapshot();
|
||||
|
||||
// Definition
|
||||
// Queries ------------------------------------------------------------
|
||||
// Security
|
||||
MDP3SecurityID security_id() const;
|
||||
InstrumentDefinition definition() const;
|
||||
void set_definition(InstrumentDefinition const&);
|
||||
|
||||
// Symbol (via definition)
|
||||
const char* symbol() const {return definition().symbol();}
|
||||
|
||||
// Last packet that affected this security
|
||||
size_t last_processed_sn() const ;
|
||||
void set_last_msg_processed_sn(size_t);
|
||||
|
||||
// Trading status
|
||||
const char *symbol() const { return definition().symbol(); }
|
||||
::mdp3::SecurityTradingStatus::Value trading_status() const;
|
||||
void set_trading_status(::mdp3::SecurityTradingStatus::Value) const;
|
||||
|
||||
// Book
|
||||
// Last packet incorporated in this snapshot
|
||||
size_t last_processed_sn() const;
|
||||
|
||||
// Books
|
||||
Book const &book(api::Side) const;
|
||||
void insert_book_level(api::Side, size_t idx,BookLevel const&);
|
||||
void update_book_level(api::Side, size_t idx,BookLevel const&);
|
||||
void remove_book_level(api::Side, size_t idx);
|
||||
|
||||
// Implied Book
|
||||
ImpliedBook const &implied_book(api::Side) const;
|
||||
void insert_implied_book_level(api::Side, size_t idx, BookLevel const&);
|
||||
void update_implied_book_level(api::Side, size_t idx, BookLevel const&);
|
||||
void remove_implied_book_level(api::Side, size_t idx);
|
||||
|
||||
// Orders
|
||||
Orders const &orders() const;
|
||||
// Last round of order changes
|
||||
OrderChanges const &order_changes() const;
|
||||
void clear_order_changes() ;
|
||||
void insert_order(MDP3Order const&);
|
||||
void update_order(MDP3Order const&);
|
||||
void remove_order(MDP3OrderKey);
|
||||
Book build_book_from_orders (api::Side ) const;
|
||||
void set_books_from_orders() ;
|
||||
// Deduced order changes, that is the implied order change that
|
||||
// Deduced order change, that is the order change that
|
||||
// would explain the last event processed in this snapshot as
|
||||
// described by order_changes() and fills()
|
||||
MaybeOrderChange deduced_order_change() const;
|
||||
|
||||
// Reset books in response to a channel reset message
|
||||
void reset_books();
|
||||
|
||||
// Fills
|
||||
Fills const&fills() const;
|
||||
void add_fill(MDP3Fill const&);
|
||||
void update_fill(MDP3Fill const&);
|
||||
void remove_fill(MDP3TradeEntryID, MDP3OrderKey );
|
||||
void clear_fills();
|
||||
|
||||
// Stats
|
||||
|
||||
// Last round of fills
|
||||
Fills const &fills() const;
|
||||
|
||||
// Stats
|
||||
md::IntPrice high_limit_price() const;
|
||||
void set_high_limit_price(md::IntPrice);
|
||||
|
||||
md::IntPrice low_limit_price() const;
|
||||
void set_low_limit_price(md::IntPrice);
|
||||
md::IntPrice max_price_variation() const;
|
||||
void set_max_price_variation(md::IntPrice);
|
||||
MDP3Quantity cleared_volume() const;
|
||||
void set_cleared_volume(MDP3Quantity);
|
||||
MDP3Quantity electronic_volume() const;
|
||||
void set_electronic_volume(MDP3Quantity);
|
||||
MDP3Quantity cleared_volume() const;
|
||||
md::IntPrice fixing_price() const;
|
||||
void set_fixing_price(md::IntPrice);
|
||||
MDP3Quantity open_interest() const;
|
||||
void set_open_interest(MDP3Quantity);
|
||||
md::IntPrice open_price() const;
|
||||
void set_open_price(md::IntPrice ) ;
|
||||
md::IntPrice high_bid() const;
|
||||
void set_high_bid(md::IntPrice ) ;
|
||||
md::IntPrice low_offer() const;
|
||||
void set_low_offer(md::IntPrice ) ;
|
||||
md::IntPrice settlement_price() const;
|
||||
void set_settlement_price(md::IntPrice ) ;
|
||||
md::IntPrice high_trade_price() const;
|
||||
void set_high_trade_price(md::IntPrice ) ;
|
||||
md::IntPrice low_trade_price() const;
|
||||
void set_low_trade_price(md::IntPrice ) ;
|
||||
|
||||
// Mutators ------------------------------------------------------------
|
||||
void set_definition(InstrumentDefinition const &);
|
||||
void set_last_msg_processed_sn(size_t);
|
||||
void set_trading_status(::mdp3::SecurityTradingStatus::Value) const;
|
||||
void insert_book_level(api::Side, size_t idx, BookLevel const &);
|
||||
void update_book_level(api::Side, size_t idx, BookLevel const &);
|
||||
void remove_book_level(api::Side, size_t idx);
|
||||
void insert_implied_book_level(api::Side, size_t idx, BookLevel const &);
|
||||
void update_implied_book_level(api::Side, size_t idx, BookLevel const &);
|
||||
void remove_implied_book_level(api::Side, size_t idx);
|
||||
void clear_order_changes();
|
||||
void insert_order(MDP3Order const &);
|
||||
void update_order(MDP3Order const &);
|
||||
void remove_order(MDP3OrderKey);
|
||||
Book build_book_from_orders(api::Side) const;
|
||||
void set_books_from_orders();
|
||||
void reset_books();
|
||||
void add_fill(MDP3Fill const &);
|
||||
void update_fill(MDP3Fill const &);
|
||||
void remove_fill(MDP3TradeEntryID, MDP3OrderKey);
|
||||
void clear_fills();
|
||||
void set_high_limit_price(md::IntPrice);
|
||||
void set_low_limit_price(md::IntPrice);
|
||||
void set_max_price_variation(md::IntPrice);
|
||||
void set_cleared_volume(MDP3Quantity);
|
||||
void set_electronic_volume(MDP3Quantity);
|
||||
void set_fixing_price(md::IntPrice);
|
||||
void set_open_interest(MDP3Quantity);
|
||||
void set_open_price(md::IntPrice);
|
||||
void set_high_bid(md::IntPrice);
|
||||
void set_low_offer(md::IntPrice);
|
||||
void set_settlement_price(md::IntPrice);
|
||||
void set_high_trade_price(md::IntPrice);
|
||||
void set_low_trade_price(md::IntPrice);
|
||||
|
||||
private:
|
||||
template<class A> void insert(api::Side,std::array<A,2> &, size_t idx,BookLevel const&);
|
||||
template<class A> void update(api::Side,std::array<A,2> &, size_t idx, BookLevel const&);
|
||||
template<class A> void remove(api::Side,std::array<A,2> &, size_t idx);
|
||||
template <class A>
|
||||
void insert(api::Side, std::array<A, 2> &, size_t idx, BookLevel const &);
|
||||
template <class A>
|
||||
void update(api::Side, std::array<A, 2> &, size_t idx, BookLevel const &);
|
||||
template <class A> void remove(api::Side, std::array<A, 2> &, size_t idx);
|
||||
struct PImpl;
|
||||
std::unique_ptr<PImpl> me;
|
||||
};
|
||||
};
|
||||
|
||||
std::ostream& operator<<(std::ostream&, InstrumentSnapshot const&);
|
||||
std::ostream &operator<<(std::ostream &, InstrumentSnapshot const &);
|
||||
|
||||
} // namespace bts::spartan::mdp3
|
||||
+22
-49
@@ -1,62 +1,35 @@
|
||||
-- drop table msg;
|
||||
-- create table if not exists
|
||||
-- msg (
|
||||
-- "HumanTransactTime" TEXT,
|
||||
-- "HumanSendingTime" TEXT,
|
||||
-- "Latency" numeric,
|
||||
-- "TransactTime" numeric,
|
||||
-- "SendingTime" numeric,
|
||||
-- "SeqNum" numeric,
|
||||
-- "MsgType" TEXT,
|
||||
-- "TimeInQueue" numeric,
|
||||
-- "MatchEventIndicator" numeric
|
||||
-- );
|
||||
|
||||
-- drop view event_len;
|
||||
-- create view event_len as
|
||||
-- select transacttime, count(*) len from msg group by transacttime;
|
||||
|
||||
-- drop view summary_event_len;
|
||||
-- create view summary_event_len as
|
||||
-- select
|
||||
-- count(*) n_events,
|
||||
-- msg_count len
|
||||
-- from
|
||||
-- (select transacttime, count(*) msg_count from msg group by transacttime) t
|
||||
-- group by
|
||||
-- len
|
||||
drop table events;
|
||||
CREATE TABLE events(
|
||||
eid numeric,
|
||||
"t_time" numeric,
|
||||
"first_sn" numeric,
|
||||
"last_sn" numeric,
|
||||
"first_st" numeric,
|
||||
"last_st" numeric
|
||||
t_time numeric,
|
||||
first_sn numeric,
|
||||
last_sn numeric,
|
||||
first_st numeric,
|
||||
last_st numeric
|
||||
);
|
||||
drop table event_items;
|
||||
CREATE TABLE event_items(
|
||||
eid numeric,
|
||||
"t_time" numeric,
|
||||
"sid" numeric,
|
||||
"status" TEXT,
|
||||
"type" TEXT,
|
||||
"change" TEXT,
|
||||
"ok" numeric,
|
||||
"side" TEXT,
|
||||
"pri" numeric,
|
||||
"price" numeric,
|
||||
"qty" numeric,
|
||||
"tid" numeric,
|
||||
"agg" TEXT
|
||||
t_time numeric,
|
||||
sid numeric,
|
||||
status TEXT,
|
||||
type TEXT,
|
||||
change TEXT,
|
||||
ok numeric,
|
||||
side TEXT,
|
||||
pri numeric,
|
||||
price numeric,
|
||||
qty numeric,
|
||||
tid numeric,
|
||||
agg TEXT
|
||||
);
|
||||
drop table securities;
|
||||
CREATE TABLE securities(
|
||||
"sid" numeric,
|
||||
"symbol" TEXT,
|
||||
"group" TEXT,
|
||||
"asset" TEXT,
|
||||
"last_t" numeric
|
||||
sid numeric,
|
||||
symbol TEXT,
|
||||
group TEXT,
|
||||
asset TEXT,
|
||||
last_t numeric
|
||||
);
|
||||
.mode csv
|
||||
.import event_items.csv event_items
|
||||
@@ -30,7 +30,6 @@ struct PacketBuffers {
|
||||
};
|
||||
|
||||
TEST_CASE("spartan_md", "[spartan_md]") {
|
||||
//auto file = "/home/ppinto/es_md/es_all.pcap";
|
||||
auto file = os::sparkd_wd() + "/data_files/small_es.pcap";
|
||||
auto feed_map = guess_feed_map(file);
|
||||
auto reader = PCapReader{file};
|
||||
|
||||
@@ -22,9 +22,41 @@
|
||||
#include <fmt/format.h>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace bts::spartan::mdp3 {
|
||||
|
||||
void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out) {
|
||||
auto one_billion = 1000000000;
|
||||
time_t unix_ts = cme_ts / one_billion;
|
||||
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
|
||||
int hours = boost_ts.time_of_day().hours();
|
||||
int minutes = boost_ts.time_of_day().minutes();
|
||||
int seconds = boost_ts.time_of_day().seconds();
|
||||
*out << std::setw(2) << std::setfill('0') << hours << ":" << std::setw(2)
|
||||
<< std::setfill('0') << minutes << ":" << std::setw(2)
|
||||
<< std::setfill('0') << seconds << "." << std::setw(9)
|
||||
<< cme_ts - (unix_ts * one_billion);
|
||||
}
|
||||
|
||||
void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out) {
|
||||
auto one_billion = 1000000000;
|
||||
time_t unix_ts = cme_ts / one_billion;
|
||||
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
|
||||
auto year = boost_ts.date().year();
|
||||
auto month = boost_ts.date().month().as_number();
|
||||
auto day = boost_ts.date().day().as_number();
|
||||
int hours = boost_ts.time_of_day().hours();
|
||||
int minutes = boost_ts.time_of_day().minutes();
|
||||
int seconds = boost_ts.time_of_day().seconds();
|
||||
*out << year << "-" << std::setw(2) << std::setfill('0') << month << "-"
|
||||
<< std::setw(2) << std::setw(2) << std::setfill('0') << day << "."
|
||||
<< std::setw(2) << std::setw(2) << std::setfill('0') << hours << ":"
|
||||
<< std::setw(2) << std::setfill('0') << minutes << ":" << std::setw(2)
|
||||
<< std::setfill('0') << seconds << "." << std::setw(9)
|
||||
<< cme_ts - (unix_ts * one_billion);
|
||||
}
|
||||
|
||||
struct Guesser {
|
||||
FeedMap result;
|
||||
PCapReader::PacketInfo current_packet;
|
||||
@@ -121,7 +153,7 @@ std::ostream &operator<<(std::ostream &o, FeedMap const &fm) {
|
||||
return o;
|
||||
}
|
||||
|
||||
bool slim_down_pcap(std::string const &file, std::string const &out_file) {
|
||||
int compress(std::string const &file, std::string const &out_file) {
|
||||
using namespace bts::spartan;
|
||||
using namespace bts::spartan::mdp3;
|
||||
|
||||
@@ -141,7 +173,7 @@ bool slim_down_pcap(std::string const &file, std::string const &out_file) {
|
||||
struct Obs : IChannelObserver {
|
||||
bool observed_loss = false;
|
||||
bool recovery_complete = false;
|
||||
|
||||
|
||||
size_t previous_packet = 0;
|
||||
size_t previous_send_ts = 0;
|
||||
TimeStamp last_print;
|
||||
@@ -156,11 +188,10 @@ bool slim_down_pcap(std::string const &file, std::string const &out_file) {
|
||||
last_print = Timer::now();
|
||||
std::stringstream str;
|
||||
cme_ts_to_time_str(ev.transact_time.value(), &str);
|
||||
LOG(Spartan, Info,
|
||||
"Time: {} -> {} pkt/sec at {}x real time",
|
||||
str.str(),
|
||||
(ev.first_packet_sn - previous_packet),
|
||||
(ev.first_packet_send_time.value() - previous_send_ts) / 1000000000);
|
||||
LOG(Spartan, Info, "Time: {} -> {} pkt/sec at {}x real time",
|
||||
str.str(), (ev.first_packet_sn - previous_packet),
|
||||
(ev.first_packet_send_time.value() - previous_send_ts) /
|
||||
1000000000);
|
||||
previous_packet = ev.first_packet_sn;
|
||||
previous_send_ts = ev.first_packet_send_time.value();
|
||||
}
|
||||
@@ -232,41 +263,262 @@ bool slim_down_pcap(std::string const &file, std::string const &out_file) {
|
||||
|
||||
while (reader.read_next())
|
||||
if (obs.observed_loss) {
|
||||
LOG(Spartan, Error, "Observed loss in {}. Cannot slim it down", file);
|
||||
return false;
|
||||
LOG(Spartan, Error, "Observed loss in {}. Cannot compress.", file);
|
||||
return 1;
|
||||
};
|
||||
return true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out) {
|
||||
auto one_billion = 1000000000;
|
||||
time_t unix_ts = cme_ts / one_billion;
|
||||
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
|
||||
int hours = boost_ts.time_of_day().hours();
|
||||
int minutes = boost_ts.time_of_day().minutes();
|
||||
int seconds = boost_ts.time_of_day().seconds();
|
||||
*out << std::setw(2) << std::setfill('0') << hours << ":" << std::setw(2)
|
||||
<< std::setfill('0') << minutes << ":" << std::setw(2)
|
||||
<< std::setfill('0') << seconds << "." << std::setw(9)
|
||||
<< cme_ts - (unix_ts * one_billion);
|
||||
struct JSON_Writer {
|
||||
std::stringstream json_buffer;
|
||||
nlohmann::json json_obj;
|
||||
mdp3::MDP3PacketView packet;
|
||||
|
||||
void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; }
|
||||
void on_end_packet() {}
|
||||
|
||||
template <class M> void operator()(M const &m) {
|
||||
json_obj["SeqNum"] = packet.seq_num();
|
||||
json_obj["SendTime"] = packet.sending_time();
|
||||
json_obj["Size"] = packet.size();
|
||||
|
||||
json_buffer.str("");
|
||||
json_buffer.clear();
|
||||
json_buffer << m;
|
||||
json_obj["Msg"] = nlohmann::json::parse(json_buffer.str());
|
||||
std::cout << json_obj.dump(4) << std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
struct PacketBuffers {
|
||||
mdp3::MDP3Packet i;
|
||||
mdp3::MDP3Packet n;
|
||||
};
|
||||
|
||||
int to_json(std::string const&pcap) {
|
||||
auto reader = PCapReader{pcap};
|
||||
JSON_Writer writer;
|
||||
reader.on_packet() = [&](auto const &, auto const &payload) {
|
||||
parse_packet(mdp3::MDP3PacketView{payload}, writer);
|
||||
return true;
|
||||
};
|
||||
while (reader.read_next())
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out) {
|
||||
auto one_billion = 1000000000;
|
||||
time_t unix_ts = cme_ts / one_billion;
|
||||
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
|
||||
auto year = boost_ts.date().year();
|
||||
auto month = boost_ts.date().month().as_number();
|
||||
auto day = boost_ts.date().day().as_number();
|
||||
int hours = boost_ts.time_of_day().hours();
|
||||
int minutes = boost_ts.time_of_day().minutes();
|
||||
int seconds = boost_ts.time_of_day().seconds();
|
||||
*out << year << "-" << std::setw(2) << std::setfill('0') << month << "-"
|
||||
<< std::setw(2) << std::setw(2) << std::setfill('0') << day << "."
|
||||
<< std::setw(2) << std::setw(2) << std::setfill('0') << hours << ":"
|
||||
<< std::setw(2) << std::setfill('0') << minutes << ":" << std::setw(2)
|
||||
<< std::setfill('0') << seconds << "." << std::setw(9)
|
||||
<< cme_ts - (unix_ts * one_billion);
|
||||
void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
|
||||
struct ObsWrapper : mdp3::IChannelObserver {
|
||||
ObsWrapper(mdp3::IChannelObserver &obs) : real_obs(obs) {}
|
||||
IChannelObserver &real_obs;
|
||||
size_t previous_packet = 0;
|
||||
size_t previous_send_ts = 0;
|
||||
TimeStamp last_print = Timer::now();
|
||||
|
||||
void on_recovery_complete(
|
||||
std::vector<mdp3::InstrumentSnapshot const *> const &s) override {
|
||||
real_obs.on_recovery_complete(s);
|
||||
}
|
||||
void on_real_time_event(mdp3::MDP3Event const &ev) override {
|
||||
if (ms_diff(Timer::now(), last_print) >= 1000) {
|
||||
last_print = Timer::now();
|
||||
std::stringstream str;
|
||||
mdp3::cme_ts_to_time_str(ev.transact_time.value(), &str);
|
||||
LOG(Spartan, Info, "Packet time: {} ({} pkt/sec {}x real time)",
|
||||
str.str(), (ev.first_packet_sn - previous_packet),
|
||||
(ev.first_packet_send_time.value() - previous_send_ts) /
|
||||
1000000000);
|
||||
previous_packet = ev.first_packet_sn;
|
||||
previous_send_ts = ev.first_packet_send_time.value();
|
||||
}
|
||||
real_obs.on_real_time_event(ev);
|
||||
}
|
||||
void on_real_time_channel_loss() override {
|
||||
real_obs.on_real_time_channel_loss();
|
||||
}
|
||||
} obs_wrapper(obs);
|
||||
|
||||
struct Ctrl : mdp3::IFeedController {
|
||||
void start_instrument_feed() override{};
|
||||
void stop_instrument_feed() override{};
|
||||
void start_snapshot_feed() override{};
|
||||
void stop_snapshot_feed() override{};
|
||||
void start_incremental_feed() override{};
|
||||
void stop_incremental_feed() override{};
|
||||
} ctrl;
|
||||
std::array<PacketBuffers, 2> buffers;
|
||||
|
||||
auto feed_map = mdp3::guess_feed_map(file);
|
||||
bts_assert(feed_map);
|
||||
auto reader = PCapReader{file};
|
||||
mdp3::ChannelHandler handler(obs_wrapper, ctrl);
|
||||
|
||||
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
||||
if ((info.dst_ip == feed_map.incremental[0].ip &&
|
||||
info.dst_port == feed_map.incremental[0].port)) {
|
||||
buffers[0].i.set_bytes(payload);
|
||||
handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view());
|
||||
} else if ((info.dst_ip == feed_map.incremental[1].ip &&
|
||||
info.dst_port == feed_map.incremental[1].port)) {
|
||||
buffers[1].i.set_bytes(payload);
|
||||
handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view());
|
||||
} else if ((info.dst_ip == feed_map.instrument.ip &&
|
||||
info.dst_port == feed_map.instrument.port)) {
|
||||
handler.on_instrument_feed(mdp3::MDP3PacketView{payload});
|
||||
} else if ((info.dst_ip == feed_map.snapshot.ip &&
|
||||
info.dst_port == feed_map.snapshot.port)) {
|
||||
handler.on_snapshot_feed(mdp3::MDP3PacketView{payload});
|
||||
}
|
||||
return true;
|
||||
};
|
||||
while (reader.read_next())
|
||||
;
|
||||
}
|
||||
int to_csv(std::string const &file, std::string const &out_dir) {
|
||||
using namespace bts::spartan::mdp3;
|
||||
|
||||
struct Obs : IChannelObserver {
|
||||
|
||||
Obs(std::string const &out_dir)
|
||||
: events(out_dir + "/events.csv"),
|
||||
event_items(out_dir + "/event_items.csv"),
|
||||
securities(out_dir + "/securities.csv") {
|
||||
|
||||
events << "eid, t_time,first_sn,last_sn,"
|
||||
"first_st,last_st"
|
||||
<< std::endl;
|
||||
|
||||
event_items << "eid, t_time,sid,status,type,"
|
||||
"change,ok,side,"
|
||||
"pri,price,qty,tid,agg"
|
||||
<< std::endl;
|
||||
|
||||
securities << "sid,symbol,group,asset,last_t" << std::endl;
|
||||
}
|
||||
|
||||
std::ofstream events;
|
||||
std::ofstream event_items;
|
||||
std::ofstream orders;
|
||||
std::ofstream securities;
|
||||
size_t event_id = 0;
|
||||
std::unordered_set<MDP3SecurityID> known_securities;
|
||||
|
||||
void on_recovery_complete(
|
||||
std::vector<InstrumentSnapshot const *> const &secs) override {
|
||||
for (auto s : secs) {
|
||||
securities << s->security_id().value() // security id
|
||||
<< "," << s->symbol() << "," // symbol
|
||||
<< s->definition().security_group() << "," // group
|
||||
<< s->definition().asset() << "," // asset
|
||||
<< s->definition().last_update_time() // last update time
|
||||
<< std::endl;
|
||||
|
||||
known_securities.insert(s->security_id());
|
||||
|
||||
for (auto const &[id, order] : s->orders())
|
||||
event_items << 0 << "," // event_id
|
||||
<< 0 << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "order_change," // type
|
||||
<< "init," // change
|
||||
<< id << "," // order key
|
||||
<< api::Side_Name(order.side) << "," // side
|
||||
<< order.priority.value() << "," // priority
|
||||
<< order.price.price() << "," // price
|
||||
<< order.quantity.value() << "," // quantity
|
||||
<< "," // trade_entry_id
|
||||
<< "n/a" // aggressor
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
void on_real_time_event(MDP3Event const &ev) override {
|
||||
event_id++;
|
||||
events << event_id << "," // event id
|
||||
<< ev.transact_time << "," // transact time
|
||||
<< ev.first_packet_sn << "," // first packet_sn
|
||||
<< ev.last_packet_sn << "," // last_packet_sn
|
||||
<< ev.first_packet_send_time << "," // first packet send time
|
||||
<< ev.last_packet_send_time // last packet send time
|
||||
<< std::endl;
|
||||
|
||||
for (auto s : ev.affected_securities) {
|
||||
|
||||
if (known_securities.count(s->security_id()) == 0) {
|
||||
known_securities.insert(s->security_id());
|
||||
securities << s->security_id().value() // security id
|
||||
<< "," << s->symbol() << "," // symbol
|
||||
<< s->definition().security_group() << "," // group
|
||||
<< s->definition().asset() << "," // asset
|
||||
<< s->definition().last_update_time() // last update time
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
for (auto const &change : s->order_changes())
|
||||
event_items << event_id << "," // event id
|
||||
<< ev.transact_time.value() // transact time
|
||||
<< "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "order_change," // type
|
||||
<< change_name(change.change) << "," // change type
|
||||
<< change.order.key.value() << "," // order key
|
||||
<< api::Side_Name(change.order.side) << "," // side
|
||||
<< change.order.priority.value() << "," // priority
|
||||
<< change.order.price.price() << "," // price
|
||||
<< change.order.quantity.value() << "," // quantity
|
||||
<< "," // trade_entry_id
|
||||
<< "n/a" // aggressor
|
||||
<< std::endl;
|
||||
|
||||
for (auto const &f : s->fills())
|
||||
event_items << event_id << "," // event id
|
||||
<< ev.transact_time.value() << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "fill," // type
|
||||
<< "," // null
|
||||
<< f.key.value() << "," // order key
|
||||
<< "," // side
|
||||
<< "," // priority
|
||||
<< f.price << "," // price
|
||||
<< f.quantity << "," // quantity
|
||||
<< f.trade_entry_id << "," // trade_entry_id
|
||||
<< aggressor_name(f.aggressor) // aggressor
|
||||
<< std::endl;
|
||||
|
||||
auto deduced_order = s->deduced_order_change();
|
||||
if (deduced_order) {
|
||||
event_items << event_id << "," // event id
|
||||
<< ev.transact_time.value() << "," // transact time
|
||||
<< s->security_id().value() << "," // security id
|
||||
<< s->trading_status() << "," // trading status
|
||||
<< "deduced_order," // type
|
||||
<< change_name(deduced_order->change) << ","
|
||||
<< deduced_order->order.key.value()
|
||||
<< "," // order key
|
||||
<< api::Side_Name(deduced_order->order.side)
|
||||
<< "," // side
|
||||
<< "," // priority
|
||||
<< deduced_order->order.price.price() << "," // price
|
||||
<< deduced_order->order.quantity.value()
|
||||
<< "," // quantity
|
||||
<< "," // trade_entry_id
|
||||
<< "n/a" // aggressor
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void on_real_time_channel_loss() override {
|
||||
LOG(Spartan,Error,"Detected loss in feed. Aborting");
|
||||
bts_assert(false);
|
||||
}
|
||||
} obs(out_dir);
|
||||
|
||||
observe_channel(file, obs);
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace bts::spartan::mdp3
|
||||
@@ -3,9 +3,14 @@
|
||||
#include <array>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace bts::spartan::mdp3 {
|
||||
|
||||
class IChannelObserver;
|
||||
|
||||
// Details the ip/port used for each of the incremental a/b, snapshot
|
||||
// and instrument feeds in a pcap.
|
||||
struct FeedMap {
|
||||
struct Feed {
|
||||
std::string ip = "";
|
||||
@@ -16,19 +21,39 @@ struct FeedMap {
|
||||
Feed snapshot;
|
||||
Feed instrument;
|
||||
operator bool() const noexcept {
|
||||
return
|
||||
incremental[0].port &&
|
||||
incremental[1].port &&
|
||||
instrument.port &&
|
||||
snapshot.port;
|
||||
return incremental[0].port && incremental[1].port && instrument.port &&
|
||||
snapshot.port;
|
||||
}
|
||||
};
|
||||
std::ostream &operator<<(std::ostream &o, FeedMap const &fm);
|
||||
|
||||
FeedMap guess_feed_map(std::string const &pcap_path);
|
||||
bool slim_down_pcap(std::string const&in, std::string const &out);
|
||||
void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out);
|
||||
void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out);
|
||||
// Examine the passed pcap and determine, if possible, the ip/port
|
||||
// associated with each of the following feeds:
|
||||
// - Incremental A
|
||||
// - Incremental B
|
||||
// - Snapshot
|
||||
// - Instrument
|
||||
FeedMap guess_feed_map(std::string const &pcap) ;
|
||||
|
||||
// Examine the passed pcap and throw away redundant packets. Including:
|
||||
// - Instrument and Snapshoft feed packets after the initial recovery
|
||||
// - Feed A/Feed B redundant packets
|
||||
int compress(std::string const&pcap_in, std::string const &pcap_out);
|
||||
|
||||
// Convert the pcap passed to json and output it to standard out (this
|
||||
// is typically only usefull for debugging)
|
||||
int to_json(std::string const&pcap);
|
||||
|
||||
// Export the passed pcap to a set of tables in CSV format. The
|
||||
// following tables are generated:
|
||||
// - events.csv : one row per event in the pcap
|
||||
// - event_items.csv : one row per item in an event. Event items
|
||||
// include: order changes, fills and deduced order
|
||||
// - securities.csv : one row per security present in the pcap
|
||||
int to_csv(std::string const &pcap, std::string const &out_dir);
|
||||
|
||||
// Observe the pcap specified, invoking the observer for each event on
|
||||
// it
|
||||
void observe_channel(std::string const &pcap, IChannelObserver &obs) ;
|
||||
|
||||
} // namespace bts::spartan::mdp3
|
||||
Reference in new issue
Block a user