Cleaned up code.

This commit is contained in:
Pedro Pinto committed 2020-08-20 09:25:38 -04:00
1 parent 6aec729b6a
commit b871cc4e09
14 files changed
+419 -728

No files matched your search

+1 -1
View File
@@ -1 +1 @@
set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/analysis/ActionableEventAnalyzer.cpp;cmds/spartan/analysis/RunAnalysis.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
+1 -36
View File
@@ -10,7 +10,6 @@
#include "lib/domain_model/sod.h"
#include "lib/logging/logging.h"
#include "lib/utils/datetime.h"
#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
@@ -29,40 +28,6 @@
namespace bts {
namespace cmd {
using namespace bts::orderman;
using namespace spartan::mdp3;
using namespace spartan;
struct TimeWriter {
mdp3::MDP3PacketView packet;
void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; }
void on_end_packet() { std::cout << packet.sending_time() << std::endl; }
template <class M> void operator()(M const &) {}
};
void print_times(std::string const &file) {
struct PacketBuffers {
spartan::mdp3::MDP3Packet i;
spartan::mdp3::MDP3Packet n;
};
auto feed_map = spartan::mdp3::guess_feed_map(file);
bts_assert(feed_map);
auto reader = PCapReader{file};
mdp3::AB_Arbitrageur ab;
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == feed_map.incremental[0].ip &&
info.dst_port == feed_map.incremental[0].port)) {
TimeWriter tw;
parse_packet(MDP3PacketView{payload}, tw);
}
return true;
};
while (reader.read_next())
;
}
int QuickTest::execute(std::vector<std::string> const &args) {
@@ -128,7 +93,7 @@ int QuickTest::execute(std::vector<std::string> const &args) {
}
} obs;
spartan::observe_channel(file, obs);
spartan::mdp3::observe_channel(file, obs);
return 0;
}
+9 -149
View File
@@ -1,6 +1,5 @@
#include "Spartan.h"
#include "server/sparkd/cmds/spartan/algo/RunTAS.h"
#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h"
@@ -18,153 +17,15 @@
namespace bts {
namespace cmd {
int slim_pcap(std::string const &file, std::string const &out_dir) {
return !spartan::mdp3::slim_down_pcap(file, out_dir);
}
int export_tables(std::string const &file, std::string const &out_dir) {
using namespace bts::spartan::mdp3;
struct Obs : IChannelObserver {
Obs(std::string const &out_dir)
: events(out_dir + "/events.csv"),
event_items(out_dir + "/event_items.csv"),
securities(out_dir + "/securities.csv") {
events << "eid, t_time,first_sn,last_sn,"
"first_st,last_st"
<< std::endl;
event_items << "eid, t_time,sid,status,type,"
"change,ok,side,"
"pri,price,qty,tid,agg"
<< std::endl;
securities << "sid,symbol,group,asset,last_t" << std::endl;
}
std::ofstream events;
std::ofstream event_items;
std::ofstream orders;
std::ofstream securities;
size_t event_id = 0;
void on_recovery_complete(
std::vector<InstrumentSnapshot const *> const &secs) override {
for (auto s : secs) {
securities << s->security_id().value() // security id
<< "," << s->symbol() << "," // symbol
<< s->definition().security_group() << "," // group
<< s->definition().asset() << "," // asset
<< s->definition().last_update_time() // last update time
<< std::endl;
for (auto const &[id, order] : s->orders())
event_items << 0 << "," // event_id
<< 0 << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "order_change," // type
<< "init," // change
<< id << "," // order key
<< api::Side_Name(order.side) << "," // side
<< order.priority.value() << "," // priority
<< order.price.price() << "," // price
<< order.quantity.value() << "," // quantity
<< "," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
}
}
void on_real_time_event(MDP3Event const &ev) override {
event_id++;
events << event_id << "," // event id
<< ev.transact_time << "," // transactr time
<< ev.first_packet_sn << "," // first packet_sn
<< ev.last_packet_sn << "," // last_packet_sn
<< ev.first_packet_send_time << "," // first packet send time
<< ev.last_packet_send_time // last packet send time
<< std::endl;
for (auto s : ev.affected_securities) {
for (auto const &change : s->order_changes())
event_items << event_id << "," // event id
<< ev.transact_time.value() // transact time
<< "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "order_change," // type
<< change_name(change.change) << "," // change type
<< change.order.key.value() << "," // order key
<< api::Side_Name(change.order.side) << "," // side
<< change.order.priority.value() << "," // priority
<< change.order.price.price() << "," // price
<< change.order.quantity.value() << "," // quantity
<< "," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
for (auto const &f : s->fills())
event_items << event_id << "," // event id
<< ev.transact_time.value() << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "fill," // type
<< "," // null
<< f.key.value() << "," // order key
<< "," // side
<< "," // priority
<< f.price << "," // price
<< f.quantity << "," // quantity
<< f.trade_entry_id << "," // trade_entry_id
<< aggressor_name(f.aggressor) // aggressor
<< std::endl;
auto deduced_order = s->deduced_order_change();
if (deduced_order) {
event_items << event_id << "," // event id
<< ev.transact_time.value() << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "deduced_order," // type
<< change_name(deduced_order->change) << ","
<< deduced_order->order.key.value()
<< "," // order key
<< api::Side_Name(deduced_order->order.side)
<< "," // side
<< "," // priority
<< deduced_order->order.price.price() << "," // price
<< deduced_order->order.quantity.value()
<< "," // quantity
<< "," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
}
}
}
void on_real_time_channel_loss() override {
std::cout << "Observed loss, resetting\n";
}
} obs(out_dir);
spartan::observe_channel(file, obs);
return 0;
}
int Spartan::execute(std::vector<std::string> const &args) {
auto error = 1;
if (args.size() > 0) {
if (args[0] == "pcap_to_msgs")
error = spartan::convert_pcap_to_csv_summary(args);
else if (args[0] == "pcap_to_json")
error = spartan::convert_pcap_to_json(args);
else if (args[0] == "pcap_to_tables")
error = export_tables(args[1], args[2]);
else if (args[0] == "slim_pcap")
error = slim_pcap(args[1], args[2]);
if (args[0] == "json" && args.size() == 2)
error = spartan::mdp3::to_json(args[1]);
else if (args[0] == "compress" && args.size() == 3)
error = spartan::mdp3::compress(args[1],args[2]);
else if (args[0] == "csv" && args.size() == 3)
error = spartan::mdp3::to_csv(args[1], args[2]);
}
if (error)
std::cout << help();
@@ -172,10 +33,9 @@ int Spartan::execute(std::vector<std::string> const &args) {
}
std::string Spartan::help() {
return fmt::format("\tpcap_to_msgs pcap_file\n"
"\tpcap_to_json pcap_file\n"
"\tslim_pcap in_pcap out_pcap\n"
"\tpcap_to_tables pcap_file output_dir\n");
return fmt::format("\tjson <pcap>\n"
"\tcompress <pcap> <out-pcap>\n"
"\tcsv <pcap> <out-dir>");
}
} // namespace cmd
@@ -1 +0,0 @@
#include "ActionableEventAnalyzer.h"
@@ -1,59 +0,0 @@
#pragma once
#include "../md/mdp3/parsing/MDP3Parser.h"
#include "lib/utils/stats.h"
#include <stdint.h>
#include <vector>
namespace bts {
namespace spartan {
struct ActionableEventAnalyzer {
struct Event {
uint64_t transact_time;
uint64_t send_time;
uint64_t actionable_window;
uint64_t exchange_latency;
uint64_t time_from_last_transaction;
};
std::vector<Event> events;
template <class M> void operator()(mdp3::MDP3PacketHeader const&ph,mdp3::MDP3MsgHeader const &, const M &m) {
Event e{m.transactTime(), ph.sending_time, 0,
ph.sending_time - m.transactTime(), 0};
if (events.size()) {
auto &previous = events.back();
previous.actionable_window =
std::max(0l, (int64_t)(previous.send_time - m.transactTime()));
previous.time_from_last_transaction = m.transactTime() - previous.transact_time;
}
events.push_back(e);
}
std::string single_stat(uint64_t Event::*field) {
std::ostringstream str;
str.precision(0);
str << std::fixed << "\tmin: "
<< stats::min(events, [field](auto const &ev) { return ev.*field; })
<< "\t\tmed: "
<< stats::med(events, [field](auto const &ev) { return ev.*field; })
<< "\t\tmax: "
<< stats::max(events, [field](auto const &ev) { return ev.*field; });
return str.str();
}
~ActionableEventAnalyzer() {
std::cout << "Data points:\n \t\t" << events.size()
<< "\nExchange Latency:\n"
<< single_stat(&Event::exchange_latency)
<< "\nTransaction Gap:\n"
<< single_stat(&Event::time_from_last_transaction)
<< "\nActionable Window:\n"
<< single_stat(&Event::actionable_window) << "\n";
}
};
} // namespace spartan
} // namespace bts
@@ -1,251 +0,0 @@
#include "server/sparkd/cmds/spartan/analysis/RunAnalysis.h"
#include "../md/mdp3/parsing/MDP3Parser.h"
#include "../util/PCapReader.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/overloaded.h"
#include "lib/utils/stats.h"
#include "lib/utils/type_name.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h"
#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h"
#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include <boost/date_time/posix_time/conversion.hpp>
#include <boost/date_time/posix_time/ptime.hpp>
#include <iostream>
#include <nlohmann/json.hpp>
namespace bts {
namespace spartan {
struct PacketBuffers {
mdp3::MDP3Packet i;
mdp3::MDP3Packet n;
};
mdp3::AB_Arbitrageur ab;
struct JSON_Writer {
std::stringstream json_buffer;
nlohmann::json json_obj;
mdp3::MDP3PacketView packet;
void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; }
void on_end_packet() {}
template <class M> void operator()(M const &m) {
json_obj["SeqNum"] = packet.seq_num();
json_obj["SendTime"] = packet.sending_time();
json_obj["Size"] = packet.size();
json_buffer.str("");
json_buffer.clear();
json_buffer << m;
json_obj["Msg"] = nlohmann::json::parse(json_buffer.str());
std::cout << json_obj.dump(4) << std::endl;
}
};
int convert_pcap_to_json(std::vector<std::string> const &args) {
std::array<PacketBuffers, 2> buffers;
auto reader = PCapReader{args[1]};
JSON_Writer writer;
reader.on_packet() = [&](auto const &, auto const &payload) {
parse_packet(mdp3::MDP3PacketView{payload}, writer);
return true;
};
while (reader.read_next())
;
return 0;
}
template <class M, decltype(((M *)nullptr)->transactTime()) = 0>
uint64_t get_transact_time(M &m) {
return m.transactTime();
}
template <class M, decltype(((M *)nullptr)->lastUpdateTime()) = 0>
uint64_t get_transact_time(M &m) {
return m.lastUpdateTime();
}
struct CSVPrinter {
std::stringstream stime_buffer;
std::stringstream ttime_buffer;
size_t previous_t_time = 0;
size_t previous_s_time = 0;
mdp3::MDP3PacketView packet;
CSVPrinter() {
std::cout
<< "HumanTransactTime,HumanSendingTime,"
"Latency,TransactTime,SendingTime,SeqNum,MsgType,TimeInQueue,"
"MatchEventindicator"
<< std::endl;
}
void on_begin_packet(mdp3::MDP3PacketView const &v) { packet = v; }
void on_end_packet() {}
template <class M> void print(M &m) {
stime_buffer.str("");
stime_buffer.clear();
mdp3::cme_ts_to_date_str(packet.sending_time(), &stime_buffer);
ttime_buffer.str("");
ttime_buffer.clear();
mdp3::cme_ts_to_date_str(get_transact_time(m), &ttime_buffer);
auto t = get_transact_time(m);
std::cout << ttime_buffer.str() << "," << stime_buffer.str() << ","
<< (packet.sending_time() - t) / 1000 << "," << t << ","
<< packet.sending_time() << "," << packet.seq_num() << ","
<< utils::type_name<M>() << ","
<< (t < previous_s_time ? ((previous_s_time - t) / 1000) : 0)
<< "," << m.matchEventIndicator().endOfEvent() << std::endl;
previous_s_time = packet.sending_time();
previous_t_time = t;
}
void operator()(::mdp3::MDIncrementalRefreshOrderBook43 &m) { print(m); }
void operator()(::mdp3::MDIncrementalRefreshOrderBook47 &m) { print(m); }
void operator()(::mdp3::MDIncrementalRefreshTradeSummary42 &m) { print(m); }
void operator()(::mdp3::MDIncrementalRefreshTradeSummary48 &m) { print(m); }
void operator()(::mdp3::MDIncrementalRefreshVolume37 &m) { print(m); }
void operator()(::mdp3::MDIncrementalRefreshDailyStatistics33 &m) {
print(m);
}
void operator()(::mdp3::MDIncrementalRefreshDailyStatistics49 &m) {
print(m);
}
void operator()(::mdp3::MDIncrementalRefreshLimitsBanding34 &m) { print(m); }
void operator()(::mdp3::MDIncrementalRefreshLimitsBanding50 &m) { print(m); }
void operator()(::mdp3::MDIncrementalRefreshBook46 &m) { print(m); }
void operator()(::mdp3::MDInstrumentDefinitionFuture54 &m) { print(m); }
void operator()(::mdp3::MDIncrementalRefreshSessionStatistics35 &m) {
print(m);
}
void operator()(::mdp3::MDIncrementalRefreshSessionStatistics51 &m) {
print(m);
}
void operator()(::mdp3::MDInstrumentDefinitionFuture27 &m) { print(m); }
void operator()(::mdp3::ChannelReset4 &m) { print(m); }
void operator()(::mdp3::MDIncrementalRefreshBook32 &m) { print(m); }
void operator()(::mdp3::MDInstrumentDefinitionOption41 &m) { print(m); }
void operator()(::mdp3::MDInstrumentDefinitionOption55 &m) { print(m); }
void operator()(::mdp3::MDInstrumentDefinitionSpread29 &m) { print(m); }
void operator()(::mdp3::MDInstrumentDefinitionSpread56 &m) { print(m); }
void operator()(::mdp3::QuoteRequest39 &m) { print(m); }
void operator()(::mdp3::SecurityStatus30 &m) { print(m); }
};
int convert_pcap_to_csv_summary(std::vector<std::string> const &args) {
std::array<PacketBuffers, 2> buffers;
auto feed_map = mdp3::guess_feed_map(args[1]);
bts_assert(feed_map);
auto reader = PCapReader{args[1]};
mdp3::AB_Arbitrageur ab;
CSVPrinter csv_printer;
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == feed_map.incremental[0].ip &&
info.dst_port == feed_map.incremental[0].port)) {
buffers[0].i = mdp3::MDP3Packet{payload};
} else if ((info.dst_ip == feed_map.incremental[0].ip &&
info.dst_port == feed_map.incremental[0].port)) {
buffers[1].i = mdp3::MDP3Packet{payload};
} else
return true;
auto action = ab(buffers[0].i.view(), buffers[1].i.view());
if (action.action == mdp3::AB_Arbitrageur::APPLY)
parse_packet(action.packet, csv_printer);
else if (action.action == mdp3::AB_Arbitrageur::RECOVER) {
std::cerr << "Unrecoverable loss detecte at sequence number: "
<< ab.expected_next_sn() - 1 << std::endl;
return false;
}
return true;
};
while (reader.read_next())
;
return 0;
}
void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
struct ObsWrapper : mdp3::IChannelObserver {
ObsWrapper(mdp3::IChannelObserver &obs) : real_obs(obs) {}
IChannelObserver &real_obs;
size_t previous_packet = 0;
size_t previous_send_ts = 0;
TimeStamp last_print = Timer::now();
void on_recovery_complete(
std::vector<mdp3::InstrumentSnapshot const *> const &s) override {
real_obs.on_recovery_complete(s);
}
void on_real_time_event(mdp3::MDP3Event const &ev) override {
if (ms_diff(Timer::now(), last_print) >= 1000) {
last_print = Timer::now();
std::stringstream str;
mdp3::cme_ts_to_time_str(ev.transact_time.value(), &str);
LOG(Spartan, Info, "Packet time: {} ({} pkt/sec {}x real time)",
str.str(), (ev.first_packet_sn - previous_packet),
(ev.first_packet_send_time.value() - previous_send_ts) /
1000000000);
previous_packet = ev.first_packet_sn;
previous_send_ts = ev.first_packet_send_time.value();
}
real_obs.on_real_time_event(ev);
}
void on_real_time_channel_loss() override {
real_obs.on_real_time_channel_loss();
}
} obs_wrapper(obs);
struct Ctrl : mdp3::IFeedController {
void start_instrument_feed() override{};
void stop_instrument_feed() override{};
void start_snapshot_feed() override{};
void stop_snapshot_feed() override{};
void start_incremental_feed() override{};
void stop_incremental_feed() override{};
} ctrl;
std::array<PacketBuffers, 2> buffers;
auto feed_map = mdp3::guess_feed_map(file);
bts_assert(feed_map);
auto reader = PCapReader{file};
mdp3::ChannelHandler handler(obs_wrapper, ctrl);
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == feed_map.incremental[0].ip &&
info.dst_port == feed_map.incremental[0].port)) {
buffers[0].i.set_bytes(payload);
handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view());
} else if ((info.dst_ip == feed_map.incremental[1].ip &&
info.dst_port == feed_map.incremental[1].port)) {
buffers[1].i.set_bytes(payload);
handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view());
} else if ((info.dst_ip == feed_map.instrument.ip &&
info.dst_port == feed_map.instrument.port)) {
handler.on_instrument_feed(mdp3::MDP3PacketView{payload});
} else if ((info.dst_ip == feed_map.snapshot.ip &&
info.dst_port == feed_map.snapshot.port)) {
handler.on_snapshot_feed(mdp3::MDP3PacketView{payload});
}
return true;
};
while (reader.read_next())
;
}
} // namespace spartan
} // namespace bts
@@ -1,14 +0,0 @@
#pragma once
#include <vector>
#include <string>
namespace bts::spartan::mdp3 {
class IChannelObserver;
}
namespace bts::spartan {
int convert_pcap_to_csv_summary(std::vector<std::string> const &args);
int convert_pcap_to_json(std::vector<std::string> const &args );
void observe_channel(std::string const &, mdp3::IChannelObserver &) ;
}
@@ -1,29 +0,0 @@
* Arbitration
on_msg (Msg* a, Msg *b) {
if (a && a->seq_num == seq_num)
seq_n++;
deliver(*a);
return;
}
if (b && b->seq_num == seq_num)
seq_n++;
deliver(*b);
return;
}
on_loss(seq_num);
}
* On startup
** In recovery
- join instrument feed
- join a, b
- join sambo
- queued events is not 0 and its an unbroken sequence
- joined the snapshot feed, and it either has a seq n < the the top of the queue (so the snapshot is useless) or we are missing some messages in the snapshot
- we are either joined with the instrument feed but still not gone through a full cycle or we have a full set of securities
- begin recovery
- b
@@ -1,29 +0,0 @@
#pragma once
#include <memory>
class mdp3::MDInstrumentDefinitionFuture27;
class mdp3::MDInstrumentDefinitionFuture54;
class mdp3::MDInstrumentDefinitionOption41;
class mdp3::MDInstrumentDefinitionOption55;
class mdp3::MDInstrumentDefinitionSpread29;
class mdp3::MDInstrumentDefinitionSpread56;
namespace bts {
namespace spartan {
class InstrumentStore {
InstrumentStore(std::string const&base_path);
void store( const mdp3::MDInstrumentDefinitionFuture27&);
void store( const mdp3::MDInstrumentDefinitionFuture54&);
void store( const mdp3::MDInstrumentDefinitionOption41&);
void store( const mdp3::MDInstrumentDefinitionOption55&);
void store( const mdp3::MDInstrumentDefinitionSpread29&);
void store( const mdp3::MDInstrumentDefinitionSpread56&);
private:
struct PImpl;
std::unique_ptr<PImpl> me;
};
}}
@@ -1,7 +1,7 @@
#pragma once
#include "mdp3_common.h"
#include "InstrumentDefinition.h"
#include "mdp3_common.h"
#include "lib/domain_model/Security.pb.h"
#include <memory>
@@ -17,103 +17,103 @@ namespace bts::spartan::mdp3 {
// - The instrument's definition
// - Price books (implied and customer)
// - Resting orders
// - Fills observed so far
// - Fills generated by the last event
// - Order changes generated in the last event
// - The deduced order for the last event
// - Market statistics
class InstrumentSnapshot {
public:
InstrumentSnapshot(InstrumentDefinition const&);
InstrumentSnapshot(InstrumentDefinition const &);
InstrumentSnapshot(InstrumentDefinition &&);
~InstrumentSnapshot();
// Definition
// Queries ------------------------------------------------------------
// Security
MDP3SecurityID security_id() const;
InstrumentDefinition definition() const;
void set_definition(InstrumentDefinition const&);
// Symbol (via definition)
const char* symbol() const {return definition().symbol();}
// Last packet that affected this security
size_t last_processed_sn() const ;
void set_last_msg_processed_sn(size_t);
// Trading status
const char *symbol() const { return definition().symbol(); }
::mdp3::SecurityTradingStatus::Value trading_status() const;
void set_trading_status(::mdp3::SecurityTradingStatus::Value) const;
// Book
// Last packet incorporated in this snapshot
size_t last_processed_sn() const;
// Books
Book const &book(api::Side) const;
void insert_book_level(api::Side, size_t idx,BookLevel const&);
void update_book_level(api::Side, size_t idx,BookLevel const&);
void remove_book_level(api::Side, size_t idx);
// Implied Book
ImpliedBook const &implied_book(api::Side) const;
void insert_implied_book_level(api::Side, size_t idx, BookLevel const&);
void update_implied_book_level(api::Side, size_t idx, BookLevel const&);
void remove_implied_book_level(api::Side, size_t idx);
// Orders
Orders const &orders() const;
// Last round of order changes
OrderChanges const &order_changes() const;
void clear_order_changes() ;
void insert_order(MDP3Order const&);
void update_order(MDP3Order const&);
void remove_order(MDP3OrderKey);
Book build_book_from_orders (api::Side ) const;
void set_books_from_orders() ;
// Deduced order changes, that is the implied order change that
// Deduced order change, that is the order change that
// would explain the last event processed in this snapshot as
// described by order_changes() and fills()
MaybeOrderChange deduced_order_change() const;
// Reset books in response to a channel reset message
void reset_books();
// Fills
Fills const&fills() const;
void add_fill(MDP3Fill const&);
void update_fill(MDP3Fill const&);
void remove_fill(MDP3TradeEntryID, MDP3OrderKey );
void clear_fills();
// Stats
// Last round of fills
Fills const &fills() const;
// Stats
md::IntPrice high_limit_price() const;
void set_high_limit_price(md::IntPrice);
md::IntPrice low_limit_price() const;
void set_low_limit_price(md::IntPrice);
md::IntPrice max_price_variation() const;
void set_max_price_variation(md::IntPrice);
MDP3Quantity cleared_volume() const;
void set_cleared_volume(MDP3Quantity);
MDP3Quantity electronic_volume() const;
void set_electronic_volume(MDP3Quantity);
MDP3Quantity cleared_volume() const;
md::IntPrice fixing_price() const;
void set_fixing_price(md::IntPrice);
MDP3Quantity open_interest() const;
void set_open_interest(MDP3Quantity);
md::IntPrice open_price() const;
void set_open_price(md::IntPrice ) ;
md::IntPrice high_bid() const;
void set_high_bid(md::IntPrice ) ;
md::IntPrice low_offer() const;
void set_low_offer(md::IntPrice ) ;
md::IntPrice settlement_price() const;
void set_settlement_price(md::IntPrice ) ;
md::IntPrice high_trade_price() const;
void set_high_trade_price(md::IntPrice ) ;
md::IntPrice low_trade_price() const;
void set_low_trade_price(md::IntPrice ) ;
// Mutators ------------------------------------------------------------
void set_definition(InstrumentDefinition const &);
void set_last_msg_processed_sn(size_t);
void set_trading_status(::mdp3::SecurityTradingStatus::Value) const;
void insert_book_level(api::Side, size_t idx, BookLevel const &);
void update_book_level(api::Side, size_t idx, BookLevel const &);
void remove_book_level(api::Side, size_t idx);
void insert_implied_book_level(api::Side, size_t idx, BookLevel const &);
void update_implied_book_level(api::Side, size_t idx, BookLevel const &);
void remove_implied_book_level(api::Side, size_t idx);
void clear_order_changes();
void insert_order(MDP3Order const &);
void update_order(MDP3Order const &);
void remove_order(MDP3OrderKey);
Book build_book_from_orders(api::Side) const;
void set_books_from_orders();
void reset_books();
void add_fill(MDP3Fill const &);
void update_fill(MDP3Fill const &);
void remove_fill(MDP3TradeEntryID, MDP3OrderKey);
void clear_fills();
void set_high_limit_price(md::IntPrice);
void set_low_limit_price(md::IntPrice);
void set_max_price_variation(md::IntPrice);
void set_cleared_volume(MDP3Quantity);
void set_electronic_volume(MDP3Quantity);
void set_fixing_price(md::IntPrice);
void set_open_interest(MDP3Quantity);
void set_open_price(md::IntPrice);
void set_high_bid(md::IntPrice);
void set_low_offer(md::IntPrice);
void set_settlement_price(md::IntPrice);
void set_high_trade_price(md::IntPrice);
void set_low_trade_price(md::IntPrice);
private:
template<class A> void insert(api::Side,std::array<A,2> &, size_t idx,BookLevel const&);
template<class A> void update(api::Side,std::array<A,2> &, size_t idx, BookLevel const&);
template<class A> void remove(api::Side,std::array<A,2> &, size_t idx);
template <class A>
void insert(api::Side, std::array<A, 2> &, size_t idx, BookLevel const &);
template <class A>
void update(api::Side, std::array<A, 2> &, size_t idx, BookLevel const &);
template <class A> void remove(api::Side, std::array<A, 2> &, size_t idx);
struct PImpl;
std::unique_ptr<PImpl> me;
};
};
std::ostream& operator<<(std::ostream&, InstrumentSnapshot const&);
std::ostream &operator<<(std::ostream &, InstrumentSnapshot const &);
} // namespace bts::spartan::mdp3
@@ -1,62 +1,35 @@
-- drop table msg;
-- create table if not exists
-- msg (
-- "HumanTransactTime" TEXT,
-- "HumanSendingTime" TEXT,
-- "Latency" numeric,
-- "TransactTime" numeric,
-- "SendingTime" numeric,
-- "SeqNum" numeric,
-- "MsgType" TEXT,
-- "TimeInQueue" numeric,
-- "MatchEventIndicator" numeric
-- );
-- drop view event_len;
-- create view event_len as
-- select transacttime, count(*) len from msg group by transacttime;
-- drop view summary_event_len;
-- create view summary_event_len as
-- select
-- count(*) n_events,
-- msg_count len
-- from
-- (select transacttime, count(*) msg_count from msg group by transacttime) t
-- group by
-- len
drop table events;
CREATE TABLE events(
eid numeric,
"t_time" numeric,
"first_sn" numeric,
"last_sn" numeric,
"first_st" numeric,
"last_st" numeric
t_time numeric,
first_sn numeric,
last_sn numeric,
first_st numeric,
last_st numeric
);
drop table event_items;
CREATE TABLE event_items(
eid numeric,
"t_time" numeric,
"sid" numeric,
"status" TEXT,
"type" TEXT,
"change" TEXT,
"ok" numeric,
"side" TEXT,
"pri" numeric,
"price" numeric,
"qty" numeric,
"tid" numeric,
"agg" TEXT
t_time numeric,
sid numeric,
status TEXT,
type TEXT,
change TEXT,
ok numeric,
side TEXT,
pri numeric,
price numeric,
qty numeric,
tid numeric,
agg TEXT
);
drop table securities;
CREATE TABLE securities(
"sid" numeric,
"symbol" TEXT,
"group" TEXT,
"asset" TEXT,
"last_t" numeric
sid numeric,
symbol TEXT,
group TEXT,
asset TEXT,
last_t numeric
);
.mode csv
.import event_items.csv event_items
@@ -30,7 +30,6 @@ struct PacketBuffers {
};
TEST_CASE("spartan_md", "[spartan_md]") {
//auto file = "/home/ppinto/es_md/es_all.pcap";
auto file = os::sparkd_wd() + "/data_files/small_es.pcap";
auto feed_map = guess_feed_map(file);
auto reader = PCapReader{file};
@@ -22,9 +22,41 @@
#include <fmt/format.h>
#include <fstream>
#include <iostream>
#include <nlohmann/json.hpp>
namespace bts::spartan::mdp3 {
void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out) {
auto one_billion = 1000000000;
time_t unix_ts = cme_ts / one_billion;
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
int hours = boost_ts.time_of_day().hours();
int minutes = boost_ts.time_of_day().minutes();
int seconds = boost_ts.time_of_day().seconds();
*out << std::setw(2) << std::setfill('0') << hours << ":" << std::setw(2)
<< std::setfill('0') << minutes << ":" << std::setw(2)
<< std::setfill('0') << seconds << "." << std::setw(9)
<< cme_ts - (unix_ts * one_billion);
}
void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out) {
auto one_billion = 1000000000;
time_t unix_ts = cme_ts / one_billion;
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
auto year = boost_ts.date().year();
auto month = boost_ts.date().month().as_number();
auto day = boost_ts.date().day().as_number();
int hours = boost_ts.time_of_day().hours();
int minutes = boost_ts.time_of_day().minutes();
int seconds = boost_ts.time_of_day().seconds();
*out << year << "-" << std::setw(2) << std::setfill('0') << month << "-"
<< std::setw(2) << std::setw(2) << std::setfill('0') << day << "."
<< std::setw(2) << std::setw(2) << std::setfill('0') << hours << ":"
<< std::setw(2) << std::setfill('0') << minutes << ":" << std::setw(2)
<< std::setfill('0') << seconds << "." << std::setw(9)
<< cme_ts - (unix_ts * one_billion);
}
struct Guesser {
FeedMap result;
PCapReader::PacketInfo current_packet;
@@ -121,7 +153,7 @@ std::ostream &operator<<(std::ostream &o, FeedMap const &fm) {
return o;
}
bool slim_down_pcap(std::string const &file, std::string const &out_file) {
int compress(std::string const &file, std::string const &out_file) {
using namespace bts::spartan;
using namespace bts::spartan::mdp3;
@@ -141,7 +173,7 @@ bool slim_down_pcap(std::string const &file, std::string const &out_file) {
struct Obs : IChannelObserver {
bool observed_loss = false;
bool recovery_complete = false;
size_t previous_packet = 0;
size_t previous_send_ts = 0;
TimeStamp last_print;
@@ -156,11 +188,10 @@ bool slim_down_pcap(std::string const &file, std::string const &out_file) {
last_print = Timer::now();
std::stringstream str;
cme_ts_to_time_str(ev.transact_time.value(), &str);
LOG(Spartan, Info,
"Time: {} -> {} pkt/sec at {}x real time",
str.str(),
(ev.first_packet_sn - previous_packet),
(ev.first_packet_send_time.value() - previous_send_ts) / 1000000000);
LOG(Spartan, Info, "Time: {} -> {} pkt/sec at {}x real time",
str.str(), (ev.first_packet_sn - previous_packet),
(ev.first_packet_send_time.value() - previous_send_ts) /
1000000000);
previous_packet = ev.first_packet_sn;
previous_send_ts = ev.first_packet_send_time.value();
}
@@ -232,41 +263,262 @@ bool slim_down_pcap(std::string const &file, std::string const &out_file) {
while (reader.read_next())
if (obs.observed_loss) {
LOG(Spartan, Error, "Observed loss in {}. Cannot slim it down", file);
return false;
LOG(Spartan, Error, "Observed loss in {}. Cannot compress.", file);
return 1;
};
return true;
return 0;
}
void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out) {
auto one_billion = 1000000000;
time_t unix_ts = cme_ts / one_billion;
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
int hours = boost_ts.time_of_day().hours();
int minutes = boost_ts.time_of_day().minutes();
int seconds = boost_ts.time_of_day().seconds();
*out << std::setw(2) << std::setfill('0') << hours << ":" << std::setw(2)
<< std::setfill('0') << minutes << ":" << std::setw(2)
<< std::setfill('0') << seconds << "." << std::setw(9)
<< cme_ts - (unix_ts * one_billion);
struct JSON_Writer {
std::stringstream json_buffer;
nlohmann::json json_obj;
mdp3::MDP3PacketView packet;
void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; }
void on_end_packet() {}
template <class M> void operator()(M const &m) {
json_obj["SeqNum"] = packet.seq_num();
json_obj["SendTime"] = packet.sending_time();
json_obj["Size"] = packet.size();
json_buffer.str("");
json_buffer.clear();
json_buffer << m;
json_obj["Msg"] = nlohmann::json::parse(json_buffer.str());
std::cout << json_obj.dump(4) << std::endl;
}
};
struct PacketBuffers {
mdp3::MDP3Packet i;
mdp3::MDP3Packet n;
};
int to_json(std::string const&pcap) {
auto reader = PCapReader{pcap};
JSON_Writer writer;
reader.on_packet() = [&](auto const &, auto const &payload) {
parse_packet(mdp3::MDP3PacketView{payload}, writer);
return true;
};
while (reader.read_next())
;
return 0;
}
void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out) {
auto one_billion = 1000000000;
time_t unix_ts = cme_ts / one_billion;
auto boost_ts = boost::posix_time::from_time_t(unix_ts);
auto year = boost_ts.date().year();
auto month = boost_ts.date().month().as_number();
auto day = boost_ts.date().day().as_number();
int hours = boost_ts.time_of_day().hours();
int minutes = boost_ts.time_of_day().minutes();
int seconds = boost_ts.time_of_day().seconds();
*out << year << "-" << std::setw(2) << std::setfill('0') << month << "-"
<< std::setw(2) << std::setw(2) << std::setfill('0') << day << "."
<< std::setw(2) << std::setw(2) << std::setfill('0') << hours << ":"
<< std::setw(2) << std::setfill('0') << minutes << ":" << std::setw(2)
<< std::setfill('0') << seconds << "." << std::setw(9)
<< cme_ts - (unix_ts * one_billion);
void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
struct ObsWrapper : mdp3::IChannelObserver {
ObsWrapper(mdp3::IChannelObserver &obs) : real_obs(obs) {}
IChannelObserver &real_obs;
size_t previous_packet = 0;
size_t previous_send_ts = 0;
TimeStamp last_print = Timer::now();
void on_recovery_complete(
std::vector<mdp3::InstrumentSnapshot const *> const &s) override {
real_obs.on_recovery_complete(s);
}
void on_real_time_event(mdp3::MDP3Event const &ev) override {
if (ms_diff(Timer::now(), last_print) >= 1000) {
last_print = Timer::now();
std::stringstream str;
mdp3::cme_ts_to_time_str(ev.transact_time.value(), &str);
LOG(Spartan, Info, "Packet time: {} ({} pkt/sec {}x real time)",
str.str(), (ev.first_packet_sn - previous_packet),
(ev.first_packet_send_time.value() - previous_send_ts) /
1000000000);
previous_packet = ev.first_packet_sn;
previous_send_ts = ev.first_packet_send_time.value();
}
real_obs.on_real_time_event(ev);
}
void on_real_time_channel_loss() override {
real_obs.on_real_time_channel_loss();
}
} obs_wrapper(obs);
struct Ctrl : mdp3::IFeedController {
void start_instrument_feed() override{};
void stop_instrument_feed() override{};
void start_snapshot_feed() override{};
void stop_snapshot_feed() override{};
void start_incremental_feed() override{};
void stop_incremental_feed() override{};
} ctrl;
std::array<PacketBuffers, 2> buffers;
auto feed_map = mdp3::guess_feed_map(file);
bts_assert(feed_map);
auto reader = PCapReader{file};
mdp3::ChannelHandler handler(obs_wrapper, ctrl);
reader.on_packet() = [&](auto const &info, auto const &payload) {
if ((info.dst_ip == feed_map.incremental[0].ip &&
info.dst_port == feed_map.incremental[0].port)) {
buffers[0].i.set_bytes(payload);
handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view());
} else if ((info.dst_ip == feed_map.incremental[1].ip &&
info.dst_port == feed_map.incremental[1].port)) {
buffers[1].i.set_bytes(payload);
handler.on_incremental_feed(buffers[0].i.view(), buffers[1].i.view());
} else if ((info.dst_ip == feed_map.instrument.ip &&
info.dst_port == feed_map.instrument.port)) {
handler.on_instrument_feed(mdp3::MDP3PacketView{payload});
} else if ((info.dst_ip == feed_map.snapshot.ip &&
info.dst_port == feed_map.snapshot.port)) {
handler.on_snapshot_feed(mdp3::MDP3PacketView{payload});
}
return true;
};
while (reader.read_next())
;
}
int to_csv(std::string const &file, std::string const &out_dir) {
using namespace bts::spartan::mdp3;
struct Obs : IChannelObserver {
Obs(std::string const &out_dir)
: events(out_dir + "/events.csv"),
event_items(out_dir + "/event_items.csv"),
securities(out_dir + "/securities.csv") {
events << "eid, t_time,first_sn,last_sn,"
"first_st,last_st"
<< std::endl;
event_items << "eid, t_time,sid,status,type,"
"change,ok,side,"
"pri,price,qty,tid,agg"
<< std::endl;
securities << "sid,symbol,group,asset,last_t" << std::endl;
}
std::ofstream events;
std::ofstream event_items;
std::ofstream orders;
std::ofstream securities;
size_t event_id = 0;
std::unordered_set<MDP3SecurityID> known_securities;
void on_recovery_complete(
std::vector<InstrumentSnapshot const *> const &secs) override {
for (auto s : secs) {
securities << s->security_id().value() // security id
<< "," << s->symbol() << "," // symbol
<< s->definition().security_group() << "," // group
<< s->definition().asset() << "," // asset
<< s->definition().last_update_time() // last update time
<< std::endl;
known_securities.insert(s->security_id());
for (auto const &[id, order] : s->orders())
event_items << 0 << "," // event_id
<< 0 << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "order_change," // type
<< "init," // change
<< id << "," // order key
<< api::Side_Name(order.side) << "," // side
<< order.priority.value() << "," // priority
<< order.price.price() << "," // price
<< order.quantity.value() << "," // quantity
<< "," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
}
}
void on_real_time_event(MDP3Event const &ev) override {
event_id++;
events << event_id << "," // event id
<< ev.transact_time << "," // transact time
<< ev.first_packet_sn << "," // first packet_sn
<< ev.last_packet_sn << "," // last_packet_sn
<< ev.first_packet_send_time << "," // first packet send time
<< ev.last_packet_send_time // last packet send time
<< std::endl;
for (auto s : ev.affected_securities) {
if (known_securities.count(s->security_id()) == 0) {
known_securities.insert(s->security_id());
securities << s->security_id().value() // security id
<< "," << s->symbol() << "," // symbol
<< s->definition().security_group() << "," // group
<< s->definition().asset() << "," // asset
<< s->definition().last_update_time() // last update time
<< std::endl;
}
for (auto const &change : s->order_changes())
event_items << event_id << "," // event id
<< ev.transact_time.value() // transact time
<< "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "order_change," // type
<< change_name(change.change) << "," // change type
<< change.order.key.value() << "," // order key
<< api::Side_Name(change.order.side) << "," // side
<< change.order.priority.value() << "," // priority
<< change.order.price.price() << "," // price
<< change.order.quantity.value() << "," // quantity
<< "," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
for (auto const &f : s->fills())
event_items << event_id << "," // event id
<< ev.transact_time.value() << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "fill," // type
<< "," // null
<< f.key.value() << "," // order key
<< "," // side
<< "," // priority
<< f.price << "," // price
<< f.quantity << "," // quantity
<< f.trade_entry_id << "," // trade_entry_id
<< aggressor_name(f.aggressor) // aggressor
<< std::endl;
auto deduced_order = s->deduced_order_change();
if (deduced_order) {
event_items << event_id << "," // event id
<< ev.transact_time.value() << "," // transact time
<< s->security_id().value() << "," // security id
<< s->trading_status() << "," // trading status
<< "deduced_order," // type
<< change_name(deduced_order->change) << ","
<< deduced_order->order.key.value()
<< "," // order key
<< api::Side_Name(deduced_order->order.side)
<< "," // side
<< "," // priority
<< deduced_order->order.price.price() << "," // price
<< deduced_order->order.quantity.value()
<< "," // quantity
<< "," // trade_entry_id
<< "n/a" // aggressor
<< std::endl;
}
}
}
void on_real_time_channel_loss() override {
LOG(Spartan,Error,"Detected loss in feed. Aborting");
bts_assert(false);
}
} obs(out_dir);
observe_channel(file, obs);
return 0;
}
} // namespace bts::spartan::mdp3
@@ -3,9 +3,14 @@
#include <array>
#include <iostream>
#include <string>
#include <vector>
namespace bts::spartan::mdp3 {
class IChannelObserver;
// Details the ip/port used for each of the incremental a/b, snapshot
// and instrument feeds in a pcap.
struct FeedMap {
struct Feed {
std::string ip = "";
@@ -16,19 +21,39 @@ struct FeedMap {
Feed snapshot;
Feed instrument;
operator bool() const noexcept {
return
incremental[0].port &&
incremental[1].port &&
instrument.port &&
snapshot.port;
return incremental[0].port && incremental[1].port && instrument.port &&
snapshot.port;
}
};
std::ostream &operator<<(std::ostream &o, FeedMap const &fm);
FeedMap guess_feed_map(std::string const &pcap_path);
bool slim_down_pcap(std::string const&in, std::string const &out);
void cme_ts_to_date_str(uint64_t cme_ts, std::stringstream *out);
void cme_ts_to_time_str(uint64_t cme_ts, std::stringstream *out);
// Examine the passed pcap and determine, if possible, the ip/port
// associated with each of the following feeds:
// - Incremental A
// - Incremental B
// - Snapshot
// - Instrument
FeedMap guess_feed_map(std::string const &pcap) ;
// Examine the passed pcap and throw away redundant packets. Including:
// - Instrument and Snapshoft feed packets after the initial recovery
// - Feed A/Feed B redundant packets
int compress(std::string const&pcap_in, std::string const &pcap_out);
// Convert the pcap passed to json and output it to standard out (this
// is typically only usefull for debugging)
int to_json(std::string const&pcap);
// Export the passed pcap to a set of tables in CSV format. The
// following tables are generated:
// - events.csv : one row per event in the pcap
// - event_items.csv : one row per item in an event. Event items
// include: order changes, fills and deduced order
// - securities.csv : one row per security present in the pcap
int to_csv(std::string const &pcap, std::string const &out_dir);
// Observe the pcap specified, invoking the observer for each event on
// it
void observe_channel(std::string const &pcap, IChannelObserver &obs) ;
} // namespace bts::spartan::mdp3