811 lines
31 KiB
C++
811 lines
31 KiB
C++
#include "mdp3_utils.h"
|
|
|
|
#include "server/sparkd/cmds/spartan/system/PCapReader.h"
|
|
#include "../common/InstrumentSnapshot.h"
|
|
#include "../feed_control/AB_Arbitrageur.h"
|
|
#include "../feed_control/IFeedController.h"
|
|
#include "../gen/MDIncrementalRefreshOrderBook47.h"
|
|
#include "../gen/MDInstrumentDefinitionFuture54.h"
|
|
#include "../gen/MDInstrumentDefinitionOption55.h"
|
|
#include "../gen/MDInstrumentDefinitionSpread56.h"
|
|
#include "../gen/SnapshotFullRefreshOrderBook53.h"
|
|
#include "../handler/ChannelHandler.h"
|
|
#include "../handler/IChannelObserver.h"
|
|
#include "../parsing/MDP3Parser.h"
|
|
#include "lib/jetfix/codec/f_toa.h"
|
|
#include "lib/config/config.h"
|
|
#include "lib/config/protos/spartan.pb.h"
|
|
|
|
#include "lib/utils/bts_chrono.h"
|
|
#include "server/sparkd/cmds/spartan/md/channel_observers/DefinitionCollector.h"
|
|
#include "server/sparkd/cmds/spartan/md/channel_observers/IcebergFinder.h"
|
|
#include "server/sparkd/cmds/spartan/md/channel_observers/DefinitionCollector.h"
|
|
#include "server/sparkd/cmds/spartan/md/channel_observers/PolygonFinder.h"
|
|
#include "server/sparkd/cmds/spartan/md/channel_observers/PolygonOpportunityFinder.h"
|
|
|
|
#include "lib/utils/TempFile.h"
|
|
#include "lib/utils/overloaded.h"
|
|
|
|
#include <Pcap++/PcapFileDevice.h>
|
|
#include <boost/algorithm/string/predicate.hpp>
|
|
#include <boost/date_time/posix_time/conversion.hpp>
|
|
#include <boost/date_time/posix_time/ptime.hpp>
|
|
#include <boost/filesystem.hpp>
|
|
#include <boost/process.hpp>
|
|
#include <boost/process/start_dir.hpp>
|
|
#include <fmt/format.h>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <nlohmann/json.hpp>
|
|
#include <sys/stat.h>
|
|
#include <sys/types.h>
|
|
|
|
namespace bts::spartan::mdp3 {
|
|
|
|
std::string drop_extensions(std::string const &str) {
|
|
size_t lastindex = str.find_last_of(".");
|
|
auto result = str.substr(0, lastindex);
|
|
if (result != str)
|
|
return drop_extensions(result);
|
|
else
|
|
return result;
|
|
}
|
|
|
|
struct Guesser {
|
|
FeedMap result;
|
|
PCapReader::PacketInfo current_packet;
|
|
|
|
void on_begin_packet(MDP3PacketView const &) {}
|
|
void on_end_packet() {}
|
|
|
|
void set_current_source(const PCapReader::PacketInfo &info) {
|
|
current_packet = info;
|
|
}
|
|
|
|
template <class M> void operator()(M const &) {
|
|
if (result.incremental[0].port == 0) {
|
|
result.incremental[0].port = current_packet.dst_port;
|
|
result.incremental[0].ip = current_packet.dst_ip;
|
|
return;
|
|
}
|
|
if (result.incremental[0].port == current_packet.dst_port &&
|
|
result.incremental[0].ip == current_packet.dst_ip)
|
|
return;
|
|
|
|
if (result.incremental[1].port == 0) {
|
|
result.incremental[1].port = current_packet.dst_port;
|
|
result.incremental[1].ip = current_packet.dst_ip;
|
|
}
|
|
}
|
|
|
|
void operator()(::mdp3::SnapshotFullRefresh52 const &) {
|
|
if (result.snapshot.port == 0) {
|
|
result.snapshot.port = current_packet.dst_port;
|
|
result.snapshot.ip = current_packet.dst_ip;
|
|
}
|
|
}
|
|
|
|
void operator()(::mdp3::SnapshotFullRefreshOrderBook53 const &) {
|
|
if (result.snapshot.port == 0) {
|
|
result.snapshot.port = current_packet.dst_port;
|
|
result.snapshot.ip = current_packet.dst_ip;
|
|
}
|
|
}
|
|
|
|
void operator()(::mdp3::MDInstrumentDefinitionFuture54 const &) {
|
|
on_definition();
|
|
}
|
|
|
|
void operator()(::mdp3::MDInstrumentDefinitionOption55 const &) {
|
|
on_definition();
|
|
}
|
|
|
|
void operator()(::mdp3::MDInstrumentDefinitionSpread56 const &) {
|
|
on_definition();
|
|
}
|
|
|
|
void on_definition() {
|
|
if (result.instrument.port)
|
|
return;
|
|
if (result.incremental[0].port == 0)
|
|
return;
|
|
if (result.incremental[1].port == 0)
|
|
return;
|
|
if (result.incremental[0].ip == current_packet.dst_ip &&
|
|
result.incremental[0].port == current_packet.dst_port)
|
|
return;
|
|
if (result.incremental[1].ip == current_packet.dst_ip &&
|
|
result.incremental[1].port == current_packet.dst_port)
|
|
return;
|
|
result.instrument.port = current_packet.dst_port;
|
|
result.instrument.ip = current_packet.dst_ip;
|
|
}
|
|
};
|
|
|
|
FeedMap guess_feed_map(std::string const &pcap_path) {
|
|
Guesser guesser;
|
|
auto reader = PCapReader{pcap_path};
|
|
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
|
guesser.set_current_source(info);
|
|
parse_packet(MDP3PacketView{payload}, guesser);
|
|
if (guesser.result)
|
|
return false;
|
|
else
|
|
return true;
|
|
};
|
|
|
|
while (reader.read_next())
|
|
;
|
|
return guesser.result;
|
|
}
|
|
|
|
std::ostream &operator<<(std::ostream &o, FeedMap const &fm) {
|
|
o << "IA:" << fm.incremental[0].ip << ":" << fm.incremental[0].port
|
|
<< " IB:" << fm.incremental[1].ip << ":" << fm.incremental[1].port
|
|
<< " N:" << fm.instrument.ip << ":" << fm.instrument.port
|
|
<< " SMBO:" << fm.snapshot.ip << ":" << fm.snapshot.port;
|
|
return o;
|
|
}
|
|
|
|
int compress(std::string const &file, std::string const &out_file) {
|
|
using namespace bts::spartan;
|
|
using namespace bts::spartan::mdp3;
|
|
|
|
struct Ctrl : IFeedController {
|
|
bool instrument_feed = false;
|
|
bool snapshot_feed = false;
|
|
bool incremental_feed = false;
|
|
|
|
void start_instrument_feed() override { instrument_feed = true; }
|
|
void stop_instrument_feed() override { instrument_feed = false; }
|
|
void start_snapshot_feed() override { snapshot_feed = true; }
|
|
void stop_snapshot_feed() override { snapshot_feed = false; }
|
|
void start_incremental_feed() override { incremental_feed = true; }
|
|
void stop_incremental_feed() override { incremental_feed = false; }
|
|
} ctrl;
|
|
|
|
struct Obs : IChannelObserver {
|
|
bool observed_loss = false;
|
|
bool recovery_complete = false;
|
|
|
|
size_t previous_packet = 0;
|
|
size_t previous_send_ts = 0;
|
|
bts::TimeStamp last_print;
|
|
|
|
void on_recovery_complete(
|
|
std::vector<InstrumentSnapshot const *> const &) override {
|
|
recovery_complete = true;
|
|
observed_loss = false;
|
|
}
|
|
|
|
void on_event_end(MDP3Event const &ev) override {
|
|
if (ms_diff(Timer::now(), last_print) >= 1000) {
|
|
last_print = Timer::now();
|
|
LOG(Spartan, Info, "Time: {} -> {} pkt/sec ({}x real time)",
|
|
ev.transact_time.to_time_string(),
|
|
(ev.first_packet_sn - previous_packet),
|
|
(ev.first_packet_send_time.value() - previous_send_ts) /
|
|
1000000000);
|
|
previous_packet = ev.first_packet_sn;
|
|
previous_send_ts = ev.first_packet_send_time.value();
|
|
}
|
|
}
|
|
void on_packet_loss() override {
|
|
observed_loss = true;
|
|
recovery_complete = false;
|
|
}
|
|
} obs;
|
|
|
|
struct IncrementalFeedBuffer {
|
|
bool written_once = false;
|
|
pcpp::RawPacket rp;
|
|
mdp3::MDP3Packet p;
|
|
};
|
|
std::array<IncrementalFeedBuffer, 2> inc_buffer;
|
|
|
|
Guesser guesser;
|
|
auto reader = PCapReader{file};
|
|
pcpp::PcapNgFileWriterDevice writer(out_file.c_str());
|
|
writer.open();
|
|
|
|
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
|
guesser.set_current_source(info);
|
|
parse_packet(MDP3PacketView{payload}, guesser);
|
|
writer.writePacket(*info.rp);
|
|
if (guesser.result)
|
|
return false;
|
|
else
|
|
return true;
|
|
};
|
|
|
|
// Run through the file long enough to be able to map feeds to IP's
|
|
while (reader.read_next())
|
|
;
|
|
|
|
auto feed_map = guesser.result;
|
|
if (!feed_map) {
|
|
LOG(Spartan, Fatal, "Could not guess feed ip/ports in {}. Aborting",
|
|
file);
|
|
return -1;
|
|
}
|
|
|
|
// Now continue reading, feeding each packet to our channel handler
|
|
mdp3::AB_Arbitrageur ab;
|
|
mdp3::ChannelHandler handler(obs, ctrl);
|
|
|
|
// For every packet extractred from the pcap decide if we need to
|
|
// write it out to our output or not.
|
|
reader.on_packet() = [&](auto const &info, auto const &payload) {
|
|
if ((info.dst_ip == feed_map.incremental[0].ip &&
|
|
info.dst_port == feed_map.incremental[0].port)) {
|
|
inc_buffer[0].rp = *info.rp;
|
|
inc_buffer[0].p.set_bytes(payload);
|
|
} else if ((info.dst_ip == feed_map.incremental[1].ip &&
|
|
info.dst_port == feed_map.incremental[1].port)) {
|
|
inc_buffer[1].rp = *info.rp;
|
|
inc_buffer[1].p.set_bytes(payload);
|
|
} else if (
|
|
info.dst_ip == feed_map.instrument.ip &&
|
|
info.dst_port == feed_map.instrument.port) {
|
|
if (ctrl.instrument_feed) {
|
|
handler.on_instrument_feed(mdp3::MDP3PacketView{payload});
|
|
writer.writePacket(*info.rp);
|
|
}
|
|
return true;
|
|
} else if (
|
|
info.dst_ip == feed_map.snapshot.ip &&
|
|
info.dst_port == feed_map.snapshot.port) {
|
|
if (ctrl.snapshot_feed) {
|
|
handler.on_snapshot_feed(mdp3::MDP3PacketView{payload});
|
|
writer.writePacket(*info.rp);
|
|
}
|
|
return true;
|
|
} else
|
|
return true;
|
|
|
|
// If we reached this point then we are dealing with an
|
|
// incremental feed packet.
|
|
auto picked = handler.on_incremental_feed(
|
|
inc_buffer[0].p.view(), inc_buffer[1].p.view());
|
|
if (picked == PickedFeed::A ||
|
|
((!inc_buffer[0].written_once || !obs.recovery_complete) &&
|
|
!inc_buffer[0].p.view().empty())) {
|
|
inc_buffer[0].written_once = true;
|
|
writer.writePacket(inc_buffer[0].rp);
|
|
}
|
|
|
|
if (picked == PickedFeed::B ||
|
|
((!inc_buffer[1].written_once || !obs.recovery_complete) &&
|
|
!inc_buffer[1].p.view().empty())) {
|
|
inc_buffer[1].written_once = true;
|
|
writer.writePacket(inc_buffer[1].rp);
|
|
}
|
|
|
|
return true;
|
|
};
|
|
|
|
while (reader.read_next())
|
|
if (obs.observed_loss) {
|
|
LOG(Spartan, Error, "Observed loss in {}. Recovery.", file);
|
|
};
|
|
return 0;
|
|
}
|
|
|
|
struct JSON_Writer {
|
|
size_t first_packet = 0;
|
|
size_t packet_count = 0;
|
|
std::stringstream json_buffer;
|
|
nlohmann::json json_obj;
|
|
mdp3::MDP3PacketView packet;
|
|
|
|
JSON_Writer(size_t first_packet = 0, size_t packet_count = 0)
|
|
: first_packet(first_packet), packet_count(packet_count) {}
|
|
|
|
void on_begin_packet(mdp3::MDP3PacketView const &p) { packet = p; }
|
|
void on_end_packet() {}
|
|
|
|
template <class M> void operator()(M const &m) {
|
|
if (first_packet == 0 ||
|
|
(packet.seq_num() >= first_packet &&
|
|
packet.seq_num() <= first_packet + packet_count)) {
|
|
json_obj["SeqNum"] = packet.seq_num();
|
|
json_obj["SendTime"] =
|
|
TimeStamp{packet.sending_time()}.to_datetime_string();
|
|
json_obj["Size"] = packet.size();
|
|
json_buffer.str("");
|
|
json_buffer.clear();
|
|
json_buffer << m;
|
|
json_obj["Msg"] = nlohmann::json::parse(json_buffer.str());
|
|
std::cout << json_obj.dump(4) << std::endl;
|
|
}
|
|
}
|
|
};
|
|
|
|
struct PacketBuffers {
|
|
mdp3::MDP3Packet i;
|
|
mdp3::MDP3Packet n;
|
|
};
|
|
|
|
int to_json(std::string const &pcap, size_t first_packet, size_t count) {
|
|
auto reader = PCapReader{pcap};
|
|
JSON_Writer writer(first_packet, count);
|
|
reader.on_packet() = [&](auto const &, auto const &payload) {
|
|
parse_packet(mdp3::MDP3PacketView{payload}, writer);
|
|
return true;
|
|
};
|
|
while (reader.read_next())
|
|
;
|
|
return 0;
|
|
}
|
|
|
|
struct ObsWrapper : mdp3::IChannelObserver {
|
|
ObsWrapper(mdp3::IChannelObserver &obs) : real_obs(obs) {}
|
|
IChannelObserver &real_obs;
|
|
size_t previous_packet = 0;
|
|
size_t previous_send_ts = 0;
|
|
bts::TimeStamp last_print = Timer::now();
|
|
|
|
void init() override { real_obs.init(); }
|
|
|
|
void finit() override { real_obs.finit(); }
|
|
|
|
void on_recovery_complete(
|
|
std::vector<mdp3::InstrumentSnapshot const *> const &s) override {
|
|
real_obs.on_recovery_complete(s);
|
|
}
|
|
void on_packet_begin(MDP3PacketView const &pv) override {
|
|
real_obs.on_packet_begin(pv);
|
|
}
|
|
void on_event_end(mdp3::MDP3Event const &ev) override {
|
|
auto period_ms = 10000;
|
|
if (ms_diff(Timer::now(), last_print) >= period_ms) {
|
|
last_print = Timer::now();
|
|
std::stringstream str2;
|
|
str2 << "Packet time: " << ev.transact_time.to_time_string()
|
|
<< " Event ID: " << ev.event_id << " ("
|
|
<< (ev.first_packet_sn - previous_packet) / (period_ms / 1000)
|
|
<< " pkt/sec, "
|
|
<< (ev.first_packet_send_time.value() - previous_send_ts) /
|
|
1000000000 / (period_ms / 1000)
|
|
<< "x speedup)";
|
|
LOG(Spartan, Info, "{}", str2.str());
|
|
previous_packet = ev.first_packet_sn;
|
|
previous_send_ts = ev.first_packet_send_time.value();
|
|
}
|
|
real_obs.on_event_end(ev);
|
|
}
|
|
void on_packet_loss() override { bts_assert(false); }
|
|
};
|
|
|
|
struct Ctrl : mdp3::IFeedController {
|
|
void start_instrument_feed() override{};
|
|
void stop_instrument_feed() override{};
|
|
void start_snapshot_feed() override{};
|
|
void stop_snapshot_feed() override{};
|
|
void start_incremental_feed() override{};
|
|
void stop_incremental_feed() override{};
|
|
};
|
|
|
|
struct PCapStreamer::PImpl {
|
|
PImpl(std::string const &file, mdp3::IChannelObserver &obs)
|
|
: obs_wrapper(obs), ctrl(), reader(file), handler(obs_wrapper, ctrl) {}
|
|
|
|
ObsWrapper obs_wrapper;
|
|
Ctrl ctrl;
|
|
PCapReader reader;
|
|
mdp3::ChannelHandler handler;
|
|
std::array<PacketBuffers, 2> buffers;
|
|
};
|
|
|
|
PCapStreamer::PCapStreamer(std::string const &file, mdp3::IChannelObserver &obs)
|
|
: me(new PImpl(file, obs)) {
|
|
Guesser guesser;
|
|
me->reader.on_packet() = [&](auto const &info, auto const &payload) {
|
|
guesser.set_current_source(info);
|
|
parse_packet(MDP3PacketView{payload}, guesser);
|
|
if (guesser.result)
|
|
return false;
|
|
else
|
|
return true;
|
|
};
|
|
|
|
while (me->reader.read_next())
|
|
;
|
|
|
|
auto feed_map = guesser.result;
|
|
if (!feed_map) {
|
|
LOG(Spartan, Fatal, "Could not guess feed ip/ports in {}. Aborting",
|
|
file);
|
|
bts_assert(false);
|
|
}
|
|
me->obs_wrapper.init();
|
|
me->reader.on_packet() = [this,
|
|
feed_map](auto const &info, auto const &payload) {
|
|
if ((info.dst_ip == feed_map.incremental[0].ip &&
|
|
info.dst_port == feed_map.incremental[0].port)) {
|
|
me->buffers[0].i.set_bytes(payload);
|
|
me->handler.on_incremental_feed(
|
|
me->buffers[0].i.view(), me->buffers[1].i.view());
|
|
} else if ((info.dst_ip == feed_map.incremental[1].ip &&
|
|
info.dst_port == feed_map.incremental[1].port)) {
|
|
me->buffers[1].i.set_bytes(payload);
|
|
me->handler.on_incremental_feed(
|
|
me->buffers[0].i.view(), me->buffers[1].i.view());
|
|
} else if ((info.dst_ip == feed_map.instrument.ip &&
|
|
info.dst_port == feed_map.instrument.port)) {
|
|
me->handler.on_instrument_feed(mdp3::MDP3PacketView{payload});
|
|
} else if ((info.dst_ip == feed_map.snapshot.ip &&
|
|
info.dst_port == feed_map.snapshot.port)) {
|
|
me->handler.on_snapshot_feed(mdp3::MDP3PacketView{payload});
|
|
}
|
|
return true;
|
|
};
|
|
}
|
|
|
|
PCapStreamer::~PCapStreamer() { me->obs_wrapper.finit(); }
|
|
|
|
bool PCapStreamer::next() { return me->reader.read_next(); }
|
|
|
|
void observe_channel(std::string const &file, mdp3::IChannelObserver &obs) {
|
|
PCapStreamer streamer(file, obs);
|
|
while (streamer.next())
|
|
;
|
|
}
|
|
|
|
void observe_channel(
|
|
std::string const &pcap,
|
|
std::initializer_list<IChannelObserver *> observers) {
|
|
|
|
struct Composite : IChannelObserver {
|
|
std::initializer_list<IChannelObserver *> const &obs;
|
|
Composite(std::initializer_list<IChannelObserver *> const &obs)
|
|
: obs(obs) {}
|
|
|
|
void init() override {
|
|
for (auto o : obs)
|
|
o->init();
|
|
}
|
|
void finit() override {
|
|
for (auto o : obs)
|
|
o->finit();
|
|
}
|
|
void on_recovery_complete(InitialSnapshot const &s) override {
|
|
for (auto o : obs)
|
|
o->on_recovery_complete(s);
|
|
}
|
|
void on_packet_begin(MDP3PacketView const &pv) override {
|
|
for (auto o : obs)
|
|
o->on_packet_begin(pv);
|
|
}
|
|
void on_event_end(MDP3Event const &ev) override {
|
|
for (auto o : obs)
|
|
o->on_event_end(ev);
|
|
}
|
|
} comp(observers);
|
|
observe_channel(pcap, comp);
|
|
}
|
|
|
|
struct CSVObs : IChannelObserver {
|
|
|
|
CSVObs(std::string const &pcap)
|
|
: write_books(config::get<config::MarketDataDownload>().write_books()),
|
|
events(drop_extensions(pcap) + ".events.csv"),
|
|
event_items(drop_extensions(pcap) + ".event_items.csv"),
|
|
securities(drop_extensions(pcap) + ".securities.csv"),
|
|
books(drop_extensions(pcap) + ".books.csv"),
|
|
packets(drop_extensions(pcap) + ".packets.csv"),
|
|
spread_legs(drop_extensions(pcap) + ".spread_legs.csv") {}
|
|
const bool write_books;
|
|
std::ofstream events;
|
|
std::ofstream event_items;
|
|
std::ofstream orders;
|
|
std::ofstream securities;
|
|
std::ofstream books;
|
|
std::ofstream packets;
|
|
std::ofstream spread_legs;
|
|
std::unordered_set<SecurityID> known_securities;
|
|
std::stringstream ts_buffer;
|
|
|
|
template <class B> void print_book(B const &b) {
|
|
char buffer[128];
|
|
for (auto level : b) {
|
|
d_toa(level.price.to_double(), buffer, sizeof(buffer), 0);
|
|
books << "," << buffer << "," << level.quantity.value() << ","
|
|
<< level.n_orders;
|
|
}
|
|
}
|
|
|
|
void on_recovery_complete(
|
|
std::vector<InstrumentSnapshot const *> const &secs) override {
|
|
for (auto s : secs) {
|
|
securities << s->security_id().value() << "," << s->symbol() << ","
|
|
<< api::SecurityType_Name(s->definition().security_type())
|
|
<< "," << s->definition().cme_security_type() << ","
|
|
<< s->definition().security_group() << ","
|
|
<< s->definition().asset() << ","
|
|
<< s->definition().implied_matching() << ","
|
|
<< s->definition().match_algorithm() << ","
|
|
<< s->definition().display_factor() << ","
|
|
<< s->definition().cash_per_point() << ","
|
|
<< s->definition().min_price_increment() << ","
|
|
<< s->definition().min_price_increment_ammount() << ","
|
|
<< s->definition().legs().size() << ","
|
|
<< s->last_update_time() << std::endl;
|
|
|
|
for (auto const &l : s->definition().legs())
|
|
spread_legs << s->security_id().value() << ","
|
|
<< l.security_id.value() << ","
|
|
<< api::Side_Name(l.side) << "," << l.ratio.value()
|
|
<< std::endl;
|
|
|
|
known_securities.insert(s->security_id());
|
|
|
|
if(write_books){
|
|
books << 0 << "," // event_id
|
|
<< 0 << "," // transact time
|
|
<< s->security_id().value() << "," // security id
|
|
<< s->trading_status() << "," // trading status
|
|
<< s->implied_matching_status(); // implied matching status
|
|
print_book(s->book(api::SIDE_BUY));
|
|
print_book(s->book(api::SIDE_SELL));
|
|
print_book(s->implied_book(api::SIDE_BUY));
|
|
print_book(s->implied_book(api::SIDE_SELL));
|
|
books << std::endl;
|
|
}
|
|
|
|
for (auto const &[id, order] : s->orders())
|
|
event_items << 0 << "," // event_id
|
|
<< 0 << "," // transact time
|
|
<< s->security_id().value() << "," // security id
|
|
<< "," // human transact time
|
|
<< s->symbol() << "," // symbol
|
|
<< s->trading_status() << "," // trading status
|
|
<< s->implied_matching_status()
|
|
<< "," // implied matching status
|
|
<< "order_change," // type
|
|
<< "init," // change
|
|
<< id << "," // order key
|
|
<< api::Side_Name(order.side) << "," // side
|
|
<< order.priority.value() << "," // priority
|
|
<< order.price.to_double() << "," // price
|
|
<< order.quantity.value() << "," // quantity
|
|
<< "0," // trade_entry_id
|
|
<< "n/a" // aggressor
|
|
<< std::endl;
|
|
}
|
|
}
|
|
|
|
void on_packet_begin(MDP3PacketView const &pv) override {
|
|
packets << pv.sending_time() << ","
|
|
<< TimeStamp(pv.sending_time()).to_datetime_string() << ","
|
|
<< pv.seq_num() << "," << pv.size() << std::endl;
|
|
}
|
|
|
|
void on_event_end(MDP3Event const &ev) override {
|
|
events << ev.transact_time.to_datetime_string()
|
|
<< "," // human transact time
|
|
<< ev.event_id << "," // event id
|
|
<< ev.transact_time << "," // transact time
|
|
<< ev.first_packet_sn << "," // first packet_sn
|
|
<< ev.last_packet_sn << "," // last_packet_sn
|
|
<< ev.first_packet_send_time << "," // first packet send time
|
|
<< ev.last_packet_send_time // last packet send time
|
|
<< std::endl;
|
|
|
|
for (auto s : ev.affected_securities()) {
|
|
|
|
if (known_securities.count(s->security_id()) == 0) {
|
|
known_securities.insert(s->security_id());
|
|
securities << s->security_id().value() << "," << s->symbol() << ","
|
|
<< api::SecurityType_Name(
|
|
s->definition().security_type())
|
|
<< "," << s->definition().cme_security_type() << ","
|
|
<< s->definition().security_group() << ","
|
|
<< s->definition().asset() << ","
|
|
<< s->definition().implied_matching() << ","
|
|
<< s->definition().match_algorithm() << ","
|
|
<< s->definition().display_factor() << ","
|
|
<< s->definition().cash_per_point() << ","
|
|
<< s->definition().min_price_increment() << ","
|
|
<< s->definition().min_price_increment_ammount() << ","
|
|
<< s->definition().legs().size() << ","
|
|
<< s->last_update_time() // last update time
|
|
<< std::endl;
|
|
}
|
|
|
|
if(write_books){
|
|
books << ev.event_id << "," // event id
|
|
<< ev.transact_time.value() << "," // transact time
|
|
<< s->security_id().value() << "," // security id
|
|
<< s->trading_status() << "," // trading status
|
|
<< s->implied_matching_status(); // implied matching status
|
|
print_book(s->book(api::SIDE_BUY));
|
|
print_book(s->book(api::SIDE_SELL));
|
|
print_book(s->implied_book(api::SIDE_BUY));
|
|
print_book(s->implied_book(api::SIDE_SELL));
|
|
books << std::endl;
|
|
}
|
|
|
|
for (auto const &change : s->order_changes())
|
|
event_items << ev.event_id << "," // event id
|
|
<< ev.transact_time.value() // transact time
|
|
<< "," // transact time
|
|
<< s->security_id().value() << "," // security id
|
|
<< ts_buffer.str() << "," // human transact time
|
|
<< s->symbol() << "," // symbol
|
|
<< s->trading_status() << "," // trading status
|
|
<< s->implied_matching_status()
|
|
<< "," // implied matching status
|
|
<< "order_change," // type
|
|
<< change_name(change.change) << "," // change type
|
|
<< change.order.key.value() << "," // order key
|
|
<< api::Side_Name(change.order.side) << "," // side
|
|
<< change.order.priority.value() << "," // priority
|
|
<< change.order.price.to_double() << "," // price
|
|
<< change.order.quantity.value() << "," // quantity
|
|
<< "0," // trade_entry_id
|
|
<< "n/a" // aggressor
|
|
<< std::endl;
|
|
|
|
for (auto const &f : s->fills())
|
|
event_items << ev.event_id << "," // event id
|
|
<< ev.transact_time.value() << "," // transact time
|
|
<< s->security_id().value() << "," // security id
|
|
<< ts_buffer.str() << "," // human transact time
|
|
<< s->symbol() << "," // symbol
|
|
<< s->trading_status() << "," // trading status
|
|
<< s->implied_matching_status()
|
|
<< "," // implied matching status
|
|
<< "fill," // type
|
|
<< "," // null
|
|
<< f.key.value() << "," // order key
|
|
<< "," // side
|
|
<< "0," // priority
|
|
<< f.price << "," // price
|
|
<< f.quantity << "," // quantity
|
|
<< f.trade_entry_id << "," // trade_entry_id
|
|
<< aggressor_name(f.aggressor) // aggressor
|
|
<< std::endl;
|
|
|
|
auto deduced_orders = s->deduced_order_changes();
|
|
for (auto &deduced_order : deduced_orders)
|
|
event_items << ev.event_id << "," // event id
|
|
<< ev.transact_time.value() << "," // transact time
|
|
<< s->security_id().value() << "," // security id
|
|
<< ts_buffer.str() << "," // human transact time
|
|
<< s->symbol() << "," // symbol
|
|
<< s->trading_status() << "," // trading status
|
|
<< s->implied_matching_status()
|
|
<< "," // implied matching status
|
|
<< "deduced_order," // type
|
|
<< change_name(deduced_order.change) << ","
|
|
<< deduced_order.order.key.value() << "," // order key
|
|
<< api::Side_Name(deduced_order.order.side)
|
|
<< "," // side
|
|
<< "0," // priority
|
|
<< deduced_order.order.price.to_double() << "," // price
|
|
<< deduced_order.order.quantity.value()
|
|
<< "," // quantity
|
|
<< "0," // trade_entry_id
|
|
<< "n/a" // aggressor
|
|
<< std::endl;
|
|
}
|
|
}
|
|
|
|
void on_packet_loss() override {
|
|
LOG(Spartan, Error, "Detected loss in feed. Aborting");
|
|
bts_assert(false);
|
|
}
|
|
}; // namespace bts::spartan::mdp3
|
|
|
|
int to_csv(std::string const &file) {
|
|
using namespace bts::spartan::mdp3;
|
|
CSVObs cvs{file};
|
|
IcebergFinder ice_finder;
|
|
PolygonFinder poly_finder;
|
|
DefinitionCollector def_collector;
|
|
|
|
observe_channel(file, {&cvs, &ice_finder, &poly_finder, &def_collector});
|
|
auto icebergs = std::ofstream(drop_extensions(file) + ".icebergs.csv");
|
|
for (auto &i : ice_finder.result) {
|
|
auto relevent_eids = i.second.all_eids();
|
|
for (auto eid : relevent_eids)
|
|
icebergs << i.first.value() << "," << eid << ","
|
|
<< i.second.total_qty().value() << ","
|
|
<< std::get<0>(i.second.show_qty_at(eid)) << ","
|
|
<< i.second.count_fills_by(eid) << std::endl;
|
|
}
|
|
|
|
auto polys = std::ofstream(drop_extensions(file) + ".polygons.csv");
|
|
auto poly_legs = std::ofstream(drop_extensions(file) + ".polygon_legs.csv");
|
|
auto &imap = def_collector.ptr_result;
|
|
for (auto &[pid, p] : poly_finder.result) {
|
|
polys << pid << "," << polygon_descriptor(p, imap) << std::endl;
|
|
for (auto &part : p.parts) {
|
|
poly_legs << pid << "," << part.sec_id << ","
|
|
<< (part.ratio > 0 ? api::Side_Name(api::SIDE_BUY)
|
|
: api::Side_Name(api::SIDE_SELL))
|
|
<< "," << std::abs(part.ratio) << std::endl;
|
|
}
|
|
}
|
|
|
|
PolygonOpportunityFinder op_finder{
|
|
poly_finder.result, def_collector.ptr_result};
|
|
observe_channel(file, {&op_finder});
|
|
|
|
auto poly_values =
|
|
std::ofstream(drop_extensions(file) + ".polygon_values.csv");
|
|
poly_values_to_csv(
|
|
op_finder.result, poly_finder.result, def_collector.ptr_result,
|
|
poly_values);
|
|
|
|
auto poly_opportunities =
|
|
std::ofstream(drop_extensions(file) + ".polygon_opportunities.csv");
|
|
poly_opportunities_to_csv(
|
|
aggregate(op_finder.result), poly_finder.result,
|
|
def_collector.ptr_result, poly_opportunities);
|
|
return 0;
|
|
}
|
|
|
|
std::unordered_map<OrderKey, IcebergInput>
|
|
find_icebergs(std::string const &pcap) {
|
|
return IcebergFinder::find_all(pcap);
|
|
}
|
|
|
|
int to_sql(std::string const &pcap) {
|
|
if (to_csv(pcap) == 0) {
|
|
auto init_script = os::sparkd_wd() + "/data_files/spartan/md.sql";
|
|
auto db_file = drop_extensions(pcap) + ".sqlite";
|
|
if (boost::filesystem::exists(db_file))
|
|
boost::filesystem::remove(db_file);
|
|
auto out_dir = boost::filesystem::path(db_file).parent_path().string();
|
|
|
|
std::cout << "Creating database " << db_file << std::endl;
|
|
auto result = boost::process::system(
|
|
fmt::format("sqlite3 {}", db_file),
|
|
boost::process::std_in < init_script,
|
|
boost::process::start_dir = out_dir);
|
|
if (result) {
|
|
std::cerr << "Error creating database" << std::endl;
|
|
return result;
|
|
}
|
|
|
|
std::initializer_list<std::string> tables = {"event_items",
|
|
"events",
|
|
"securities",
|
|
"books",
|
|
"icebergs",
|
|
"spread_legs",
|
|
"polygons",
|
|
"polygon_legs",
|
|
"packets",
|
|
"polygon_values",
|
|
"polygon_opportunities"};
|
|
|
|
std::string import = ".mode csv\n";
|
|
for (auto const &t : tables)
|
|
import += fmt::format(
|
|
".import {0}.{1}.csv {1}\n", drop_extensions(pcap), t, t);
|
|
|
|
utils::TempFile f;
|
|
std::ofstream{f.path()} << import;
|
|
std::cout << "Importing data CSVs" << std::endl;
|
|
result = boost::process::system(
|
|
fmt::format("sqlite3 {}", db_file), boost::process::std_in < f.path(),
|
|
boost::process::start_dir = out_dir);
|
|
|
|
if (result) {
|
|
std::cerr << "Error importing CSVs" << std::endl;
|
|
return result;
|
|
} else
|
|
for (auto const &t : tables)
|
|
boost::filesystem::remove(
|
|
fmt::format("{0}.{1}.csv", drop_extensions(pcap), t));
|
|
|
|
return 0;
|
|
} else
|
|
return -1;
|
|
}
|
|
|
|
} // namespace bts::spartan::mdp3
|