Files
spark/src/lib/md/relay/RelayRecorder.cpp
T

336 lines
9.7 KiB
C++

#ifndef NO_CPP0X
#include "RelayRecorder.h"
#include "lib/config/config.h"
#include "lib/config/protos/market_data.pb.h"
#include "lib/config/protos/system_config.pb.h"
#include "lib/logging/logging.h"
#include "lib/md/SecurityIDRecorder.h"
#include "lib/md/protos/co2.pb.h"
#include "lib/utils/HTTPStream.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/datetime.h"
#include <boost/filesystem.hpp>
#include <google/protobuf/text_format.h>
#include <memory>
#include <thread>
using namespace std;
using namespace bts::co2;
namespace bts {
namespace md {
const char *magic = "Co2";
const uint32_t recorder_version = 3;
template <class T> void writebuf(ofstream &out, const T &pb) {
auto size = static_cast<std::uint32_t>(pb.ByteSizeLong());
out.write(reinterpret_cast<char *>(&size), sizeof(size));
bts_assert(pb.SerializeToOstream(&out));
}
template <class T> bool readbuf(istream &in, T *result) {
if (result) {
uint32_t size;
in.read(reinterpret_cast<char *>(&size), sizeof(size));
vector<char> buffer(size);
in.read(&buffer[0], size);
if (!in.fail()) {
result->ParsePartialFromArray(&buffer[0], size);
return true;
}
}
return false;
}
RelayRecorder::RelayRecorder(
const std::string &base_path, const std::string &filename, uint32_t naggle)
: _filename(base_path + "/" + filename), _file() {
LOG(md, Info, "Recording market data to: {}", _filename);
boost::system::error_code ec;
boost::filesystem::create_directories(base_path, ec);
_file.open(_filename, ios::out | ios::binary | ios::trunc);
bts_assert(_file.is_open());
CoconutReplayHeader header;
header.set_magic(magic);
header.set_version(recorder_version);
header.set_timestamp(datetime::ll_now());
header.set_host(config::get<config::SystemConfig>().config_host());
header.set_naggle_ms(naggle);
writebuf(_file, header);
}
RelayRecorder::~RelayRecorder() {}
void RelayRecorder::write(const co2::MDReply &data, bool *) {
writebuf(_file, data);
flush();
}
void RelayRecorder::flush() { _file.flush(); }
struct ConsoleReplayWriter : IRelayWriter {
ConsoleReplayWriter() : count(0) {}
int count;
virtual void write(const co2::MDReply &data, bool *) override {
++count;
cout << count << " MDReply {" << endl;
data.PrintDebugString();
cout << "}" << endl;
}
};
struct ConsoleTraceWriter : IRelayWriter {
int count;
security_key key;
ConsoleTraceWriter(security_key k) : count(0), key(k) {}
virtual void write(const co2::MDReply &data, bool *) override {
++count;
for (const auto &chan : data.channels()) {
for (const auto &update : chan.updates()) {
if (update.key() == key) {
cout << count << " Update {" << endl;
update.PrintDebugString();
cout << "}" << endl;
}
}
}
}
};
#ifndef _WIN32
void record_security_keys(
md::SecurityIDRecorder &idrec, const co2::MDReply &reply) {
std::vector<const Security *> data;
for (const auto &chan : reply.channels()) {
for (const auto &upd : chan.updates()) {
if (upd.has_security()) {
data.push_back(&upd.security());
}
}
}
idrec.bulk_insert(data, "REPLAY: ");
}
#endif
void replay_recording(istream &in, IRelayWriter &writer, int32_t naggle_ms) {
bts_assert(!in.bad());
CoconutReplayHeader header;
size_t count = 0;
#ifndef _WIN32
md::SecurityIDRecorder idrec{{}};
#endif
if (readbuf(in, &header) && header.magic() == magic &&
header.version() == recorder_version) {
co2::MDReply message;
while (readbuf(in, &message)) {
bool has_securities = false;
writer.write(message, &has_securities);
if (has_securities) {
// record security ids to avoid breaking simulator
#ifndef _WIN32
record_security_keys(idrec, message);
#endif
}
message.Clear();
if (naggle_ms > 0)
std::this_thread::sleep_for(
std::chrono::duration<int, std::milli>(naggle_ms));
else if (naggle_ms == -1 /*interactive*/) {
++count;
cout << count << endl;
cin.get();
}
}
} else {
cout << "Invalid file format." << endl;
}
}
void replay_file(
const std::string &filename,
IRelayWriter &writer,
int32_t naggle_ms,
bool loop) {
do {
ifstream in;
in.open(filename, ios::binary | ios::in);
if (in.is_open()) {
replay_recording(in, writer, naggle_ms);
} else {
LOG(md, Diag, "Cannot read file {}", filename);
if (loop)
sleep_for(10s);
}
} while (loop);
}
bool replay_url(
const std::string &url,
IRelayWriter &writer,
int32_t naggle_ms,
bool loop) {
// split up url parts
size_t http_len = BTS_ARRAYSIZE("http://") - 1;
auto after_host = url.find('/', http_len);
auto host = url.substr(http_len, after_host - http_len);
auto path = url.substr(after_host);
auto last_slash = url.rfind('/');
const auto filename =
os::app_data_path("data/replay/") / url.substr(last_slash + 1);
utils::download_file_if_newer(host, path, filename.string());
if (boost::filesystem::exists(filename)) {
replay_file(filename.string(), writer, naggle_ms, loop);
return true;
}
return false;
}
void replay_url_with_fallback(
const std::string &url,
IRelayWriter &writer,
int32_t naggle_ms,
bool loop) {
if (!replay_url(url, writer, naggle_ms, loop)) {
const auto fallback = config::get<ReplayOptions>().fallback();
if (boost::filesystem::exists(fallback)) {
replay_file(fallback, writer, naggle_ms, loop);
}
}
}
struct SecurityExtractorDone : public std::bad_exception {};
struct SecurityExtractor : public IRelayWriter {
size_t _count;
std::function<void(const api::Security &)> _cb;
std::set<uint32_t> _keys;
SecurityExtractor(
std::function<void(const api::Security &)> &cb, size_t count)
: _count(count), _cb(cb), _keys() {}
~SecurityExtractor() override{};
void write(const co2::MDReply &data, bool *) override {
for (auto const &ch : data.channels()) {
for (auto const &cr : ch.updates()) {
if (cr.has_security() &&
_keys.find(cr.security().id()) == _keys.end()) {
_keys.insert(cr.security().id());
_cb(cr.security());
}
}
}
if (_keys.size() >= _count) {
throw SecurityExtractorDone();
}
}
};
void get_securities_from_replay(std::function<void(const api::Security &)> cb) {
SecurityExtractor extractor(cb, 100);
auto url = config::get<ReplayOptions>().url();
try {
replay_url_with_fallback(url, extractor, 0, false);
} catch (const SecurityExtractorDone &) {
}
}
shared_ptr<IRelayWriter> mkrelayproducer(bool shared);
/*this is the public one*/
void replay_recording(
const string &filename, int32_t naggle_ms, const std::string fallback) {
auto writer = mkrelayproducer(true);
if (config::get<ReplayOptions>().interactive()) {
naggle_ms = -1;
}
// Try relay_url/replay_file
// If it fails resort to the fallback location
if (filename.find("http://") == 0) {
// loops forever
replay_url(filename, *writer, naggle_ms);
} else if (boost::filesystem::exists(filename)) {
// loops forever
replay_file(filename, *writer, naggle_ms);
}
// try fallback
if (boost::filesystem::exists(fallback)) {
replay_file(fallback, *writer, naggle_ms);
}
}
void print_recording(const string &filename) {
ConsoleReplayWriter writer;
replay_file(filename, writer, 0);
}
void trace_recording(const string &filename, security_key key) {
ConsoleTraceWriter writer(key);
replay_file(filename, writer, 0);
}
void unprint_recording(const std::string &src, const std::string &dest) {
// quick and dirty minimal error checking
ifstream in;
in.open(src, ios::binary | ios::in);
ofstream out;
out.open(dest, ios::binary | ios::out);
// error handling
if (!(in.good() && out.good())) {
cout << "Error opening files." << endl;
return;
}
// Write out the header
CoconutReplayHeader header;
header.set_magic(magic);
header.set_version(recorder_version);
header.set_timestamp(datetime::ll_now());
header.set_host(config::get<config::SystemConfig>().host_name());
writebuf(out, header);
int index = 0;
string message_buffer;
co2::MDReply reply_buffer;
while (in) {
// '1 MDReply {'
in >> index;
if (in.eof())
break;
cout << index << " ";
char buffer[128];
in.getline(buffer, BTS_ARRAYSIZE(buffer), '{');
bts_assert(string(buffer) == " MDReply ");
// getline consumes the {
message_buffer.clear();
char bcount = 1;
while (bcount > 0 && !in.eof()) {
char ch;
in.get(ch);
if (ch == '{')
++bcount;
if (ch == '}')
--bcount;
if (bcount > 0)
message_buffer += ch;
}
cout << "buffered ";
reply_buffer.Clear();
if (google::protobuf::TextFormat::ParseFromString(
message_buffer, &reply_buffer)) {
cout << "parsed ";
writebuf(out, reply_buffer);
cout << "written";
}
cout << endl;
}
out.flush();
}
} // namespace md
} // namespace bts
#endif