326 lines
9.5 KiB
C++
326 lines
9.5 KiB
C++
#ifndef NO_CPP0X
|
|
|
|
#include "RelayRecorder.h"
|
|
#include "lib/config/config.h"
|
|
#include "lib/utils/HTTPStream.h"
|
|
#include "lib/utils/datetime.h"
|
|
#include "lib/logging/logging.h"
|
|
#include "lib/utils/bts_chrono.h"
|
|
|
|
#include "lib/md/common.h"
|
|
#include "lib/md/protos/Co2.pb.h"
|
|
#include "lib/md/SecurityIDRecorder.h"
|
|
#include "lib/config/protos/system.pb.h"
|
|
#include "lib/config/protos/MarketData.pb.h"
|
|
#include "lib/utils/bts_chrono.h"
|
|
|
|
#include <boost/filesystem.hpp>
|
|
#include <google/protobuf/text_format.h>
|
|
#include <memory>
|
|
#include <string.h>
|
|
#include <thread>
|
|
|
|
using namespace std;
|
|
using namespace bts::co2;
|
|
|
|
namespace bts {
|
|
namespace md {
|
|
|
|
const char* magic = "Co2";
|
|
const uint32_t recorder_version = 3;
|
|
|
|
template <class T>
|
|
void writebuf(ofstream& out, const T& pb) {
|
|
uint32_t size = pb.ByteSize();
|
|
out.write(reinterpret_cast<char*>(&size), sizeof(size));
|
|
bts_assert(pb.SerializeToOstream(&out));
|
|
}
|
|
|
|
template <class T>
|
|
bool readbuf(istream& in, T* result) {
|
|
if (result) {
|
|
uint32_t size;
|
|
in.read(reinterpret_cast<char*>(&size), sizeof(size));
|
|
vector<char> buffer(size);
|
|
in.read(&buffer[0], size);
|
|
if (!in.fail()) {
|
|
result->ParsePartialFromArray(&buffer[0], size);
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
RelayRecorder::RelayRecorder(const std::string& base_path, const std::string& filename, uint32_t naggle):
|
|
_filename(base_path+"/"+filename),
|
|
_file()
|
|
{
|
|
LOG(md,Info,"Recording market data to: {}" , _filename);
|
|
boost::system::error_code ec;
|
|
boost::filesystem::create_directories(base_path, ec);
|
|
_file.open(_filename, ios::out | ios::binary | ios::trunc);
|
|
bts_assert(_file.is_open());
|
|
|
|
CoconutReplayHeader header;
|
|
header.set_magic(magic);
|
|
header.set_version(recorder_version);
|
|
header.set_timestamp(datetime::ll_now());
|
|
header.set_host(config::get<config::System>().config_host());
|
|
header.set_naggle_ms(naggle);
|
|
writebuf(_file, header);
|
|
}
|
|
|
|
RelayRecorder::~RelayRecorder() { }
|
|
|
|
void RelayRecorder::write(const co2::MDReply& data, bool*) {
|
|
writebuf(_file, data);
|
|
flush();
|
|
}
|
|
|
|
void RelayRecorder::flush() {
|
|
_file.flush();
|
|
}
|
|
|
|
struct ConsoleReplayWriter : IRelayWriter
|
|
{
|
|
ConsoleReplayWriter() : count(0) { }
|
|
int count;
|
|
virtual void write(const co2::MDReply& data, bool*) override
|
|
{
|
|
++count;
|
|
cout << count << " MDReply {" << endl;
|
|
data.PrintDebugString();
|
|
cout << "}" << endl;
|
|
}
|
|
};
|
|
|
|
struct ConsoleTraceWriter : IRelayWriter
|
|
{
|
|
int count;
|
|
security_key key;
|
|
ConsoleTraceWriter(security_key k): count(0), key(k) { }
|
|
virtual void write(const co2::MDReply & data, bool*) override {
|
|
++count;
|
|
for (const auto& chan : data.channels()) {
|
|
for (const auto& update : chan.updates()) {
|
|
if (update.key() == key) {
|
|
cout << count << " Update {" << endl;
|
|
update.PrintDebugString();
|
|
cout << "}" << endl;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
#ifndef _WIN32
|
|
void record_security_keys(md::SecurityIDRecorder& idrec, const co2::MDReply& reply) {
|
|
std::vector<const Security*> data;
|
|
for (const auto& chan : reply.channels()) {
|
|
for (const auto& upd : chan.updates()) {
|
|
if (upd.has_security()) {
|
|
data.push_back(&upd.security());
|
|
}
|
|
}
|
|
}
|
|
idrec.bulk_insert(data, "REPLAY: ");
|
|
}
|
|
#endif
|
|
|
|
void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) {
|
|
bts_assert(!in.bad());
|
|
CoconutReplayHeader header;
|
|
size_t count = 0;
|
|
#ifndef _WIN32
|
|
md::SecurityIDRecorder idrec {{}};
|
|
#endif
|
|
if (readbuf(in, &header) && header.magic() == magic && header.version() == recorder_version) {
|
|
co2::MDReply message;
|
|
while (readbuf(in, &message)) {
|
|
bool has_securities = false;
|
|
writer.write(message, &has_securities);
|
|
if (has_securities) {
|
|
// record security ids to avoid breaking simulator
|
|
#ifndef _WIN32
|
|
record_security_keys(idrec, message);
|
|
#endif
|
|
}
|
|
message.Clear();
|
|
if (naggle_ms >0) std::this_thread::sleep_for(std::chrono::duration<int, std::milli>(naggle_ms));
|
|
else if (naggle_ms == -1/*interactive*/) {
|
|
++count;
|
|
cout << count << endl;
|
|
cin.get();
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
cout << "Invalid file format." << endl;
|
|
}
|
|
}
|
|
|
|
void replay_file(const std::string& filename, IRelayWriter& writer, int32_t naggle_ms, bool loop) {
|
|
do {
|
|
ifstream in;
|
|
in.open(filename, ios::binary | ios::in);
|
|
if (in.is_open()) {
|
|
replay_recording(in, writer, naggle_ms);
|
|
}
|
|
else {
|
|
LOG(md,Diag, "Cannot read file {}", filename);
|
|
if (loop)
|
|
sleep_for(10s);
|
|
}
|
|
} while (loop);
|
|
}
|
|
|
|
bool replay_url(const std::string& url, IRelayWriter& writer, int32_t naggle_ms, bool loop) {
|
|
// split up url parts
|
|
size_t http_len = BTS_ARRAYSIZE("http://") - 1;
|
|
auto after_host = url.find('/', http_len);
|
|
auto host = url.substr(http_len, after_host-http_len);
|
|
auto path = url.substr(after_host);
|
|
auto last_slash = url.rfind('/');
|
|
auto filename = url.substr(last_slash+1);
|
|
filename = os::sparkdata_path("data/replay/") + filename;
|
|
utils::download_file_if_newer(host, path, filename);
|
|
if (boost::filesystem::exists(filename)) {
|
|
replay_file(filename, writer, naggle_ms, loop);
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void replay_url_with_fallback(const std::string& url, IRelayWriter& writer, int32_t naggle_ms, bool loop) {
|
|
if (!replay_url(url, writer, naggle_ms, loop)) {
|
|
const auto fallback = config::get<ReplayOptions>().fallback();
|
|
if (boost::filesystem::exists(fallback)) {
|
|
replay_file(fallback, writer, naggle_ms, loop);
|
|
}
|
|
}
|
|
}
|
|
|
|
struct SecurityExtractorDone : public std::bad_exception {};
|
|
struct SecurityExtractor : public IRelayWriter {
|
|
size_t _count;
|
|
std::function<void(const api::Security&)> _cb;
|
|
std::set<uint32_t> _keys;
|
|
SecurityExtractor(std::function<void(const api::Security&)>& cb, size_t count)
|
|
: _count(count), _cb(cb), _keys() { }
|
|
~SecurityExtractor() override {};
|
|
void write(const co2::MDReply& data, bool*) override {
|
|
for (auto const &ch : data.channels()) {
|
|
for (auto const &cr : ch.updates()) {
|
|
if (cr.has_security() && _keys.find(cr.security().id()) == _keys.end()) {
|
|
_keys.insert(cr.security().id());
|
|
_cb(cr.security());
|
|
}
|
|
}
|
|
}
|
|
if (_keys.size() >= _count) {
|
|
throw SecurityExtractorDone();
|
|
}
|
|
}
|
|
};
|
|
|
|
void get_securities_from_replay(std::function<void(const api::Security&)> cb) {
|
|
SecurityExtractor extractor(cb, 100);
|
|
auto url = config::get<ReplayOptions>().url();
|
|
try {
|
|
replay_url_with_fallback(url, extractor, 0, false);
|
|
} catch(const SecurityExtractorDone&) { }
|
|
}
|
|
|
|
|
|
shared_ptr<IRelayWriter> mkrelayproducer(bool shared);
|
|
|
|
/*this is the public one*/
|
|
void replay_recording(const string& filename, int32_t naggle_ms, const std::string fallback) {
|
|
auto writer = mkrelayproducer(true);
|
|
if (config::get<ReplayOptions>().interactive()) {
|
|
naggle_ms = -1;
|
|
}
|
|
|
|
// Try relay_url/replay_file
|
|
// If it fails resort to the fallback location
|
|
|
|
if (filename.find("http://") == 0) {
|
|
// loops forever
|
|
replay_url(filename, *writer, naggle_ms);
|
|
}
|
|
else if (boost::filesystem::exists(filename)) {
|
|
// loops forever
|
|
replay_file(filename, *writer, naggle_ms);
|
|
}
|
|
// try fallback
|
|
if (boost::filesystem::exists(fallback)) {
|
|
replay_file(fallback, *writer, naggle_ms);
|
|
}
|
|
}
|
|
void print_recording(const string& filename) {
|
|
ConsoleReplayWriter writer;
|
|
replay_file(filename, writer, 0);
|
|
}
|
|
|
|
void trace_recording(const string& filename, security_key key) {
|
|
ConsoleTraceWriter writer(key);
|
|
replay_file(filename, writer, 0);
|
|
}
|
|
|
|
void unprint_recording(const std::string& src, const std::string& dest) {
|
|
// quick and dirty minimal error checking
|
|
ifstream in;
|
|
in.open(src, ios::binary | ios::in);
|
|
ofstream out;
|
|
out.open(dest, ios::binary | ios::out);
|
|
|
|
// error handling
|
|
if (!(in.good() && out.good())) {
|
|
cout << "Error opening files." << endl;
|
|
return;
|
|
}
|
|
|
|
// Write out the header
|
|
CoconutReplayHeader header;
|
|
header.set_magic(magic);
|
|
header.set_version(recorder_version);
|
|
header.set_timestamp(datetime::ll_now());
|
|
header.set_host(config::get<config::System>().host_name());
|
|
writebuf(out, header);
|
|
|
|
int index = 0;
|
|
string message_buffer;
|
|
co2::MDReply reply_buffer;
|
|
while (in) {
|
|
// '1 MDReply {'
|
|
in >> index;
|
|
if (in.eof()) break;
|
|
cout << index << " ";
|
|
char buffer[128];
|
|
in.getline(buffer, BTS_ARRAYSIZE(buffer), '{');
|
|
bts_assert(string(buffer) == " MDReply ");
|
|
// getline consumes the {
|
|
message_buffer.clear();
|
|
char bcount = 1;
|
|
while (bcount > 0 && !in.eof()) {
|
|
char ch;
|
|
in.get(ch);
|
|
if (ch == '{') ++bcount;
|
|
if (ch == '}') --bcount;
|
|
if (bcount > 0) message_buffer += ch;
|
|
}
|
|
cout << "buffered ";
|
|
reply_buffer.Clear();
|
|
if (google::protobuf::TextFormat::ParseFromString(message_buffer, &reply_buffer)) {
|
|
cout << "parsed ";
|
|
writebuf(out, reply_buffer);
|
|
cout << "written";
|
|
}
|
|
cout << endl;
|
|
}
|
|
out.flush();
|
|
}
|
|
|
|
}}//namespace
|
|
#endif
|