#ifndef NO_CPP0X #include "RelayRecorder.h" #include "lib/config/config.h" #include "lib/utils/HTTPStream.h" #include "lib/utils/datetime.h" #include "lib/logging/logging.h" #include "lib/utils/bts_chrono.h" #include "lib/md/common.h" #include "lib/md/protos/Co2.pb.h" #include "lib/md/SecurityIDRecorder.h" #include "lib/config/protos/system.pb.h" #include "lib/config/protos/MarketData.pb.h" #include "lib/utils/bts_chrono.h" #include #include #include #include #include using namespace std; using namespace bts::co2; namespace bts { namespace md { const char* magic = "Co2"; const uint32_t recorder_version = 3; template void writebuf(ofstream& out, const T& pb) { uint32_t size = pb.ByteSize(); out.write(reinterpret_cast(&size), sizeof(size)); bts_assert(pb.SerializeToOstream(&out)); } template bool readbuf(istream& in, T* result) { if (result) { uint32_t size; in.read(reinterpret_cast(&size), sizeof(size)); vector buffer(size); in.read(&buffer[0], size); if (!in.fail()) { result->ParsePartialFromArray(&buffer[0], size); return true; } } return false; } RelayRecorder::RelayRecorder(const std::string& base_path, const std::string& filename, uint32_t naggle): _filename(base_path+"/"+filename), _file() { LOG(md,Info,"Recording market data to: {}" , _filename); boost::system::error_code ec; boost::filesystem::create_directories(base_path, ec); _file.open(_filename, ios::out | ios::binary | ios::trunc); bts_assert(_file.is_open()); CoconutReplayHeader header; header.set_magic(magic); header.set_version(recorder_version); header.set_timestamp(datetime::ll_now()); header.set_host(config::get().config_host()); header.set_naggle_ms(naggle); writebuf(_file, header); } RelayRecorder::~RelayRecorder() { } void RelayRecorder::write(const co2::MDReply& data, bool*) { writebuf(_file, data); flush(); } void RelayRecorder::flush() { _file.flush(); } struct ConsoleReplayWriter : IRelayWriter { ConsoleReplayWriter() : count(0) { } int count; virtual void write(const co2::MDReply& data, bool*) override { ++count; cout << count << " MDReply {" << endl; data.PrintDebugString(); cout << "}" << endl; } }; struct ConsoleTraceWriter : IRelayWriter { int count; security_key key; ConsoleTraceWriter(security_key k): count(0), key(k) { } virtual void write(const co2::MDReply & data, bool*) override { ++count; for (const auto& chan : data.channels()) { for (const auto& update : chan.updates()) { if (update.key() == key) { cout << count << " Update {" << endl; update.PrintDebugString(); cout << "}" << endl; } } } } }; #ifndef _WIN32 void record_security_keys(md::SecurityIDRecorder& idrec, const co2::MDReply& reply) { std::vector data; for (const auto& chan : reply.channels()) { for (const auto& upd : chan.updates()) { if (upd.has_security()) { data.push_back(&upd.security()); } } } idrec.bulk_insert(data, "REPLAY: "); } #endif void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) { bts_assert(!in.bad()); CoconutReplayHeader header; size_t count = 0; #ifndef _WIN32 md::SecurityIDRecorder idrec {{}}; #endif if (readbuf(in, &header) && header.magic() == magic && header.version() == recorder_version) { co2::MDReply message; while (readbuf(in, &message)) { bool has_securities = false; writer.write(message, &has_securities); if (has_securities) { // record security ids to avoid breaking simulator #ifndef _WIN32 record_security_keys(idrec, message); #endif } message.Clear(); if (naggle_ms >0) std::this_thread::sleep_for(std::chrono::duration(naggle_ms)); else if (naggle_ms == -1/*interactive*/) { ++count; cout << count << endl; cin.get(); } } } else { cout << "Invalid file format." << endl; } } void replay_file(const std::string& filename, IRelayWriter& writer, int32_t naggle_ms, bool loop) { do { ifstream in; in.open(filename, ios::binary | ios::in); if (in.is_open()) { replay_recording(in, writer, naggle_ms); } else { LOG(md,Diag, "Cannot read file {}", filename); if (loop) sleep_for(10s); } } while (loop); } bool replay_url(const std::string& url, IRelayWriter& writer, int32_t naggle_ms, bool loop) { // split up url parts size_t http_len = BTS_ARRAYSIZE("http://") - 1; auto after_host = url.find('/', http_len); auto host = url.substr(http_len, after_host-http_len); auto path = url.substr(after_host); auto last_slash = url.rfind('/'); auto filename = url.substr(last_slash+1); filename = os::sparkdata_path("data/replay/") + filename; utils::download_file_if_newer(host, path, filename); if (boost::filesystem::exists(filename)) { replay_file(filename, writer, naggle_ms, loop); return true; } return false; } void replay_url_with_fallback(const std::string& url, IRelayWriter& writer, int32_t naggle_ms, bool loop) { if (!replay_url(url, writer, naggle_ms, loop)) { const auto fallback = config::get().fallback(); if (boost::filesystem::exists(fallback)) { replay_file(fallback, writer, naggle_ms, loop); } } } struct SecurityExtractorDone : public std::bad_exception {}; struct SecurityExtractor : public IRelayWriter { size_t _count; std::function _cb; std::set _keys; SecurityExtractor(std::function& cb, size_t count) : _count(count), _cb(cb), _keys() { } ~SecurityExtractor() override {}; void write(const co2::MDReply& data, bool*) override { for (auto const &ch : data.channels()) { for (auto const &cr : ch.updates()) { if (cr.has_security() && _keys.find(cr.security().id()) == _keys.end()) { _keys.insert(cr.security().id()); _cb(cr.security()); } } } if (_keys.size() >= _count) { throw SecurityExtractorDone(); } } }; void get_securities_from_replay(std::function cb) { SecurityExtractor extractor(cb, 100); auto url = config::get().url(); try { replay_url_with_fallback(url, extractor, 0, false); } catch(const SecurityExtractorDone&) { } } shared_ptr mkrelayproducer(bool shared); /*this is the public one*/ void replay_recording(const string& filename, int32_t naggle_ms, const std::string fallback) { auto writer = mkrelayproducer(true); if (config::get().interactive()) { naggle_ms = -1; } // Try relay_url/replay_file // If it fails resort to the fallback location if (filename.find("http://") == 0) { // loops forever replay_url(filename, *writer, naggle_ms); } else if (boost::filesystem::exists(filename)) { // loops forever replay_file(filename, *writer, naggle_ms); } // try fallback if (boost::filesystem::exists(fallback)) { replay_file(fallback, *writer, naggle_ms); } } void print_recording(const string& filename) { ConsoleReplayWriter writer; replay_file(filename, writer, 0); } void trace_recording(const string& filename, security_key key) { ConsoleTraceWriter writer(key); replay_file(filename, writer, 0); } void unprint_recording(const std::string& src, const std::string& dest) { // quick and dirty minimal error checking ifstream in; in.open(src, ios::binary | ios::in); ofstream out; out.open(dest, ios::binary | ios::out); // error handling if (!(in.good() && out.good())) { cout << "Error opening files." << endl; return; } // Write out the header CoconutReplayHeader header; header.set_magic(magic); header.set_version(recorder_version); header.set_timestamp(datetime::ll_now()); header.set_host(config::get().host_name()); writebuf(out, header); int index = 0; string message_buffer; co2::MDReply reply_buffer; while (in) { // '1 MDReply {' in >> index; if (in.eof()) break; cout << index << " "; char buffer[128]; in.getline(buffer, BTS_ARRAYSIZE(buffer), '{'); bts_assert(string(buffer) == " MDReply "); // getline consumes the { message_buffer.clear(); char bcount = 1; while (bcount > 0 && !in.eof()) { char ch; in.get(ch); if (ch == '{') ++bcount; if (ch == '}') --bcount; if (bcount > 0) message_buffer += ch; } cout << "buffered "; reply_buffer.Clear(); if (google::protobuf::TextFormat::ParseFromString(message_buffer, &reply_buffer)) { cout << "parsed "; writebuf(out, reply_buffer); cout << "written"; } cout << endl; } out.flush(); } }}//namespace #endif