MarketDataGateway still exists, but is greatly simplified. It creates several `IMarketDataProvider`s, each of which generates a list of `IMarketDataSource`s. The market data sources provide market data to an `IMarketDataListener`. * EmumbaMarketDataProvider * CCApiMarketDataProvider * PyMarketDataProvider * OnixsMarketDataProvider * ReplayMarketDataProvider As a side effect, individual py market data sources are now explicitly limited to a single venue (it probably should have been this way all along, since the shared memory writers do not expect updates to involve multiple venues).
208 lines
7.2 KiB
C++
208 lines
7.2 KiB
C++
#include "MDReplayer.h"
|
|
#include "lib/core/TimeStamp.h"
|
|
#include "lib/domain/Book.h"
|
|
#include "lib/domain/Price.h"
|
|
#include "lib/domain/Quantity.h"
|
|
#include "lib/domain/Security.h"
|
|
#include "lib/domain/SecurityCommon.h"
|
|
#include "lib/domain/SecurityID.h"
|
|
#include "lib/domain/SecurityMarketDataStatistics.h"
|
|
#include "lib/domain/Side.h"
|
|
#include "lib/domain/TradeRecord.h"
|
|
#include "lib/domain_model/Security.pb.h"
|
|
#include "lib/logging/logging.h"
|
|
#include "lib/md/GeneralMarketDataListener.h"
|
|
#include "lib/md/IMarketDataListener.h"
|
|
#include "lib/md/MDProducer.h"
|
|
#include "lib/md/protos/co2.pb.h"
|
|
#include "lib/os/os.h"
|
|
#include "lib/persistence/IDGenerator.h"
|
|
#include "lib/utils/PBUtils.h"
|
|
#include "lib/utils/bts_assert.h"
|
|
#include "server/sparkd/md/recording/MDRecorder.h"
|
|
|
|
#include <cstdint>
|
|
#include <fstream>
|
|
#include <string>
|
|
|
|
namespace bts::md::recording {
|
|
|
|
using namespace bts::md;
|
|
|
|
void copy_books(const bts::co2::BookMessage &from, bts::domain::Book &to) {
|
|
to.clear_to_end();
|
|
const auto copy_side = [](const auto &from_side, auto &to_side) {
|
|
size_t i = 0;
|
|
auto sz = to_side.size();
|
|
for (const bts::co2::PriceSizeMessage &pz : from_side) {
|
|
if (i < sz) {
|
|
auto &to_b = to_side[i++];
|
|
to_b.total_quantity = to_b.direct_quantity =
|
|
bts::domain::Quantity(pz.size());
|
|
// reason for doing this is that bts::domain::PriceSize expects an
|
|
// exponent of -9 only
|
|
bts::md::IntPrice p_aux =
|
|
bts::md::IntPrice(pz.mantissa(), pz.exponent());
|
|
to_b.price = bts::domain::Price::from_double(p_aux.price());
|
|
}
|
|
}
|
|
};
|
|
copy_side(from.bids(), to[bts::domain::Side::Buy]);
|
|
copy_side(from.asks(), to[bts::domain::Side::Sell]);
|
|
}
|
|
|
|
struct MDReplayer ::PImpl {
|
|
// TODO, read from config
|
|
std::map<SecurityKey, co2::TotalBook> books = {};
|
|
bts::api::Venue venue;
|
|
std::set<uint32_t> sec_keys = {};
|
|
uint64_t start_time = 0;
|
|
std::shared_ptr<IMarketDataListener> listener{nullptr};
|
|
bool publish_boostrap_books_only;
|
|
double replay_speed;
|
|
|
|
PImpl(bts::api::Venue venue, double replay_speed)
|
|
: venue(venue), replay_speed(replay_speed) {}
|
|
};
|
|
|
|
MDReplayer::MDReplayer(bts::api::Venue venue, double replay_speed)
|
|
: me(new PImpl(venue, replay_speed)) {
|
|
LOG(md, Info, "MDReplayer::MDReplayer venue: {}", Venue_Name(venue));
|
|
}
|
|
|
|
MDReplayer::~MDReplayer() {}
|
|
|
|
void MDReplayer::start(std::shared_ptr<IMarketDataListener> listener) {
|
|
LOG(md, Info, "MDReplayer::start venue: {}", Venue_Name(me->venue));
|
|
bts_assert(listener != nullptr);
|
|
me->listener = std::move(listener);
|
|
me->start_time = bts::core::TimeStamp::now_ns();
|
|
std::string directory = MDRecorder::recording_path();
|
|
auto include_file = [&](const std::string &file_name) {
|
|
return MDRecorder::is_md_recording_file(file_name, me->venue);
|
|
};
|
|
std::string file_name = os::find_newest_file(directory, include_file);
|
|
if (file_name.empty()) {
|
|
LOG(md, Error, "MDReplayer::run could not find a file to replay");
|
|
return;
|
|
}
|
|
file_name = directory + "/" + file_name;
|
|
LOG(md, Info, "MDReplayer::run venue: {} file_name: {}",
|
|
bts::api::Venue_Name(me->venue), file_name);
|
|
while (true) {
|
|
_replay_file_once(file_name);
|
|
}
|
|
}
|
|
|
|
void MDReplayer::_replay_file_once(const std::string &file_name) {
|
|
auto log_progress = [](auto i) {
|
|
if (i++ % 10000 == 0)
|
|
LOG(md, Info, "MDReplayer::_replay_once no of decoded msgs: {}", i);
|
|
};
|
|
LOG(md, Info, "MDReplayer::_replay_file_once replaying file: {}", file_name);
|
|
std::fstream in(file_name, std::ios::binary | std::ios::in);
|
|
co2::RecordedMDMessage rec_msg;
|
|
int i = 0;
|
|
int64_t beg_ns = bts::core::TimeStamp::now_ns();
|
|
int64_t last_msg_ts = 0;
|
|
double replay_speed = me->replay_speed > 0 ? me->replay_speed : 1;
|
|
while (!in.eof() && utils::pb::readbuf(in, &rec_msg)) {
|
|
if (i++ == 0) {
|
|
last_msg_ts = rec_msg.timestamp();
|
|
_replay_message(rec_msg);
|
|
} else {
|
|
bts_assert(last_msg_ts != 0);
|
|
int64_t this_msg_ts = rec_msg.timestamp();
|
|
bts_assert(this_msg_ts >= last_msg_ts);
|
|
int64_t ts_diff = this_msg_ts - last_msg_ts;
|
|
int64_t adjusted_ts = ts_diff / replay_speed;
|
|
last_msg_ts = this_msg_ts;
|
|
std::this_thread::sleep_for(std::chrono::nanoseconds(adjusted_ts));
|
|
_replay_message(rec_msg);
|
|
}
|
|
log_progress(i);
|
|
}
|
|
int64_t end_ns = bts::core::TimeStamp::now_ns();
|
|
LOG(md, Info, "MDReplayer::_replay_file_once time: {} ns", end_ns - beg_ns);
|
|
in.close();
|
|
}
|
|
|
|
void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) {
|
|
auto get_book = [&]() -> const co2::TotalBook & {
|
|
SecurityKey sk = SecurityKey(m.sec_key());
|
|
const co2::TotalBook &book = m.total_book();
|
|
if (!me->publish_boostrap_books_only) {
|
|
LOG(md, Diag,
|
|
"MDReplayer::_replay_message::get_book "
|
|
"publish_boostrap_books_only: {} sk: {} "
|
|
"returning new "
|
|
"book",
|
|
me->publish_boostrap_books_only, sk);
|
|
return book;
|
|
}
|
|
auto it = me->books.find(sk);
|
|
if (it == me->books.end()) {
|
|
me->books.insert({sk, book});
|
|
LOG(md, Diag,
|
|
"MDReplayer::_replay_message::get_book "
|
|
"publish_boostrap_books_only: {} sk: {} "
|
|
"returning new "
|
|
"book",
|
|
me->publish_boostrap_books_only, sk);
|
|
}
|
|
it = me->books.find(sk);
|
|
bts_assert(it != me->books.end());
|
|
const co2::TotalBook &ex_book = it->second;
|
|
LOG(md, Diag,
|
|
"MDReplayer::_replay_message::get_book publish_boostrap_books_only: "
|
|
"{} sk: {} "
|
|
"returning existing "
|
|
"book",
|
|
me->publish_boostrap_books_only, sk);
|
|
return ex_book;
|
|
};
|
|
switch (m.MDRecord_case()) {
|
|
case (co2::RecordedMDMessage::MDRecordCase::kSecurity): {
|
|
LOG(md, Diag, "MDReplayer::_replay_message m: {}",
|
|
utils::pb::to_string(m));
|
|
bts::domain::Security new_sec = m.security();
|
|
me->sec_keys.insert(m.security().id());
|
|
std::vector<domain::Security const *> sec_vec = {};
|
|
sec_vec.push_back(&new_sec);
|
|
me->listener->on_securities(sec_vec);
|
|
break;
|
|
}
|
|
case (co2::RecordedMDMessage::MDRecordCase::kTotalBook): {
|
|
// TODO book replay
|
|
LOG(md, Diag, "MDReplayer::_replay_message m: {}",
|
|
utils::pb::to_string(m));
|
|
const bts::co2::TotalBook &tot_book = get_book();
|
|
bts_assert(me->sec_keys.find(m.sec_key()) != me->sec_keys.end());
|
|
if (tot_book.has_direct_book()) {
|
|
|
|
// we can cache this instance and re-use it for perf
|
|
bts::domain::Book book = {};
|
|
copy_books(tot_book.direct_book(), book);
|
|
book.set_security_id(bts::domain::SecurityID(m.sec_key()));
|
|
me->listener->on_book(book);
|
|
}
|
|
break;
|
|
}
|
|
case (co2::RecordedMDMessage::MDRecordCase::kMisc): {
|
|
bts::domain::SecurityMarketDataStatistics stats(m.sec_key(), m.misc());
|
|
me->listener->on_security_statistics(stats);
|
|
break;
|
|
}
|
|
case (co2::RecordedMDMessage::MDRecordCase::kTrade): {
|
|
domain::TradeRecord tr;
|
|
domain::DomainSecurityUtils::convert(m.trade(), tr);
|
|
me->listener->on_trades({&tr});
|
|
break;
|
|
}
|
|
default: {
|
|
}
|
|
}
|
|
}
|
|
|
|
} // namespace bts::md::recording
|