refactor: Split MDRelay and QueryServer up into smaller files

This code is inherently complicated. It's hard to follow when everything
is in one giant file.
This commit is contained in:
John Colvin committed 2026-03-20 17:02:44 +00:00
1 parent 35d65f107b
commit d7b42e3cee
19 files changed
+980 -817

No files matched your search

+4
View File
@@ -2,6 +2,7 @@ services:
sparkd:
volumes:
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
build:
dockerfile: src/app_sparkd/Dockerfile
args:
@@ -161,3 +162,6 @@ volumes:
name: "md_publisher_22019"
historic_md:
name: "historic_md"
md_recording:
name: "md_recording"
external: true
+1 -1
View File
@@ -10,7 +10,7 @@
#include "spark/md/MBOServiceServer.h"
#include "spark/md/MDClient.h"
#include "spark/md/SyntheticSecurityService.h"
#include "spark/md/relay/QueryServer.h"
#include "spark/md/relay/query_server.h"
#include "spark/md/service/MarketDataServiceServer.h"
#include "spark/metrics/metrics.h"
#include "spark/metrics/system_metrics.h"
+1 -1
View File
@@ -6,7 +6,7 @@
#include "spark/config/protos/quickbus_cfg.pb.h"
#include "spark/md/MDClient.h"
#include "spark/md/SyntheticSecurityService.h"
#include "spark/md/relay/QueryServer.h"
#include "spark/md/relay/query_server.h"
#include "spark/metrics/metrics.h"
#include "spark/metrics/system_metrics.h"
#include "spark/quickbus/core/Quickbus.h"
@@ -91,7 +91,11 @@ function(get_cpp_sources dest)
${PROJECT_NAME}/md/cme/MDP3Utils.cpp
${PROJECT_NAME}/md/common.cpp
${PROJECT_NAME}/md/ice/ICEUtils.cpp
${PROJECT_NAME}/md/relay/QueryServer.cpp
${PROJECT_NAME}/md/relay/internal/md_consumer_group.cpp
${PROJECT_NAME}/md/relay/internal/md_relay_server.cpp
${PROJECT_NAME}/md/relay/internal/query_server_impl.cpp
${PROJECT_NAME}/md/relay/i_query_server.cpp
${PROJECT_NAME}/md/relay/query_server.cpp
${PROJECT_NAME}/md/relay/RelayClient.cpp
${PROJECT_NAME}/md/relay/RelayRecorder.cpp
${PROJECT_NAME}/md/service/MarketDataServiceServer.cpp
@@ -1,770 +0,0 @@
#ifndef _WIN32
#include "QueryServer.h"
#include "spark/config/config.h"
#include "spark/domain/DomainSecurityUtils.h"
#include "spark/domain_model/Security.h"
#include "spark/md/SymbolLoader.h"
#include "spark/utils/AuthToken.h"
#include "spark/utils/Dispatcher.h"
#include "spark/utils/TimeStamp.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/helper_macros.h"
#include "../MBOProcessor.h"
#include "../MDClient.h"
#include "../MDConsumer.h"
#include "spark/logging/logging.h"
#include "spark/api/apicore.pb.h"
#include "spark/api/public/ServiceContext.h"
#include "spark/api/service_helpers/AdminServiceUtils.h"
#include "spark/api/services/AdminService.client.pb.h"
#include "spark/api/services/MarketDataRelay.server.pb.h"
#include "spark/config/protos/cme_reference_data.pb.h"
#include "spark/config/protos/system_config.pb.h"
#include "spark/md/SecurityCache.h"
#include "spark/md/protos/co2.pb.h"
#include "spark/utils/ConcurrentAppendCollection.h"
#ifndef _WIN32
#include "spark/metrics/metrics.h"
#endif
#include <list>
#include <memory>
#include <optional>
#include <set>
#include <unordered_map>
#include <vector>
#include <boost/container/flat_map.hpp>
#include <google/protobuf/arena.h>
using namespace bts::co2;
using bts::domain::DomainSecurityUtils;
using std::shared_ptr;
using std::string;
using std::unique_ptr;
namespace bts {
namespace md {
/*
* NORMAL OPERATION:
* Client sends time=0 to server
* Server sends complete snapshot back (time=502)
* client sends time=502 to server
* Server sends any changes that have happened after time=502 and any securities
* discovered since time=502
*
* ERROR RECOVERY:
* Client sends time=0 to server
* Server sends complete snapshot back (time=502)
* Server detects heartbeat failure and recovers. All timestamps > 502
* Client sends time=502 to server
* Server sends complete snapshot back (time=605)
* - Since the server shared memory has been recreated the client will
* automatically overwrite all stale data.
* - If a security has been deleted the client will leave it frozen (OK)
*
* We scan for bad heartbeats on a different time
* If a bad heartbeat is detected we try to construct a new Consumer
* When a good consumer is created we std::move the old consumer and handles to
* delayed queue for deletion This way we don't have to lock anything during the
* recovery process because all pointers will remain valid for the duration of
* existing calls.
*
*/
struct DatedHandle {
TimeStamp discovered;
MDConsumerHandle handle;
};
struct MDConsumerGroup {
// probably just have to send the consumer in from outside
MDConsumerGroup(
const UserSymbol &symbol,
const MDConsumer &consumer,
metrics::MetricsContainer &stats)
: symbol(symbol), channel(consumer.get_channel(symbol)), handles(),
mic() {
set_consumer(consumer);
// create and publish a stat for this group
#ifndef _WIN32
if (symbol.type() != SecurityType::SECURITYTYPE_SYNTHETIC) {
std::vector<metrics::Label> labels;
labels.push_back({"symbol", symbol.symbol()});
labels.push_back(
{"type",
SecurityType_Name(symbol.type()).substr(13) /*SECURITYTYPE_*/});
labels.push_back(
{"exch", Venue_Name(symbol.exchange()).substr(6) /*VENUE_*/});
stats.publish(std::make_shared<metrics::LazyCounterMetricL>(
"spark_md_book_updates",
[=, this]() {
return channel.direct_count() + channel.implied_count();
},
labels));
stats.publish(std::make_shared<metrics::LazyCounterMetricL>(
"spark_md_security_count",
[=, this]() { return channel.security_count(); }, labels));
stats.publish(std::make_shared<metrics::LazyCounterMetricL>(
"spark_md_trade_count",
[=, this]() { return channel.trade_count(); }, labels));
stats.publish(std::make_shared<metrics::LazyCounterMetricD>(
"spark_md_gap_count",
[=, this]() {
auto g = channel.gap_count();
return g == 0 ? os::nan() : g;
},
labels));
}
#endif
}
MDConsumerGroup(const MDConsumerGroup &other) = delete;
// recovery has occured and we need to clear our our collection of handles
// and wait for new ones
void set_consumer(const MDConsumer &consumer) {
// create a new set of handles and swap with the old.
// If the new consumer is bad we still have to swap our handles because
// the consumer that owns our handles will be scheduled for deletion
auto temphandles =
std::make_shared<ConcurrentAppendCollection<DatedHandle>>();
handles.swap(temphandles);
channel = consumer.get_channel(symbol);
}
void add_security(MDConsumerHandle &handle) {
bts_assert(handles);
bts_assert_d(handle.is_initialized());
bts_assert(handle.get_usersymbol() == symbol);
DatedHandle h = {TimeStamp::now(), handle};
handles->push_back(h);
if (this->mic == "")
set_mic(h);
}
void set_mic(const DatedHandle &h) {
switch (channel.exchange()) {
case api::VENUE_ICE:
case api::VENUE_SIMICE:
this->mic = h.handle.get_normalized_security().ice().mic_code();
break;
case api::VENUE_GLBX:
case api::VENUE_SIM: {
const auto &cme_symb_mics =
config::get<config::CmeSymbolMics>().cme_mic();
for (auto mic = cme_symb_mics.begin(); mic != cme_symb_mics.end();
mic++) {
const auto &micsym = mic->first;
if (micsym == symbol.symbol()) {
this->mic = api::CMEMics_Name(mic->second);
break;
}
}
break;
}
case api::VENUE_BTS:
case api::VENUE_UNSET:
this->mic = "BTS";
break;
case api::VENUE_DERIBIT:
this->mic = "DERIBIT";
break;
case api::VENUE_FTX:
this->mic = "FTX";
break;
case api::VENUE_BINANCE:
this->mic = "BINANCE";
break;
case api::VENUE_BINANCE_COIN_FUTURES:
this->mic = "BINANCE_COIN_FUTURES";
break;
case api::VENUE_BINANCE_US:
this->mic = "BINANCE_US";
break;
case api::VENUE_COINBASE:
this->mic = "COINBASE";
break;
case api::VENUE_BINANCE_USDS_FUTURES:
this->mic = "BINANCE_USDS_FUTURES";
break;
case api::VENUE_OKX:
this->mic = "OKX";
break;
case api::VENUE_CBOE:
this->mic = "CBOE";
break;
case api::VENUE_TWSE:
this->mic = "TWSE";
break;
case api::VENUE_TAIFEX:
this->mic = "TAIFEX";
break;
case api::VENUE_BYBIT:
this->mic = "BYBIT";
break;
case api::VENUE_GATEIO:
this->mic = "GATEIO";
break;
case api::VENUE_CBOECFE:
case api::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
case api::Venue_INT_MAX_SENTINEL_DO_NOT_USE_:
bts_assert(false);
this->mic = "UNKNOWN";
break;
}
}
UserSymbol symbol;
MDConsumerChannel channel;
// Can add to handles without invalidating readers (no lock needed)
// When we recover we swap a new vector into this unique_ptr and
// delayed-delete
shared_ptr<ConcurrentAppendCollection<DatedHandle>> handles;
std::string mic;
};
void convert(PriceSize ps, PriceSizeMessage *pps) {
pps->set_mantissa(ps.mantissa());
pps->set_exponent(ps.exponent());
pps->set_size(ps.size());
}
void set_price(PriceSize ps, co2::TradeMessage *ptrade) {
ptrade->set_mantissa(ps.mantissa());
ptrade->set_exponent(ps.exponent());
ptrade->set_size(ps.size());
}
void set_price(const IntPrice &price, co2::PriceMessage *dest) {
dest->set_mantissa(price.mantissa());
dest->set_exponent(price.exponent());
}
void copy_misc(IntPrice settlement, MiscData misc, co2::MiscMessage *pmisc) {
set_price(settlement, pmisc->mutable_settlement());
pmisc->set_settlement_date(misc.settlement_date.to_short());
set_price(
misc.previous_settlement_price,
pmisc->mutable_previous_settlement_price());
pmisc->set_previous_settlement_date(
misc.previous_settlement_date.to_short());
pmisc->set_open_price(misc.open_price);
pmisc->set_open_date(misc.open_date.to_short());
pmisc->set_close_price(misc.close_price);
pmisc->set_close_date(misc.close_date.to_short());
pmisc->set_fixing_price(misc.fixing_price);
pmisc->set_session_high(misc.session_high);
pmisc->set_session_low(misc.session_low);
// pmisc->set_session_high_bid(misc.session_high_bid);
// pmisc->set_session_low_offer(misc.session_low_offer);
}
// Handles market data queries and responds
class QueryServerImpl : public IQueryServer {
MDClient &_client;
MDConsumer _consumer;
int _sub;
boost::container::flat_map<UserSymbol, shared_ptr<MDConsumerGroup>> _groups;
public:
QueryServerImpl(MDClient &client, metrics::MetricsContainer &stats)
: _client(client), _consumer(client.current_consumer()), _sub(0),
_groups() {
for (const auto &sym : client.symbols()) {
_groups.emplace(std::make_pair(
sym, std::make_unique<MDConsumerGroup>(sym, _consumer, stats)));
}
_groups.shrink_to_fit();
_sub = client.on_consumer().subscribe(
[=, this](const MDConsumer &c) { handle_recovery(c); });
subscribe_securities();
}
QueryServerImpl(const QueryServerImpl &) = delete;
~QueryServerImpl() { _client.on_consumer().unsubscribe(_sub); }
void subscribe_securities() {
LOG(md, Info, "QueryServer::subscribe_securities");
auto id = _consumer.id();
_consumer.on_security(
[this, id](const SecurityPtr &sec, MDConsumerHandle handle) {
if (_consumer.id() == id) {
auto iter = _groups.find(handle.get_usersymbol());
bts_assert(iter != _groups.end());
const auto &pgroup = iter->second;
bts_assert(pgroup);
LOG(md, Info,
"QueryServer::_consumer.on_security symbol:{} "
"sec:{}",
handle.get_usersymbol().symbol(),
bts::securities::descriptor(*sec));
pgroup->add_security(handle);
}
});
}
// not-reentrant
void handle_recovery(const MDConsumer &consumer) {
MDConsumer temp(consumer);
if (temp.all_heartbeats_ok()) {
_consumer.swap(temp);
LOG(md, Info,
"Recovering from bad MD. Old consumer {} new consumer "
"{}",
_consumer.id(), temp.id());
for (auto &pair : _groups) {
pair.second->set_consumer(_consumer);
}
// _handle_index.clear();
subscribe_securities();
utils::Dispatcher::background().post(
[temp]() {
LOG(md, Info, "QServer destroying consumer {}", temp.id());
},
60000);
}
}
// the hot part
void fill(
const SecurityChannel &chan,
MDConsumerGroup &group,
ChannelReply *reply,
const Entitlements *entitlements,
bool bypass_entitlement) const {
bts_assert(reply);
reply->mutable_channel()->CopyFrom(chan);
reply->mutable_channel()->set_time(TimeStamp::now().to_uint64());
bool entitled = bypass_entitlement || chan.exchange() == api::VENUE_BTS;
if (entitlements && !entitled)
entitled = entitlements->count(group.mic);
reply->mutable_channel()->set_entitled(entitled);
auto flags = (MDRequest::Flags)chan.flags();
// Get a copy of handles. If recovery happens during this call
// we can safely continue using this pointer for 60 seconds
auto phandles = group.handles;
if (!phandles)
return;
ChannelUpdate temp_update;
for (const auto &instrument : *phandles) {
bool snapshot = false;
MDConsumerHandle handle = instrument.handle;
temp_update.Clear();
bts_assert_d(handle.is_initialized());
if (!handle.is_initialized())
continue;
ChannelUpdate *pupd = &temp_update;
pupd->set_key(instrument.handle.key());
// If the security was just discovered try to ensure all
// data is sent
if (instrument.discovered > TimeStamp(chan.time())) {
auto security = handle.get_normalized_security();
pupd->mutable_security()->CopyFrom(security);
snapshot = true;
}
// if we are not entitled AND this is the first request
// since we lost our entitlement, zero out books
if (!entitled && chan.entitled()) {
if (HASFLAG(flags, MDRequest::BBO) ||
HASFLAG(flags, MDRequest::DIRECT)) {
auto *p = pupd->mutable_direct();
for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) {
convert(NO_MARKET, p->add_bids());
convert(NO_MARKET, p->add_asks());
}
}
if ((HASFLAG(flags, MDRequest::IMPLIED) ||
HASFLAG(flags, MDRequest::BBO)) &&
!handle.is_synthetic()) {
auto *p = pupd->mutable_implied();
for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) {
convert(NO_MARKET, p->add_bids());
convert(NO_MARKET, p->add_asks());
}
}
}
if (entitled) {
// If we are entitled now but this is the first request
// where we are ensure we send all values
if (!chan.entitled())
snapshot = true;
if (HASFLAG(flags, MDRequest::BBO) ||
HASFLAG(flags, MDRequest::DIRECT)) {
std::array<PriceSize, COCONUT_MAX_DEPTH> bids, asks;
TimeStamp time(snapshot ? 0 : chan.time());
size_t copy_depth =
HASFLAG(flags, MDRequest::DIRECT) ? COCONUT_MAX_DEPTH : 1;
auto result = handle.get_direct_book(
time, copy_depth, bids.data(), asks.data());
if (result == HandleReadResult::SUCCESS) {
auto *p = pupd->mutable_direct();
for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) {
PriceSize b = bids[x];
if (b != NO_MARKET) {
convert(b, p->add_bids());
}
PriceSize a = asks[x];
if (a != NO_MARKET) {
convert(a, p->add_asks());
}
}
} else if (result == HandleReadResult::BUSY) {
LOG(md, Info,
"Busy read on handle {}. Retry on next "
"request",
instrument.handle.key());
// If we get a busy reading BBO (which happens
// 1-2x per day for a single relay client) We move
// the timestamp back so the next request will try
// to read this value again. ALTERNATIVE: We could
// spin until the value was not busy but it could
// be that the writer was context switched out and
// we will have to wait multiple ms to unblock the
// request. This fix assumes it is better to send
// the update now with 1 stale value than to block
// the call and allow all the values to be stale
// until this one can be read.
reply->mutable_channel()->set_time(chan.time());
}
}
if ((HASFLAG(flags, MDRequest::IMPLIED) ||
HASFLAG(flags, MDRequest::BBO)) &&
!handle.is_synthetic()) {
std::array<PriceSize, COCONUT_MAX_IMPLIED_DEPTH> bids, asks;
TimeStamp time(snapshot ? 0 : chan.time());
size_t copy_depth = HASFLAG(flags, MDRequest::IMPLIED)
? COCONUT_MAX_IMPLIED_DEPTH
: 1;
auto result = handle.get_implied_book(
time, copy_depth, bids.data(), asks.data());
if (result == HandleReadResult::SUCCESS) {
auto *p = pupd->mutable_implied();
for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) {
PriceSize b = bids[x];
if (b != NO_MARKET) {
convert(b, p->add_bids());
}
PriceSize a = asks[x];
if (a != NO_MARKET) {
convert(a, p->add_asks());
}
}
} else if (result == HandleReadResult::BUSY) {
LOG(md, Info,
"Busy read on handle (implied) {}. Retry on "
"next request",
instrument.handle.key());
reply->mutable_channel()->set_time(chan.time());
}
}
if (HASFLAG(flags, MDRequest::LAST_TRADE)) {
TimeStamp time(snapshot ? 0 : chan.time());
PriceSize price;
datetime::LLDateTime trade_time;
size_t volume;
int from_spread;
if (handle.get_last_trade(
time, price, trade_time, &volume, &from_spread)) {
auto *trade = pupd->mutable_last_trade();
trade->set_key(handle.key());
trade->set_time(trade_time);
trade->set_total_volume(volume);
trade->set_from_spread(from_spread);
set_price(price, trade);
}
}
if (HASFLAG(flags, MDRequest::MISC)) {
TimeStamp time(snapshot ? 0 : chan.time());
if (handle.misc_timestamp() > time) {
copy_misc(
handle.settlement_price(), handle.misc(),
pupd->mutable_misc());
}
}
}
if (pupd->has_direct() || pupd->has_implied() ||
pupd->has_last_trade() || pupd->has_misc() ||
pupd->has_security()) {
reply->add_updates()->CopyFrom(*pupd);
}
}
reply->mutable_channel()->set_segment_id(
group.channel.create_time().to_uint64());
bool is_same_segment =
(chan.segment_id() == group.channel.create_time().to_uint64());
if (HASFLAG(flags, MDRequest::TRADE_LOG)) {
int64_t request_cursor = is_same_segment ? chan.trade_cursor() : 0;
auto cursor = group.channel.read_trades(
*reply->mutable_trade_log(), request_cursor);
if (cursor != chan.trade_cursor()) {
reply->mutable_channel()->set_trade_cursor(cursor);
}
}
if (HASFLAG(flags, MDRequest::RFQ)) {
int64_t request_cursor = is_same_segment ? chan.rfq_cursor() : 0;
auto cursor =
group.channel.read_rfqs(*reply->mutable_rfq_log(), request_cursor);
if (cursor != chan.rfq_cursor()) {
reply->mutable_channel()->set_rfq_cursor(cursor);
}
}
}
void query(
const MDRequest &request,
MDReply *reply,
const Entitlements *entitlements,
bool bypass_entitlement) const {
bts_assert(reply != nullptr);
for (auto &ch : request.channels()) {
auto groupiter =
_groups.find(UserSymbol(ch.symbol(), ch.exchange(), ch.type()));
if (groupiter != _groups.end()) {
auto &group = groupiter->second;
fill(
ch, *group, reply->add_channels(), entitlements,
bypass_entitlement);
}
}
}
void get_names(ExpirationNameListReply *reply) const {
bts_assert(reply != nullptr);
Security security;
for (auto &groupiter : _groups) {
const auto &group = groupiter.second;
ExpirationNameList *psec = reply->add_securities();
psec->mutable_product()->CopyFrom(group->symbol.to_product());
for (const auto &h : *(group->handles)) {
security.CopyFrom(h.handle.get_normalized_security());
std::string desc;
if (!DomainSecurityUtils::get_security_desc(security, desc)) {
LOG(md, Error,
"QueryServerImpl::get_names exchange: {} not handled",
Venue_Name(security.exchange()));
continue;
}
psec->mutable_names()->insert({h.handle.key(), desc});
}
}
}
void get_securities(
google::protobuf::RepeatedField<security_key> keys,
api::SecurityList *result) const {
bts_assert(result);
// Ensure keys are unique
std::set<security_key> unique_keys(keys.begin(), keys.end());
for (auto k : unique_keys) {
auto ptr = _consumer.get_security(SecurityKey(k));
if (ptr) {
result->add_security()->CopyFrom(*ptr);
}
}
}
};
struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase {
const QueryServerImpl &_db;
utils::Dispatcher &_threadpool;
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>> _entitlements;
#ifndef _WIN32
std::shared_ptr<metrics::HistogramMetric<5>> _time, _size;
#endif
MDRelayServer(
const QueryServerImpl &db,
utils::Dispatcher &threadpool,
bts::api::ServiceContext ctx,
metrics::MetricsContainer &stats)
: bts::co2::AuthenticatedMarketDataRelayBase(ctx), _db(db),
_threadpool(threadpool), _entitlements()
#ifndef _WIN32
,
_time(std::make_shared<metrics::HistogramMetric<5>>(
"spark_qserver_response_time_ms",
std::array<double, 5>({5.0, 10.0, 20.0, 50.0, 100.0}))),
_size(std::make_shared<metrics::HistogramMetric<5>>(
"spark_qserver_size_bytes",
std::array<double, 5>(
{10240 /*10k*/, 102400 /*100k*/, 1024 * 1024 /*1mb*/,
1024 * 1024 * 10 /*10mb*/, 1024 * 1024 * 100 /*100mb*/}))) {
stats.publish(_time);
stats.publish(_size);
}
#else
{
}
#endif
void update_entitlements(
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>>
&entitlements) {
_entitlements = entitlements;
}
static const Entitlements *get_entitlements(
uint32_t user_id,
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>>
&entitlements) {
if (entitlements) {
auto iter = entitlements->find(user_id);
if (iter != entitlements->end()) {
return &iter->second;
}
}
return nullptr;
}
virtual void requestmd(
const api::AuthToken &token,
const MDRequest &request,
const std::function<void(const MDReply &)> &reply) override {
TimeStamp start = TimeStamp::now();
_threadpool.post([=, this]() {
google::protobuf::Arena arena;
MDReply *preply =
google::protobuf::Arena::CreateMessage<MDReply>(&arena);
// addref to keep shared ptr alive
api::user::TokenPayload payload;
auto ents = _entitlements;
const Entitlements *user_entitlement = nullptr;
bool bypass_entitlement = false;
if (api::auth::decode_token(token, &payload)) {
bypass_entitlement = payload.wheel();
if (!bypass_entitlement) {
user_entitlement = get_entitlements(payload.id(), ents);
}
}
// disabling md entitlements
_db.query(
request, preply, user_entitlement, true || bypass_entitlement);
reply(*preply);
#ifndef _WIN32
auto elapsed = (TimeStamp::now() - start);
_time->observe(elapsed.milliseconds());
_size->observe(preply->ByteSizeLong());
#endif
});
}
virtual void request_products(
const api::AuthToken &,
const api::VoidMessage &,
const std::function<void(const ProductListReply &)> &reply) override {
ProductListReply list;
const auto syms = SymbolLoader::load();
for (const auto &i : syms) {
auto prod = list.add_products();
prod->set_exchange(i.exchange());
prod->set_type(i.type());
prod->set_symbol(i.symbol());
}
reply(list);
}
virtual void request_names(
const api::AuthToken &,
const api::VoidMessage &,
const std::function<void(const ExpirationNameListReply &)> &reply)
override {
_threadpool.post([=, this]() {
google::protobuf::Arena arena;
ExpirationNameListReply *preply =
google::protobuf::Arena::CreateMessage<ExpirationNameListReply>(
&arena);
_db.get_names(preply);
reply(*preply);
});
}
virtual void get_securities(
const api::AuthToken &,
const SecurityKeyList &keys,
const std::function<void(const api::SecurityList &)> &reply) override {
_threadpool.post([=, this]() {
google::protobuf::Arena arena;
api::SecurityList *preply =
google::protobuf::Arena::CreateMessage<api::SecurityList>(&arena);
_db.get_securities(keys.keys(), preply);
reply(*preply);
});
}
};
struct QueryServer {
QueryServer(
utils::Dispatcher &threadpool,
const bts::api::ServiceContext &ctx,
MDClient &client,
metrics::MetricsContainer &stats)
: db(client, stats), relay(db, threadpool, ctx, stats),
admin_client(ctx.locate<api::admin::AdminService>()) {}
QueryServerImpl db;
MDRelayServer relay;
api::admin::AdminService admin_client;
rxcpp::composite_subscription entitlement_sub;
};
QueryServerPtr run_query_server(
const bts::api::ServiceContext &ctx,
MDClient &client,
metrics::MetricsContainer &stats) {
auto &threadpool = utils::Dispatcher::get_named("MDQuery");
QueryServerPtr container =
std::make_shared<QueryServer>(threadpool, ctx, client, stats);
// Subscribe to entitlement data
container->entitlement_sub =
container->admin_client.subscribe_md_entitlement_data().subscribe(
[=](const api::admin::Md_entitlement_dataUpdate &update) {
const auto &all_updates = update.all();
utils::Dispatcher::background().post([all_updates, container]() {
auto entitlements = std::make_shared<
std::unordered_map<uint32_t, Entitlements>>();
for (const auto &row : all_updates) {
if (row && row->enabled()) {
auto iter = entitlements->find(row->user_id());
if (iter == entitlements->end()) {
Entitlements ents;
ents.insert(row->mic());
entitlements->insert(
iter,
std::make_pair(row->user_id(), std::move(ents)));
} else {
iter->second.insert(row->mic());
}
}
}
container->relay.update_entitlements(entitlements);
});
});
// Add a few threads to keep md latency low
for (int x = 1; x < std::min(4, os::num_cores()); ++x) {
threadpool.add_thread();
}
return container;
}
std::unique_ptr<IQueryServer> create_query_server(
MDClient &mdc, metrics::MetricsContainer &mets) {
return std::make_unique<QueryServerImpl>(mdc, mets);
}
} // namespace md
} // namespace bts
#endif
@@ -1,43 +0,0 @@
#pragma once
#include "spark/md/ISyntheticSecurityService.h"
#include "spark/md/UserSymbol.h"
#include "spark/utils/bts_export.h"
#include "spark/utils/bts_no_copy.h"
#include <boost/container/flat_set.hpp>
#include <memory>
#include <set>
#include "spark/md/protos/co2.pb.h"
namespace bts {
namespace api {
class ServiceContext;
}
namespace metrics {
class MetricsContainer;
}
namespace md {
class MDClient;
typedef boost::container::flat_set<std::string> Entitlements;
// provides a polling interface on top of market data
struct IQueryServer {
virtual ~IQueryServer() {}
virtual void query(
const co2::MDRequest &request,
co2::MDReply *reply,
const Entitlements *,
bool) const = 0;
};
std::unique_ptr<IQueryServer> create_query_server(
MDClient &, metrics::MetricsContainer &);
struct QueryServer;
typedef std::shared_ptr<QueryServer> QueryServerPtr;
QueryServerPtr run_query_server(
const bts::api::ServiceContext &, MDClient &, metrics::MetricsContainer &);
} // namespace md
} // namespace bts
@@ -0,0 +1,10 @@
#pragma once
#include <boost/container/flat_set.hpp>
#include <string>
namespace bts::md {
typedef boost::container::flat_set<std::string> Entitlements;
} // namespace bts::md
@@ -0,0 +1,7 @@
#include "i_query_server.h"
namespace bts::md {
IQueryServer::~IQueryServer() = default;
} // namespace bts::md
@@ -0,0 +1,18 @@
#pragma once
#include "spark/md/protos/co2.pb.h"
#include "spark/md/relay/entitlements.h"
namespace bts::md {
struct IQueryServer {
virtual ~IQueryServer();
virtual void query(
const co2::MDRequest &request,
co2::MDReply *reply,
const Entitlements *entitlements,
bool bypass_entitlement) const = 0;
};
} // namespace bts::md
@@ -0,0 +1,13 @@
#pragma once
#include "spark/md/MDConsumerHandle.h"
#include "spark/utils/TimeStamp.h"
namespace bts::md {
struct DatedHandle {
utils::TimeStamp discovered;
MDConsumerHandle handle;
};
} // namespace bts::md
@@ -0,0 +1,141 @@
#include "md_consumer_group.h"
#include "spark/config/protos/cme_reference_data.pb.h"
namespace bts::md {
// probably just have to send the consumer in from outside
MDConsumerGroup::MDConsumerGroup(
const UserSymbol &symbol,
const MDConsumer &consumer,
metrics::MetricsContainer &stats)
: symbol(symbol), channel(consumer.get_channel(symbol)), handles(), mic() {
set_consumer(consumer);
// create and publish a stat for this group
#ifndef _WIN32
if (symbol.type() != SecurityType::SECURITYTYPE_SYNTHETIC) {
std::vector<metrics::Label> labels;
labels.push_back({"symbol", symbol.symbol()});
labels.push_back(
{"type",
SecurityType_Name(symbol.type()).substr(13) /*SECURITYTYPE_*/});
labels.push_back(
{"exch", Venue_Name(symbol.exchange()).substr(6) /*VENUE_*/});
stats.publish(std::make_shared<metrics::LazyCounterMetricL>(
"spark_md_book_updates",
[=, this]() {
return channel.direct_count() + channel.implied_count();
},
labels));
stats.publish(std::make_shared<metrics::LazyCounterMetricL>(
"spark_md_security_count",
[=, this]() { return channel.security_count(); }, labels));
stats.publish(std::make_shared<metrics::LazyCounterMetricL>(
"spark_md_trade_count", [=, this]() { return channel.trade_count(); },
labels));
stats.publish(std::make_shared<metrics::LazyCounterMetricD>(
"spark_md_gap_count",
[=, this]() {
auto g = channel.gap_count();
return g == 0 ? os::nan() : g;
},
labels));
}
#endif
}
// recovery has occured and we need to clear our our collection of handles
// and wait for new ones
void MDConsumerGroup::set_consumer(const MDConsumer &consumer) {
// create a new set of handles and swap with the old.
// If the new consumer is bad we still have to swap our handles because
// the consumer that owns our handles will be scheduled for deletion
auto temphandles =
std::make_shared<ConcurrentAppendCollection<DatedHandle>>();
handles.swap(temphandles);
channel = consumer.get_channel(symbol);
}
void MDConsumerGroup::add_security(MDConsumerHandle &handle) {
bts_assert(handles);
bts_assert_d(handle.is_initialized());
bts_assert(handle.get_usersymbol() == symbol);
DatedHandle h = {TimeStamp::now(), handle};
handles->push_back(h);
if (this->mic == "")
set_mic(h);
}
void MDConsumerGroup::set_mic(const DatedHandle &h) {
switch (channel.exchange()) {
case api::VENUE_ICE:
case api::VENUE_SIMICE:
this->mic = h.handle.get_normalized_security().ice().mic_code();
break;
case api::VENUE_GLBX:
case api::VENUE_SIM: {
const auto &cme_symb_mics =
config::get<config::CmeSymbolMics>().cme_mic();
for (auto mic = cme_symb_mics.begin(); mic != cme_symb_mics.end();
mic++) {
const auto &micsym = mic->first;
if (micsym == symbol.symbol()) {
this->mic = api::CMEMics_Name(mic->second);
break;
}
}
break;
}
case api::VENUE_BTS:
case api::VENUE_UNSET:
this->mic = "BTS";
break;
case api::VENUE_DERIBIT:
this->mic = "DERIBIT";
break;
case api::VENUE_FTX:
this->mic = "FTX";
break;
case api::VENUE_BINANCE:
this->mic = "BINANCE";
break;
case api::VENUE_BINANCE_COIN_FUTURES:
this->mic = "BINANCE_COIN_FUTURES";
break;
case api::VENUE_BINANCE_US:
this->mic = "BINANCE_US";
break;
case api::VENUE_COINBASE:
this->mic = "COINBASE";
break;
case api::VENUE_BINANCE_USDS_FUTURES:
this->mic = "BINANCE_USDS_FUTURES";
break;
case api::VENUE_OKX:
this->mic = "OKX";
break;
case api::VENUE_CBOE:
this->mic = "CBOE";
break;
case api::VENUE_TWSE:
this->mic = "TWSE";
break;
case api::VENUE_TAIFEX:
this->mic = "TAIFEX";
break;
case api::VENUE_BYBIT:
this->mic = "BYBIT";
break;
case api::VENUE_GATEIO:
this->mic = "GATEIO";
break;
case api::VENUE_CBOECFE:
case api::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
case api::Venue_INT_MAX_SENTINEL_DO_NOT_USE_:
bts_assert(false);
this->mic = "UNKNOWN";
break;
}
}
} // namespace bts::md
@@ -0,0 +1,37 @@
#pragma once
#include "spark/md/MDConsumer.h"
#include "spark/md/UserSymbol.h"
#include "spark/md/relay/internal/dated_handle.h"
#include "spark/metrics/metrics.h"
#include "spark/utils/ConcurrentAppendCollection.h"
namespace bts::md {
struct MDConsumerGroup {
// probably just have to send the consumer in from outside
MDConsumerGroup(
const UserSymbol &symbol,
const MDConsumer &consumer,
metrics::MetricsContainer &stats);
MDConsumerGroup(const MDConsumerGroup &other) = delete;
// recovery has occured and we need to clear our our collection of handles
// and wait for new ones
void set_consumer(const MDConsumer &consumer);
void add_security(MDConsumerHandle &handle);
void set_mic(const DatedHandle &h);
UserSymbol symbol;
MDConsumerChannel channel;
// Can add to handles without invalidating readers (no lock needed)
// When we recover we swap a new vector into this unique_ptr and
// delayed-delete
std::shared_ptr<ConcurrentAppendCollection<DatedHandle>> handles;
std::string mic;
};
} // namespace bts::md
@@ -0,0 +1,130 @@
#include "md_relay_server.h"
#include "spark/api/public/ServiceContext.h"
#include "spark/api/services/AuthService.pb.h"
#include "spark/md/SymbolLoader.h"
#include "spark/utils/AuthToken.h"
namespace bts::md {
using bts::api::auth::decode_token;
using bts::co2::ExpirationNameListReply;
using bts::co2::MDReply;
using bts::co2::MDRequest;
using bts::co2::ProductListReply;
using bts::co2::SecurityKeyList;
MDRelayServer::MDRelayServer(
const QueryServerImpl &db,
utils::Dispatcher &threadpool,
bts::api::ServiceContext ctx,
metrics::MetricsContainer &stats)
: bts::co2::AuthenticatedMarketDataRelayBase(ctx), _db(db),
_threadpool(threadpool), _entitlements()
#ifndef _WIN32
,
_time(std::make_shared<metrics::HistogramMetric<5>>(
"spark_qserver_response_time_ms",
std::array<double, 5>({5.0, 10.0, 20.0, 50.0, 100.0}))),
_size(std::make_shared<metrics::HistogramMetric<5>>(
"spark_qserver_size_bytes",
std::array<double, 5>(
{10240 /*10k*/, 102400 /*100k*/, 1024 * 1024 /*1mb*/,
1024 * 1024 * 10 /*10mb*/, 1024 * 1024 * 100 /*100mb*/}))) {
stats.publish(_time);
stats.publish(_size);
}
#else
{
}
#endif
void MDRelayServer::update_entitlements(
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>> &entitlements) {
_entitlements = entitlements;
}
const Entitlements *MDRelayServer::get_entitlements(
uint32_t user_id,
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>> &entitlements) {
if (entitlements) {
auto iter = entitlements->find(user_id);
if (iter != entitlements->end()) {
return &iter->second;
}
}
return nullptr;
}
void MDRelayServer::requestmd(
const api::AuthToken &token,
const MDRequest &request,
const std::function<void(const MDReply &)> &reply) {
TimeStamp start = TimeStamp::now();
_threadpool.post([=, this]() {
google::protobuf::Arena arena;
MDReply *preply = google::protobuf::Arena::CreateMessage<MDReply>(&arena);
// addref to keep shared ptr alive
api::user::TokenPayload payload;
auto ents = _entitlements;
const Entitlements *user_entitlement = nullptr;
bool bypass_entitlement = false;
if (decode_token(token, &payload)) {
bypass_entitlement = payload.wheel();
if (!bypass_entitlement) {
user_entitlement = get_entitlements(payload.id(), ents);
}
}
// disabling md entitlements
_db.query(request, preply, user_entitlement, true || bypass_entitlement);
reply(*preply);
#ifndef _WIN32
auto elapsed = (TimeStamp::now() - start);
_time->observe(elapsed.milliseconds());
_size->observe(preply->ByteSizeLong());
#endif
});
}
void MDRelayServer::request_products(
const api::AuthToken &,
const api::VoidMessage &,
const std::function<void(const ProductListReply &)> &reply) {
ProductListReply list;
const auto syms = SymbolLoader::load();
for (const auto &i : syms) {
auto prod = list.add_products();
prod->set_exchange(i.exchange());
prod->set_type(i.type());
prod->set_symbol(i.symbol());
}
reply(list);
}
void MDRelayServer::request_names(
const api::AuthToken &,
const api::VoidMessage &,
const std::function<void(const ExpirationNameListReply &)> &reply) {
_threadpool.post([=, this]() {
google::protobuf::Arena arena;
ExpirationNameListReply *preply =
google::protobuf::Arena::CreateMessage<ExpirationNameListReply>(
&arena);
_db.get_names(preply);
reply(*preply);
});
}
void MDRelayServer::get_securities(
const api::AuthToken &,
const SecurityKeyList &keys,
const std::function<void(const api::SecurityList &)> &reply) {
_threadpool.post([=, this]() {
google::protobuf::Arena arena;
api::SecurityList *preply =
google::protobuf::Arena::CreateMessage<api::SecurityList>(&arena);
_db.get_securities(keys.keys(), preply);
reply(*preply);
});
}
} // namespace bts::md
@@ -0,0 +1,54 @@
#pragma once
#include "spark/api/services/MarketDataRelay.server.pb.h"
#include "spark/md/relay/internal/query_server_impl.h"
namespace bts::md {
struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase {
const QueryServerImpl &_db;
utils::Dispatcher &_threadpool;
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>> _entitlements;
#ifndef _WIN32
std::shared_ptr<metrics::HistogramMetric<5>> _time, _size;
#endif
MDRelayServer(
const QueryServerImpl &db,
utils::Dispatcher &threadpool,
bts::api::ServiceContext ctx,
metrics::MetricsContainer &stats);
void update_entitlements(
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>>
&entitlements);
static const Entitlements *get_entitlements(
uint32_t user_id,
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>>
&entitlements);
virtual void requestmd(
const api::AuthToken &token,
const co2::MDRequest &request,
const std::function<void(const co2::MDReply &)> &reply) override;
virtual void request_products(
const api::AuthToken &,
const api::VoidMessage &,
const std::function<void(const co2::ProductListReply &)> &reply)
override;
virtual void request_names(
const api::AuthToken &,
const api::VoidMessage &,
const std::function<void(const co2::ExpirationNameListReply &)> &reply)
override;
virtual void get_securities(
const api::AuthToken &,
const co2::SecurityKeyList &keys,
const std::function<void(const api::SecurityList &)> &reply) override;
};
} // namespace bts::md
@@ -0,0 +1,360 @@
#include "query_server_impl.h"
#include "spark/domain/DomainSecurityUtils.h"
#include "spark/domain_model/Security.h"
#include "spark/md/relay/internal/md_consumer_group.h"
#include "spark/utils/helper_macros.h"
namespace bts::md {
using bts::co2::ChannelReply;
using bts::co2::ChannelUpdate;
using bts::co2::ExpirationNameList;
using bts::co2::ExpirationNameListReply;
using bts::co2::MDReply;
using bts::co2::MDRequest;
using bts::co2::PriceSizeMessage;
using bts::co2::SecurityChannel;
using bts::domain::DomainSecurityUtils;
static void convert(PriceSize ps, PriceSizeMessage *pps) {
pps->set_mantissa(ps.mantissa());
pps->set_exponent(ps.exponent());
pps->set_size(ps.size());
}
static void set_price(PriceSize ps, co2::TradeMessage *ptrade) {
ptrade->set_mantissa(ps.mantissa());
ptrade->set_exponent(ps.exponent());
ptrade->set_size(ps.size());
}
static void set_price(const IntPrice &price, co2::PriceMessage *dest) {
dest->set_mantissa(price.mantissa());
dest->set_exponent(price.exponent());
}
static void copy_misc(
IntPrice settlement, MiscData misc, co2::MiscMessage *pmisc) {
set_price(settlement, pmisc->mutable_settlement());
pmisc->set_settlement_date(misc.settlement_date.to_short());
set_price(
misc.previous_settlement_price,
pmisc->mutable_previous_settlement_price());
pmisc->set_previous_settlement_date(
misc.previous_settlement_date.to_short());
pmisc->set_open_price(misc.open_price);
pmisc->set_open_date(misc.open_date.to_short());
pmisc->set_close_price(misc.close_price);
pmisc->set_close_date(misc.close_date.to_short());
pmisc->set_fixing_price(misc.fixing_price);
pmisc->set_session_high(misc.session_high);
pmisc->set_session_low(misc.session_low);
// pmisc->set_session_high_bid(misc.session_high_bid);
// pmisc->set_session_low_offer(misc.session_low_offer);
}
// Handles market data queries and responds
QueryServerImpl::QueryServerImpl(
MDClient &client, metrics::MetricsContainer &stats)
: _client(client), _consumer(client.current_consumer()), _sub(0),
_groups() {
for (const auto &sym : client.symbols()) {
_groups.emplace(std::make_pair(
sym, std::make_unique<MDConsumerGroup>(sym, _consumer, stats)));
}
_groups.shrink_to_fit();
_sub = client.on_consumer().subscribe(
[=, this](const MDConsumer &c) { handle_recovery(c); });
subscribe_securities();
}
QueryServerImpl::~QueryServerImpl() { _client.on_consumer().unsubscribe(_sub); }
void QueryServerImpl::subscribe_securities() {
LOG(md, Info, "QueryServer::subscribe_securities");
auto id = _consumer.id();
_consumer.on_security(
[this, id](const SecurityPtr &sec, MDConsumerHandle handle) {
if (_consumer.id() == id) {
auto iter = _groups.find(handle.get_usersymbol());
bts_assert(iter != _groups.end());
const auto &pgroup = iter->second;
bts_assert(pgroup);
LOG(md, Info,
"QueryServer::_consumer.on_security symbol:{} "
"sec:{}",
handle.get_usersymbol().symbol(),
bts::securities::descriptor(*sec));
pgroup->add_security(handle);
}
});
}
// not-reentrant
void QueryServerImpl::handle_recovery(const MDConsumer &consumer) {
MDConsumer temp(consumer);
if (temp.all_heartbeats_ok()) {
_consumer.swap(temp);
LOG(md, Info,
"Recovering from bad MD. Old consumer {} new consumer "
"{}",
_consumer.id(), temp.id());
for (auto &pair : _groups) {
pair.second->set_consumer(_consumer);
}
// _handle_index.clear();
subscribe_securities();
utils::Dispatcher::background().post(
[temp]() {
LOG(md, Info, "QServer destroying consumer {}", temp.id());
},
60000);
}
}
// the hot part
void QueryServerImpl::fill(
const SecurityChannel &chan,
MDConsumerGroup &group,
ChannelReply *reply,
const Entitlements *entitlements,
bool bypass_entitlement) const {
bts_assert(reply);
reply->mutable_channel()->CopyFrom(chan);
reply->mutable_channel()->set_time(TimeStamp::now().to_uint64());
bool entitled = bypass_entitlement || chan.exchange() == api::VENUE_BTS;
if (entitlements && !entitled)
entitled = entitlements->count(group.mic);
reply->mutable_channel()->set_entitled(entitled);
auto flags = (MDRequest::Flags)chan.flags();
// Get a copy of handles. If recovery happens during this call
// we can safely continue using this pointer for 60 seconds
auto phandles = group.handles;
if (!phandles)
return;
ChannelUpdate temp_update;
for (const auto &instrument : *phandles) {
bool snapshot = false;
MDConsumerHandle handle = instrument.handle;
temp_update.Clear();
bts_assert_d(handle.is_initialized());
if (!handle.is_initialized())
continue;
ChannelUpdate *pupd = &temp_update;
pupd->set_key(instrument.handle.key());
// If the security was just discovered try to ensure all
// data is sent
if (instrument.discovered > TimeStamp(chan.time())) {
auto security = handle.get_normalized_security();
pupd->mutable_security()->CopyFrom(security);
snapshot = true;
}
// if we are not entitled AND this is the first request
// since we lost our entitlement, zero out books
if (!entitled && chan.entitled()) {
if (HASFLAG(flags, MDRequest::BBO) ||
HASFLAG(flags, MDRequest::DIRECT)) {
auto *p = pupd->mutable_direct();
for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) {
convert(NO_MARKET, p->add_bids());
convert(NO_MARKET, p->add_asks());
}
}
if ((HASFLAG(flags, MDRequest::IMPLIED) ||
HASFLAG(flags, MDRequest::BBO)) &&
!handle.is_synthetic()) {
auto *p = pupd->mutable_implied();
for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) {
convert(NO_MARKET, p->add_bids());
convert(NO_MARKET, p->add_asks());
}
}
}
if (entitled) {
// If we are entitled now but this is the first request
// where we are ensure we send all values
if (!chan.entitled())
snapshot = true;
if (HASFLAG(flags, MDRequest::BBO) ||
HASFLAG(flags, MDRequest::DIRECT)) {
std::array<PriceSize, COCONUT_MAX_DEPTH> bids, asks;
TimeStamp time(snapshot ? 0 : chan.time());
size_t copy_depth =
HASFLAG(flags, MDRequest::DIRECT) ? COCONUT_MAX_DEPTH : 1;
auto result = handle.get_direct_book(
time, copy_depth, bids.data(), asks.data());
if (result == HandleReadResult::SUCCESS) {
auto *p = pupd->mutable_direct();
for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) {
PriceSize b = bids[x];
if (b != NO_MARKET) {
convert(b, p->add_bids());
}
PriceSize a = asks[x];
if (a != NO_MARKET) {
convert(a, p->add_asks());
}
}
} else if (result == HandleReadResult::BUSY) {
LOG(md, Info,
"Busy read on handle {}. Retry on next "
"request",
instrument.handle.key());
// If we get a busy reading BBO (which happens
// 1-2x per day for a single relay client) We move
// the timestamp back so the next request will try
// to read this value again. ALTERNATIVE: We could
// spin until the value was not busy but it could
// be that the writer was context switched out and
// we will have to wait multiple ms to unblock the
// request. This fix assumes it is better to send
// the update now with 1 stale value than to block
// the call and allow all the values to be stale
// until this one can be read.
reply->mutable_channel()->set_time(chan.time());
}
}
if ((HASFLAG(flags, MDRequest::IMPLIED) ||
HASFLAG(flags, MDRequest::BBO)) &&
!handle.is_synthetic()) {
std::array<PriceSize, COCONUT_MAX_IMPLIED_DEPTH> bids, asks;
TimeStamp time(snapshot ? 0 : chan.time());
size_t copy_depth = HASFLAG(flags, MDRequest::IMPLIED)
? COCONUT_MAX_IMPLIED_DEPTH
: 1;
auto result = handle.get_implied_book(
time, copy_depth, bids.data(), asks.data());
if (result == HandleReadResult::SUCCESS) {
auto *p = pupd->mutable_implied();
for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) {
PriceSize b = bids[x];
if (b != NO_MARKET) {
convert(b, p->add_bids());
}
PriceSize a = asks[x];
if (a != NO_MARKET) {
convert(a, p->add_asks());
}
}
} else if (result == HandleReadResult::BUSY) {
LOG(md, Info,
"Busy read on handle (implied) {}. Retry on "
"next request",
instrument.handle.key());
reply->mutable_channel()->set_time(chan.time());
}
}
if (HASFLAG(flags, MDRequest::LAST_TRADE)) {
TimeStamp time(snapshot ? 0 : chan.time());
PriceSize price;
datetime::LLDateTime trade_time;
size_t volume;
int from_spread;
if (handle.get_last_trade(
time, price, trade_time, &volume, &from_spread)) {
auto *trade = pupd->mutable_last_trade();
trade->set_key(handle.key());
trade->set_time(trade_time);
trade->set_total_volume(volume);
trade->set_from_spread(from_spread);
set_price(price, trade);
}
}
if (HASFLAG(flags, MDRequest::MISC)) {
TimeStamp time(snapshot ? 0 : chan.time());
if (handle.misc_timestamp() > time) {
copy_misc(
handle.settlement_price(), handle.misc(),
pupd->mutable_misc());
}
}
}
if (pupd->has_direct() || pupd->has_implied() || pupd->has_last_trade() ||
pupd->has_misc() || pupd->has_security()) {
reply->add_updates()->CopyFrom(*pupd);
}
}
reply->mutable_channel()->set_segment_id(
group.channel.create_time().to_uint64());
bool is_same_segment =
(chan.segment_id() == group.channel.create_time().to_uint64());
if (HASFLAG(flags, MDRequest::TRADE_LOG)) {
int64_t request_cursor = is_same_segment ? chan.trade_cursor() : 0;
auto cursor = group.channel.read_trades(
*reply->mutable_trade_log(), request_cursor);
if (cursor != chan.trade_cursor()) {
reply->mutable_channel()->set_trade_cursor(cursor);
}
}
if (HASFLAG(flags, MDRequest::RFQ)) {
int64_t request_cursor = is_same_segment ? chan.rfq_cursor() : 0;
auto cursor =
group.channel.read_rfqs(*reply->mutable_rfq_log(), request_cursor);
if (cursor != chan.rfq_cursor()) {
reply->mutable_channel()->set_rfq_cursor(cursor);
}
}
}
void QueryServerImpl::query(
const MDRequest &request,
MDReply *reply,
const Entitlements *entitlements,
bool bypass_entitlement) const {
bts_assert(reply != nullptr);
for (auto &ch : request.channels()) {
auto groupiter =
_groups.find(UserSymbol(ch.symbol(), ch.exchange(), ch.type()));
if (groupiter != _groups.end()) {
auto &group = groupiter->second;
fill(
ch, *group, reply->add_channels(), entitlements,
bypass_entitlement);
}
}
}
void QueryServerImpl::get_names(ExpirationNameListReply *reply) const {
bts_assert(reply != nullptr);
Security security;
for (auto &groupiter : _groups) {
const auto &group = groupiter.second;
ExpirationNameList *psec = reply->add_securities();
psec->mutable_product()->CopyFrom(group->symbol.to_product());
for (const auto &h : *(group->handles)) {
security.CopyFrom(h.handle.get_normalized_security());
std::string desc;
if (!DomainSecurityUtils::get_security_desc(security, desc)) {
LOG(md, Error,
"QueryServerImpl::get_names exchange: {} not handled",
Venue_Name(security.exchange()));
continue;
}
psec->mutable_names()->insert({h.handle.key(), desc});
}
}
}
void QueryServerImpl::get_securities(
google::protobuf::RepeatedField<security_key> keys,
api::SecurityList *result) const {
bts_assert(result);
// Ensure keys are unique
std::set<security_key> unique_keys(keys.begin(), keys.end());
for (auto k : unique_keys) {
auto ptr = _consumer.get_security(SecurityKey(k));
if (ptr) {
result->add_security()->CopyFrom(*ptr);
}
}
}
} // namespace bts::md
@@ -0,0 +1,52 @@
#pragma once
#include "spark/api/services/MarketDataRelay.pb.h"
#include "spark/md/MDClient.h"
#include "spark/md/relay/i_query_server.h"
#include "spark/md/relay/internal/md_consumer_group.h"
#include "spark/metrics/metrics.h"
#include <boost/container/flat_map.hpp>
namespace bts::md {
class QueryServerImpl : public IQueryServer {
MDClient &_client;
MDConsumer _consumer;
int _sub;
boost::container::flat_map<UserSymbol, std::shared_ptr<MDConsumerGroup>>
_groups;
public:
QueryServerImpl(MDClient &client, metrics::MetricsContainer &stats);
QueryServerImpl(const QueryServerImpl &) = delete;
~QueryServerImpl();
void subscribe_securities();
// not-reentrant
void handle_recovery(const MDConsumer &consumer);
// the hot part
void fill(
const co2::SecurityChannel &chan,
MDConsumerGroup &group,
co2::ChannelReply *reply,
const Entitlements *entitlements,
bool bypass_entitlement) const;
void query(
const co2::MDRequest &request,
co2::MDReply *reply,
const Entitlements *entitlements,
bool bypass_entitlement) const;
void get_names(co2::ExpirationNameListReply *reply) const;
void get_securities(
google::protobuf::RepeatedField<security_key> keys,
api::SecurityList *result) const;
};
} // namespace bts::md
@@ -0,0 +1,114 @@
#ifndef _WIN32
#include "query_server.h"
#include "spark/utils/Dispatcher.h"
#include "../MDClient.h"
#include "spark/api/public/ServiceContext.h"
#include "spark/api/services/AdminService.client.pb.h"
#include "spark/api/services/MarketDataRelay.server.pb.h"
#include "spark/md/relay/internal/md_relay_server.h"
#ifndef _WIN32
#include "spark/metrics/metrics.h"
#endif
#include <memory>
#include <unordered_map>
#include <boost/container/flat_map.hpp>
#include <google/protobuf/arena.h>
using namespace bts::co2;
using std::string;
using std::unique_ptr;
namespace bts {
namespace md {
/*
* NORMAL OPERATION:
* Client sends time=0 to server
* Server sends complete snapshot back (time=502)
* client sends time=502 to server
* Server sends any changes that have happened after time=502 and any securities
* discovered since time=502
*
* ERROR RECOVERY:
* Client sends time=0 to server
* Server sends complete snapshot back (time=502)
* Server detects heartbeat failure and recovers. All timestamps > 502
* Client sends time=502 to server
* Server sends complete snapshot back (time=605)
* - Since the server shared memory has been recreated the client will
* automatically overwrite all stale data.
* - If a security has been deleted the client will leave it frozen (OK)
*
* We scan for bad heartbeats on a different time
* If a bad heartbeat is detected we try to construct a new Consumer
* When a good consumer is created we std::move the old consumer and handles to
* delayed queue for deletion This way we don't have to lock anything during the
* recovery process because all pointers will remain valid for the duration of
* existing calls.
*
*/
QueryServer::QueryServer(
utils::Dispatcher &threadpool,
const bts::api::ServiceContext &ctx,
MDClient &client,
metrics::MetricsContainer &stats)
: db(client, stats), relay(db, threadpool, ctx, stats),
admin_client(ctx.locate<api::admin::AdminService>()) {}
QueryServerPtr run_query_server(
const bts::api::ServiceContext &ctx,
MDClient &client,
metrics::MetricsContainer &stats) {
auto &threadpool = utils::Dispatcher::get_named("MDQuery");
QueryServerPtr container =
std::make_shared<QueryServer>(threadpool, ctx, client, stats);
// Subscribe to entitlement data
container->entitlement_sub =
container->admin_client.subscribe_md_entitlement_data().subscribe(
[=](const api::admin::Md_entitlement_dataUpdate &update) {
const auto &all_updates = update.all();
utils::Dispatcher::background().post([all_updates, container]() {
auto entitlements = std::make_shared<
std::unordered_map<uint32_t, Entitlements>>();
for (const auto &row : all_updates) {
if (row && row->enabled()) {
auto iter = entitlements->find(row->user_id());
if (iter == entitlements->end()) {
Entitlements ents;
ents.insert(row->mic());
entitlements->insert(
iter,
std::make_pair(row->user_id(), std::move(ents)));
} else {
iter->second.insert(row->mic());
}
}
}
container->relay.update_entitlements(entitlements);
});
});
// Add a few threads to keep md latency low
for (int x = 1; x < std::min(4, os::num_cores()); ++x) {
threadpool.add_thread();
}
return container;
}
std::unique_ptr<IQueryServer> create_query_server(
MDClient &mdc, metrics::MetricsContainer &mets) {
return std::make_unique<QueryServerImpl>(mdc, mets);
}
} // namespace md
} // namespace bts
#endif
@@ -0,0 +1,32 @@
#pragma once
#include "spark/api/services/AdminService.client.pb.h"
#include "spark/md/relay/internal/md_relay_server.h"
#include "spark/md/relay/internal/query_server_impl.h"
#include "spark/metrics/metrics.h"
namespace bts::md {
struct QueryServer {
QueryServer(
utils::Dispatcher &threadpool,
const api::ServiceContext &ctx,
MDClient &client,
metrics::MetricsContainer &stats);
QueryServerImpl db;
MDRelayServer relay;
api::admin::AdminService admin_client;
rxcpp::composite_subscription entitlement_sub;
};
typedef std::shared_ptr<QueryServer> QueryServerPtr;
class MDClient;
std::unique_ptr<IQueryServer> create_query_server(
MDClient &, metrics::MetricsContainer &);
QueryServerPtr run_query_server(
const bts::api::ServiceContext &, MDClient &, metrics::MetricsContainer &);
} // namespace bts::md
+1 -1
View File
@@ -3,9 +3,9 @@
#include "../SecurityCache.h"
#include "../SettlementRecorder.h"
#include "../coconut2.h"
#include "../relay/QueryServer.h"
#include "Co2TestInstrument.h"
#include "spark/config/config.h"
#include "spark/md/relay/query_server.h"
#include "spark/metrics/metrics.h"
#include "spark/os/os.h"
#include "spark/persistence/SqliteStore.h"