refactor: Split MDRelay and QueryServer up into smaller files
This code is inherently complicated. It's hard to follow when everything is in one giant file.
This commit is contained in:
1 parent
35d65f107b
commit
d7b42e3cee
19 files changed
+980
-817
No files matched your search
@@ -2,6 +2,7 @@ services:
|
||||
sparkd:
|
||||
volumes:
|
||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
|
||||
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
|
||||
build:
|
||||
dockerfile: src/app_sparkd/Dockerfile
|
||||
args:
|
||||
@@ -161,3 +162,6 @@ volumes:
|
||||
name: "md_publisher_22019"
|
||||
historic_md:
|
||||
name: "historic_md"
|
||||
md_recording:
|
||||
name: "md_recording"
|
||||
external: true
|
||||
@@ -10,7 +10,7 @@
|
||||
#include "spark/md/MBOServiceServer.h"
|
||||
#include "spark/md/MDClient.h"
|
||||
#include "spark/md/SyntheticSecurityService.h"
|
||||
#include "spark/md/relay/QueryServer.h"
|
||||
#include "spark/md/relay/query_server.h"
|
||||
#include "spark/md/service/MarketDataServiceServer.h"
|
||||
#include "spark/metrics/metrics.h"
|
||||
#include "spark/metrics/system_metrics.h"
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
#include "spark/config/protos/quickbus_cfg.pb.h"
|
||||
#include "spark/md/MDClient.h"
|
||||
#include "spark/md/SyntheticSecurityService.h"
|
||||
#include "spark/md/relay/QueryServer.h"
|
||||
#include "spark/md/relay/query_server.h"
|
||||
#include "spark/metrics/metrics.h"
|
||||
#include "spark/metrics/system_metrics.h"
|
||||
#include "spark/quickbus/core/Quickbus.h"
|
||||
|
||||
@@ -91,7 +91,11 @@ function(get_cpp_sources dest)
|
||||
${PROJECT_NAME}/md/cme/MDP3Utils.cpp
|
||||
${PROJECT_NAME}/md/common.cpp
|
||||
${PROJECT_NAME}/md/ice/ICEUtils.cpp
|
||||
${PROJECT_NAME}/md/relay/QueryServer.cpp
|
||||
${PROJECT_NAME}/md/relay/internal/md_consumer_group.cpp
|
||||
${PROJECT_NAME}/md/relay/internal/md_relay_server.cpp
|
||||
${PROJECT_NAME}/md/relay/internal/query_server_impl.cpp
|
||||
${PROJECT_NAME}/md/relay/i_query_server.cpp
|
||||
${PROJECT_NAME}/md/relay/query_server.cpp
|
||||
${PROJECT_NAME}/md/relay/RelayClient.cpp
|
||||
${PROJECT_NAME}/md/relay/RelayRecorder.cpp
|
||||
${PROJECT_NAME}/md/service/MarketDataServiceServer.cpp
|
||||
|
||||
@@ -1,770 +0,0 @@
|
||||
#ifndef _WIN32
|
||||
#include "QueryServer.h"
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/domain/DomainSecurityUtils.h"
|
||||
#include "spark/domain_model/Security.h"
|
||||
#include "spark/md/SymbolLoader.h"
|
||||
#include "spark/utils/AuthToken.h"
|
||||
#include "spark/utils/Dispatcher.h"
|
||||
#include "spark/utils/TimeStamp.h"
|
||||
#include "spark/utils/bts_assert.h"
|
||||
#include "spark/utils/helper_macros.h"
|
||||
|
||||
#include "../MBOProcessor.h"
|
||||
#include "../MDClient.h"
|
||||
#include "../MDConsumer.h"
|
||||
|
||||
#include "spark/logging/logging.h"
|
||||
|
||||
#include "spark/api/apicore.pb.h"
|
||||
#include "spark/api/public/ServiceContext.h"
|
||||
#include "spark/api/service_helpers/AdminServiceUtils.h"
|
||||
#include "spark/api/services/AdminService.client.pb.h"
|
||||
#include "spark/api/services/MarketDataRelay.server.pb.h"
|
||||
#include "spark/config/protos/cme_reference_data.pb.h"
|
||||
#include "spark/config/protos/system_config.pb.h"
|
||||
#include "spark/md/SecurityCache.h"
|
||||
#include "spark/md/protos/co2.pb.h"
|
||||
#include "spark/utils/ConcurrentAppendCollection.h"
|
||||
#ifndef _WIN32
|
||||
#include "spark/metrics/metrics.h"
|
||||
#endif
|
||||
|
||||
#include <list>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/container/flat_map.hpp>
|
||||
#include <google/protobuf/arena.h>
|
||||
|
||||
using namespace bts::co2;
|
||||
using bts::domain::DomainSecurityUtils;
|
||||
using std::shared_ptr;
|
||||
using std::string;
|
||||
using std::unique_ptr;
|
||||
|
||||
namespace bts {
|
||||
namespace md {
|
||||
|
||||
/*
|
||||
* NORMAL OPERATION:
|
||||
* Client sends time=0 to server
|
||||
* Server sends complete snapshot back (time=502)
|
||||
* client sends time=502 to server
|
||||
* Server sends any changes that have happened after time=502 and any securities
|
||||
* discovered since time=502
|
||||
*
|
||||
* ERROR RECOVERY:
|
||||
* Client sends time=0 to server
|
||||
* Server sends complete snapshot back (time=502)
|
||||
* Server detects heartbeat failure and recovers. All timestamps > 502
|
||||
* Client sends time=502 to server
|
||||
* Server sends complete snapshot back (time=605)
|
||||
* - Since the server shared memory has been recreated the client will
|
||||
* automatically overwrite all stale data.
|
||||
* - If a security has been deleted the client will leave it frozen (OK)
|
||||
*
|
||||
* We scan for bad heartbeats on a different time
|
||||
* If a bad heartbeat is detected we try to construct a new Consumer
|
||||
* When a good consumer is created we std::move the old consumer and handles to
|
||||
* delayed queue for deletion This way we don't have to lock anything during the
|
||||
* recovery process because all pointers will remain valid for the duration of
|
||||
* existing calls.
|
||||
*
|
||||
*/
|
||||
struct DatedHandle {
|
||||
TimeStamp discovered;
|
||||
MDConsumerHandle handle;
|
||||
};
|
||||
|
||||
struct MDConsumerGroup {
|
||||
// probably just have to send the consumer in from outside
|
||||
MDConsumerGroup(
|
||||
const UserSymbol &symbol,
|
||||
const MDConsumer &consumer,
|
||||
metrics::MetricsContainer &stats)
|
||||
: symbol(symbol), channel(consumer.get_channel(symbol)), handles(),
|
||||
mic() {
|
||||
set_consumer(consumer);
|
||||
|
||||
// create and publish a stat for this group
|
||||
#ifndef _WIN32
|
||||
if (symbol.type() != SecurityType::SECURITYTYPE_SYNTHETIC) {
|
||||
std::vector<metrics::Label> labels;
|
||||
labels.push_back({"symbol", symbol.symbol()});
|
||||
labels.push_back(
|
||||
{"type",
|
||||
SecurityType_Name(symbol.type()).substr(13) /*SECURITYTYPE_*/});
|
||||
labels.push_back(
|
||||
{"exch", Venue_Name(symbol.exchange()).substr(6) /*VENUE_*/});
|
||||
stats.publish(std::make_shared<metrics::LazyCounterMetricL>(
|
||||
"spark_md_book_updates",
|
||||
[=, this]() {
|
||||
return channel.direct_count() + channel.implied_count();
|
||||
},
|
||||
labels));
|
||||
stats.publish(std::make_shared<metrics::LazyCounterMetricL>(
|
||||
"spark_md_security_count",
|
||||
[=, this]() { return channel.security_count(); }, labels));
|
||||
stats.publish(std::make_shared<metrics::LazyCounterMetricL>(
|
||||
"spark_md_trade_count",
|
||||
[=, this]() { return channel.trade_count(); }, labels));
|
||||
stats.publish(std::make_shared<metrics::LazyCounterMetricD>(
|
||||
"spark_md_gap_count",
|
||||
[=, this]() {
|
||||
auto g = channel.gap_count();
|
||||
return g == 0 ? os::nan() : g;
|
||||
},
|
||||
labels));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
MDConsumerGroup(const MDConsumerGroup &other) = delete;
|
||||
|
||||
// recovery has occured and we need to clear our our collection of handles
|
||||
// and wait for new ones
|
||||
void set_consumer(const MDConsumer &consumer) {
|
||||
// create a new set of handles and swap with the old.
|
||||
// If the new consumer is bad we still have to swap our handles because
|
||||
// the consumer that owns our handles will be scheduled for deletion
|
||||
auto temphandles =
|
||||
std::make_shared<ConcurrentAppendCollection<DatedHandle>>();
|
||||
handles.swap(temphandles);
|
||||
channel = consumer.get_channel(symbol);
|
||||
}
|
||||
|
||||
void add_security(MDConsumerHandle &handle) {
|
||||
bts_assert(handles);
|
||||
bts_assert_d(handle.is_initialized());
|
||||
bts_assert(handle.get_usersymbol() == symbol);
|
||||
DatedHandle h = {TimeStamp::now(), handle};
|
||||
handles->push_back(h);
|
||||
if (this->mic == "")
|
||||
set_mic(h);
|
||||
}
|
||||
|
||||
void set_mic(const DatedHandle &h) {
|
||||
switch (channel.exchange()) {
|
||||
case api::VENUE_ICE:
|
||||
case api::VENUE_SIMICE:
|
||||
this->mic = h.handle.get_normalized_security().ice().mic_code();
|
||||
break;
|
||||
case api::VENUE_GLBX:
|
||||
case api::VENUE_SIM: {
|
||||
const auto &cme_symb_mics =
|
||||
config::get<config::CmeSymbolMics>().cme_mic();
|
||||
for (auto mic = cme_symb_mics.begin(); mic != cme_symb_mics.end();
|
||||
mic++) {
|
||||
const auto &micsym = mic->first;
|
||||
if (micsym == symbol.symbol()) {
|
||||
this->mic = api::CMEMics_Name(mic->second);
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case api::VENUE_BTS:
|
||||
case api::VENUE_UNSET:
|
||||
this->mic = "BTS";
|
||||
break;
|
||||
case api::VENUE_DERIBIT:
|
||||
this->mic = "DERIBIT";
|
||||
break;
|
||||
case api::VENUE_FTX:
|
||||
this->mic = "FTX";
|
||||
break;
|
||||
case api::VENUE_BINANCE:
|
||||
this->mic = "BINANCE";
|
||||
break;
|
||||
case api::VENUE_BINANCE_COIN_FUTURES:
|
||||
this->mic = "BINANCE_COIN_FUTURES";
|
||||
break;
|
||||
case api::VENUE_BINANCE_US:
|
||||
this->mic = "BINANCE_US";
|
||||
break;
|
||||
case api::VENUE_COINBASE:
|
||||
this->mic = "COINBASE";
|
||||
break;
|
||||
case api::VENUE_BINANCE_USDS_FUTURES:
|
||||
this->mic = "BINANCE_USDS_FUTURES";
|
||||
break;
|
||||
case api::VENUE_OKX:
|
||||
this->mic = "OKX";
|
||||
break;
|
||||
case api::VENUE_CBOE:
|
||||
this->mic = "CBOE";
|
||||
break;
|
||||
case api::VENUE_TWSE:
|
||||
this->mic = "TWSE";
|
||||
break;
|
||||
case api::VENUE_TAIFEX:
|
||||
this->mic = "TAIFEX";
|
||||
break;
|
||||
case api::VENUE_BYBIT:
|
||||
this->mic = "BYBIT";
|
||||
break;
|
||||
case api::VENUE_GATEIO:
|
||||
this->mic = "GATEIO";
|
||||
break;
|
||||
case api::VENUE_CBOECFE:
|
||||
case api::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
|
||||
case api::Venue_INT_MAX_SENTINEL_DO_NOT_USE_:
|
||||
bts_assert(false);
|
||||
this->mic = "UNKNOWN";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
UserSymbol symbol;
|
||||
MDConsumerChannel channel;
|
||||
|
||||
// Can add to handles without invalidating readers (no lock needed)
|
||||
// When we recover we swap a new vector into this unique_ptr and
|
||||
// delayed-delete
|
||||
shared_ptr<ConcurrentAppendCollection<DatedHandle>> handles;
|
||||
std::string mic;
|
||||
};
|
||||
|
||||
void convert(PriceSize ps, PriceSizeMessage *pps) {
|
||||
pps->set_mantissa(ps.mantissa());
|
||||
pps->set_exponent(ps.exponent());
|
||||
pps->set_size(ps.size());
|
||||
}
|
||||
void set_price(PriceSize ps, co2::TradeMessage *ptrade) {
|
||||
ptrade->set_mantissa(ps.mantissa());
|
||||
ptrade->set_exponent(ps.exponent());
|
||||
ptrade->set_size(ps.size());
|
||||
}
|
||||
|
||||
void set_price(const IntPrice &price, co2::PriceMessage *dest) {
|
||||
dest->set_mantissa(price.mantissa());
|
||||
dest->set_exponent(price.exponent());
|
||||
}
|
||||
|
||||
void copy_misc(IntPrice settlement, MiscData misc, co2::MiscMessage *pmisc) {
|
||||
set_price(settlement, pmisc->mutable_settlement());
|
||||
pmisc->set_settlement_date(misc.settlement_date.to_short());
|
||||
set_price(
|
||||
misc.previous_settlement_price,
|
||||
pmisc->mutable_previous_settlement_price());
|
||||
pmisc->set_previous_settlement_date(
|
||||
misc.previous_settlement_date.to_short());
|
||||
pmisc->set_open_price(misc.open_price);
|
||||
pmisc->set_open_date(misc.open_date.to_short());
|
||||
pmisc->set_close_price(misc.close_price);
|
||||
pmisc->set_close_date(misc.close_date.to_short());
|
||||
pmisc->set_fixing_price(misc.fixing_price);
|
||||
pmisc->set_session_high(misc.session_high);
|
||||
pmisc->set_session_low(misc.session_low);
|
||||
// pmisc->set_session_high_bid(misc.session_high_bid);
|
||||
// pmisc->set_session_low_offer(misc.session_low_offer);
|
||||
}
|
||||
|
||||
// Handles market data queries and responds
|
||||
class QueryServerImpl : public IQueryServer {
|
||||
MDClient &_client;
|
||||
MDConsumer _consumer;
|
||||
int _sub;
|
||||
boost::container::flat_map<UserSymbol, shared_ptr<MDConsumerGroup>> _groups;
|
||||
|
||||
public:
|
||||
QueryServerImpl(MDClient &client, metrics::MetricsContainer &stats)
|
||||
: _client(client), _consumer(client.current_consumer()), _sub(0),
|
||||
_groups() {
|
||||
for (const auto &sym : client.symbols()) {
|
||||
_groups.emplace(std::make_pair(
|
||||
sym, std::make_unique<MDConsumerGroup>(sym, _consumer, stats)));
|
||||
}
|
||||
_groups.shrink_to_fit();
|
||||
|
||||
_sub = client.on_consumer().subscribe(
|
||||
[=, this](const MDConsumer &c) { handle_recovery(c); });
|
||||
subscribe_securities();
|
||||
}
|
||||
QueryServerImpl(const QueryServerImpl &) = delete;
|
||||
~QueryServerImpl() { _client.on_consumer().unsubscribe(_sub); }
|
||||
|
||||
void subscribe_securities() {
|
||||
LOG(md, Info, "QueryServer::subscribe_securities");
|
||||
auto id = _consumer.id();
|
||||
_consumer.on_security(
|
||||
[this, id](const SecurityPtr &sec, MDConsumerHandle handle) {
|
||||
if (_consumer.id() == id) {
|
||||
auto iter = _groups.find(handle.get_usersymbol());
|
||||
bts_assert(iter != _groups.end());
|
||||
const auto &pgroup = iter->second;
|
||||
bts_assert(pgroup);
|
||||
LOG(md, Info,
|
||||
"QueryServer::_consumer.on_security symbol:{} "
|
||||
"sec:{}",
|
||||
handle.get_usersymbol().symbol(),
|
||||
bts::securities::descriptor(*sec));
|
||||
pgroup->add_security(handle);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// not-reentrant
|
||||
void handle_recovery(const MDConsumer &consumer) {
|
||||
MDConsumer temp(consumer);
|
||||
if (temp.all_heartbeats_ok()) {
|
||||
_consumer.swap(temp);
|
||||
LOG(md, Info,
|
||||
"Recovering from bad MD. Old consumer {} new consumer "
|
||||
"{}",
|
||||
_consumer.id(), temp.id());
|
||||
for (auto &pair : _groups) {
|
||||
pair.second->set_consumer(_consumer);
|
||||
}
|
||||
// _handle_index.clear();
|
||||
subscribe_securities();
|
||||
utils::Dispatcher::background().post(
|
||||
[temp]() {
|
||||
LOG(md, Info, "QServer destroying consumer {}", temp.id());
|
||||
},
|
||||
60000);
|
||||
}
|
||||
}
|
||||
|
||||
// the hot part
|
||||
void fill(
|
||||
const SecurityChannel &chan,
|
||||
MDConsumerGroup &group,
|
||||
ChannelReply *reply,
|
||||
const Entitlements *entitlements,
|
||||
bool bypass_entitlement) const {
|
||||
bts_assert(reply);
|
||||
|
||||
reply->mutable_channel()->CopyFrom(chan);
|
||||
reply->mutable_channel()->set_time(TimeStamp::now().to_uint64());
|
||||
|
||||
bool entitled = bypass_entitlement || chan.exchange() == api::VENUE_BTS;
|
||||
if (entitlements && !entitled)
|
||||
entitled = entitlements->count(group.mic);
|
||||
reply->mutable_channel()->set_entitled(entitled);
|
||||
auto flags = (MDRequest::Flags)chan.flags();
|
||||
|
||||
// Get a copy of handles. If recovery happens during this call
|
||||
// we can safely continue using this pointer for 60 seconds
|
||||
auto phandles = group.handles;
|
||||
if (!phandles)
|
||||
return;
|
||||
|
||||
ChannelUpdate temp_update;
|
||||
for (const auto &instrument : *phandles) {
|
||||
bool snapshot = false;
|
||||
MDConsumerHandle handle = instrument.handle;
|
||||
temp_update.Clear();
|
||||
bts_assert_d(handle.is_initialized());
|
||||
if (!handle.is_initialized())
|
||||
continue;
|
||||
ChannelUpdate *pupd = &temp_update;
|
||||
pupd->set_key(instrument.handle.key());
|
||||
// If the security was just discovered try to ensure all
|
||||
// data is sent
|
||||
if (instrument.discovered > TimeStamp(chan.time())) {
|
||||
auto security = handle.get_normalized_security();
|
||||
pupd->mutable_security()->CopyFrom(security);
|
||||
snapshot = true;
|
||||
}
|
||||
// if we are not entitled AND this is the first request
|
||||
// since we lost our entitlement, zero out books
|
||||
if (!entitled && chan.entitled()) {
|
||||
if (HASFLAG(flags, MDRequest::BBO) ||
|
||||
HASFLAG(flags, MDRequest::DIRECT)) {
|
||||
auto *p = pupd->mutable_direct();
|
||||
for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) {
|
||||
convert(NO_MARKET, p->add_bids());
|
||||
convert(NO_MARKET, p->add_asks());
|
||||
}
|
||||
}
|
||||
if ((HASFLAG(flags, MDRequest::IMPLIED) ||
|
||||
HASFLAG(flags, MDRequest::BBO)) &&
|
||||
!handle.is_synthetic()) {
|
||||
auto *p = pupd->mutable_implied();
|
||||
for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) {
|
||||
convert(NO_MARKET, p->add_bids());
|
||||
convert(NO_MARKET, p->add_asks());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (entitled) {
|
||||
// If we are entitled now but this is the first request
|
||||
// where we are ensure we send all values
|
||||
if (!chan.entitled())
|
||||
snapshot = true;
|
||||
|
||||
if (HASFLAG(flags, MDRequest::BBO) ||
|
||||
HASFLAG(flags, MDRequest::DIRECT)) {
|
||||
std::array<PriceSize, COCONUT_MAX_DEPTH> bids, asks;
|
||||
TimeStamp time(snapshot ? 0 : chan.time());
|
||||
size_t copy_depth =
|
||||
HASFLAG(flags, MDRequest::DIRECT) ? COCONUT_MAX_DEPTH : 1;
|
||||
auto result = handle.get_direct_book(
|
||||
time, copy_depth, bids.data(), asks.data());
|
||||
if (result == HandleReadResult::SUCCESS) {
|
||||
auto *p = pupd->mutable_direct();
|
||||
for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) {
|
||||
PriceSize b = bids[x];
|
||||
if (b != NO_MARKET) {
|
||||
convert(b, p->add_bids());
|
||||
}
|
||||
PriceSize a = asks[x];
|
||||
if (a != NO_MARKET) {
|
||||
convert(a, p->add_asks());
|
||||
}
|
||||
}
|
||||
} else if (result == HandleReadResult::BUSY) {
|
||||
LOG(md, Info,
|
||||
"Busy read on handle {}. Retry on next "
|
||||
"request",
|
||||
instrument.handle.key());
|
||||
// If we get a busy reading BBO (which happens
|
||||
// 1-2x per day for a single relay client) We move
|
||||
// the timestamp back so the next request will try
|
||||
// to read this value again. ALTERNATIVE: We could
|
||||
// spin until the value was not busy but it could
|
||||
// be that the writer was context switched out and
|
||||
// we will have to wait multiple ms to unblock the
|
||||
// request. This fix assumes it is better to send
|
||||
// the update now with 1 stale value than to block
|
||||
// the call and allow all the values to be stale
|
||||
// until this one can be read.
|
||||
reply->mutable_channel()->set_time(chan.time());
|
||||
}
|
||||
}
|
||||
if ((HASFLAG(flags, MDRequest::IMPLIED) ||
|
||||
HASFLAG(flags, MDRequest::BBO)) &&
|
||||
!handle.is_synthetic()) {
|
||||
std::array<PriceSize, COCONUT_MAX_IMPLIED_DEPTH> bids, asks;
|
||||
TimeStamp time(snapshot ? 0 : chan.time());
|
||||
size_t copy_depth = HASFLAG(flags, MDRequest::IMPLIED)
|
||||
? COCONUT_MAX_IMPLIED_DEPTH
|
||||
: 1;
|
||||
auto result = handle.get_implied_book(
|
||||
time, copy_depth, bids.data(), asks.data());
|
||||
if (result == HandleReadResult::SUCCESS) {
|
||||
auto *p = pupd->mutable_implied();
|
||||
for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) {
|
||||
PriceSize b = bids[x];
|
||||
if (b != NO_MARKET) {
|
||||
convert(b, p->add_bids());
|
||||
}
|
||||
PriceSize a = asks[x];
|
||||
if (a != NO_MARKET) {
|
||||
convert(a, p->add_asks());
|
||||
}
|
||||
}
|
||||
} else if (result == HandleReadResult::BUSY) {
|
||||
LOG(md, Info,
|
||||
"Busy read on handle (implied) {}. Retry on "
|
||||
"next request",
|
||||
instrument.handle.key());
|
||||
reply->mutable_channel()->set_time(chan.time());
|
||||
}
|
||||
}
|
||||
if (HASFLAG(flags, MDRequest::LAST_TRADE)) {
|
||||
TimeStamp time(snapshot ? 0 : chan.time());
|
||||
PriceSize price;
|
||||
datetime::LLDateTime trade_time;
|
||||
size_t volume;
|
||||
int from_spread;
|
||||
if (handle.get_last_trade(
|
||||
time, price, trade_time, &volume, &from_spread)) {
|
||||
auto *trade = pupd->mutable_last_trade();
|
||||
trade->set_key(handle.key());
|
||||
trade->set_time(trade_time);
|
||||
trade->set_total_volume(volume);
|
||||
trade->set_from_spread(from_spread);
|
||||
set_price(price, trade);
|
||||
}
|
||||
}
|
||||
if (HASFLAG(flags, MDRequest::MISC)) {
|
||||
TimeStamp time(snapshot ? 0 : chan.time());
|
||||
if (handle.misc_timestamp() > time) {
|
||||
copy_misc(
|
||||
handle.settlement_price(), handle.misc(),
|
||||
pupd->mutable_misc());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (pupd->has_direct() || pupd->has_implied() ||
|
||||
pupd->has_last_trade() || pupd->has_misc() ||
|
||||
pupd->has_security()) {
|
||||
reply->add_updates()->CopyFrom(*pupd);
|
||||
}
|
||||
}
|
||||
|
||||
reply->mutable_channel()->set_segment_id(
|
||||
group.channel.create_time().to_uint64());
|
||||
bool is_same_segment =
|
||||
(chan.segment_id() == group.channel.create_time().to_uint64());
|
||||
|
||||
if (HASFLAG(flags, MDRequest::TRADE_LOG)) {
|
||||
int64_t request_cursor = is_same_segment ? chan.trade_cursor() : 0;
|
||||
auto cursor = group.channel.read_trades(
|
||||
*reply->mutable_trade_log(), request_cursor);
|
||||
if (cursor != chan.trade_cursor()) {
|
||||
reply->mutable_channel()->set_trade_cursor(cursor);
|
||||
}
|
||||
}
|
||||
if (HASFLAG(flags, MDRequest::RFQ)) {
|
||||
int64_t request_cursor = is_same_segment ? chan.rfq_cursor() : 0;
|
||||
auto cursor =
|
||||
group.channel.read_rfqs(*reply->mutable_rfq_log(), request_cursor);
|
||||
if (cursor != chan.rfq_cursor()) {
|
||||
reply->mutable_channel()->set_rfq_cursor(cursor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void query(
|
||||
const MDRequest &request,
|
||||
MDReply *reply,
|
||||
const Entitlements *entitlements,
|
||||
bool bypass_entitlement) const {
|
||||
bts_assert(reply != nullptr);
|
||||
for (auto &ch : request.channels()) {
|
||||
auto groupiter =
|
||||
_groups.find(UserSymbol(ch.symbol(), ch.exchange(), ch.type()));
|
||||
if (groupiter != _groups.end()) {
|
||||
auto &group = groupiter->second;
|
||||
fill(
|
||||
ch, *group, reply->add_channels(), entitlements,
|
||||
bypass_entitlement);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void get_names(ExpirationNameListReply *reply) const {
|
||||
bts_assert(reply != nullptr);
|
||||
Security security;
|
||||
for (auto &groupiter : _groups) {
|
||||
const auto &group = groupiter.second;
|
||||
ExpirationNameList *psec = reply->add_securities();
|
||||
psec->mutable_product()->CopyFrom(group->symbol.to_product());
|
||||
for (const auto &h : *(group->handles)) {
|
||||
security.CopyFrom(h.handle.get_normalized_security());
|
||||
std::string desc;
|
||||
if (!DomainSecurityUtils::get_security_desc(security, desc)) {
|
||||
LOG(md, Error,
|
||||
"QueryServerImpl::get_names exchange: {} not handled",
|
||||
Venue_Name(security.exchange()));
|
||||
continue;
|
||||
}
|
||||
psec->mutable_names()->insert({h.handle.key(), desc});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void get_securities(
|
||||
google::protobuf::RepeatedField<security_key> keys,
|
||||
api::SecurityList *result) const {
|
||||
bts_assert(result);
|
||||
// Ensure keys are unique
|
||||
std::set<security_key> unique_keys(keys.begin(), keys.end());
|
||||
for (auto k : unique_keys) {
|
||||
auto ptr = _consumer.get_security(SecurityKey(k));
|
||||
if (ptr) {
|
||||
result->add_security()->CopyFrom(*ptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase {
|
||||
const QueryServerImpl &_db;
|
||||
utils::Dispatcher &_threadpool;
|
||||
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>> _entitlements;
|
||||
#ifndef _WIN32
|
||||
std::shared_ptr<metrics::HistogramMetric<5>> _time, _size;
|
||||
#endif
|
||||
|
||||
MDRelayServer(
|
||||
const QueryServerImpl &db,
|
||||
utils::Dispatcher &threadpool,
|
||||
bts::api::ServiceContext ctx,
|
||||
metrics::MetricsContainer &stats)
|
||||
: bts::co2::AuthenticatedMarketDataRelayBase(ctx), _db(db),
|
||||
_threadpool(threadpool), _entitlements()
|
||||
#ifndef _WIN32
|
||||
,
|
||||
_time(std::make_shared<metrics::HistogramMetric<5>>(
|
||||
"spark_qserver_response_time_ms",
|
||||
std::array<double, 5>({5.0, 10.0, 20.0, 50.0, 100.0}))),
|
||||
_size(std::make_shared<metrics::HistogramMetric<5>>(
|
||||
"spark_qserver_size_bytes",
|
||||
std::array<double, 5>(
|
||||
{10240 /*10k*/, 102400 /*100k*/, 1024 * 1024 /*1mb*/,
|
||||
1024 * 1024 * 10 /*10mb*/, 1024 * 1024 * 100 /*100mb*/}))) {
|
||||
stats.publish(_time);
|
||||
stats.publish(_size);
|
||||
}
|
||||
#else
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
void update_entitlements(
|
||||
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>>
|
||||
&entitlements) {
|
||||
_entitlements = entitlements;
|
||||
}
|
||||
|
||||
static const Entitlements *get_entitlements(
|
||||
uint32_t user_id,
|
||||
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>>
|
||||
&entitlements) {
|
||||
if (entitlements) {
|
||||
auto iter = entitlements->find(user_id);
|
||||
if (iter != entitlements->end()) {
|
||||
return &iter->second;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
virtual void requestmd(
|
||||
const api::AuthToken &token,
|
||||
const MDRequest &request,
|
||||
const std::function<void(const MDReply &)> &reply) override {
|
||||
TimeStamp start = TimeStamp::now();
|
||||
_threadpool.post([=, this]() {
|
||||
google::protobuf::Arena arena;
|
||||
MDReply *preply =
|
||||
google::protobuf::Arena::CreateMessage<MDReply>(&arena);
|
||||
// addref to keep shared ptr alive
|
||||
api::user::TokenPayload payload;
|
||||
auto ents = _entitlements;
|
||||
const Entitlements *user_entitlement = nullptr;
|
||||
bool bypass_entitlement = false;
|
||||
if (api::auth::decode_token(token, &payload)) {
|
||||
bypass_entitlement = payload.wheel();
|
||||
if (!bypass_entitlement) {
|
||||
user_entitlement = get_entitlements(payload.id(), ents);
|
||||
}
|
||||
}
|
||||
// disabling md entitlements
|
||||
_db.query(
|
||||
request, preply, user_entitlement, true || bypass_entitlement);
|
||||
reply(*preply);
|
||||
#ifndef _WIN32
|
||||
auto elapsed = (TimeStamp::now() - start);
|
||||
_time->observe(elapsed.milliseconds());
|
||||
_size->observe(preply->ByteSizeLong());
|
||||
#endif
|
||||
});
|
||||
}
|
||||
|
||||
virtual void request_products(
|
||||
const api::AuthToken &,
|
||||
const api::VoidMessage &,
|
||||
const std::function<void(const ProductListReply &)> &reply) override {
|
||||
ProductListReply list;
|
||||
const auto syms = SymbolLoader::load();
|
||||
for (const auto &i : syms) {
|
||||
auto prod = list.add_products();
|
||||
prod->set_exchange(i.exchange());
|
||||
prod->set_type(i.type());
|
||||
prod->set_symbol(i.symbol());
|
||||
}
|
||||
reply(list);
|
||||
}
|
||||
|
||||
virtual void request_names(
|
||||
const api::AuthToken &,
|
||||
const api::VoidMessage &,
|
||||
const std::function<void(const ExpirationNameListReply &)> &reply)
|
||||
override {
|
||||
_threadpool.post([=, this]() {
|
||||
google::protobuf::Arena arena;
|
||||
ExpirationNameListReply *preply =
|
||||
google::protobuf::Arena::CreateMessage<ExpirationNameListReply>(
|
||||
&arena);
|
||||
_db.get_names(preply);
|
||||
reply(*preply);
|
||||
});
|
||||
}
|
||||
|
||||
virtual void get_securities(
|
||||
const api::AuthToken &,
|
||||
const SecurityKeyList &keys,
|
||||
const std::function<void(const api::SecurityList &)> &reply) override {
|
||||
_threadpool.post([=, this]() {
|
||||
google::protobuf::Arena arena;
|
||||
api::SecurityList *preply =
|
||||
google::protobuf::Arena::CreateMessage<api::SecurityList>(&arena);
|
||||
_db.get_securities(keys.keys(), preply);
|
||||
reply(*preply);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
struct QueryServer {
|
||||
QueryServer(
|
||||
utils::Dispatcher &threadpool,
|
||||
const bts::api::ServiceContext &ctx,
|
||||
MDClient &client,
|
||||
metrics::MetricsContainer &stats)
|
||||
: db(client, stats), relay(db, threadpool, ctx, stats),
|
||||
admin_client(ctx.locate<api::admin::AdminService>()) {}
|
||||
QueryServerImpl db;
|
||||
MDRelayServer relay;
|
||||
api::admin::AdminService admin_client;
|
||||
rxcpp::composite_subscription entitlement_sub;
|
||||
};
|
||||
|
||||
QueryServerPtr run_query_server(
|
||||
const bts::api::ServiceContext &ctx,
|
||||
MDClient &client,
|
||||
metrics::MetricsContainer &stats) {
|
||||
auto &threadpool = utils::Dispatcher::get_named("MDQuery");
|
||||
|
||||
QueryServerPtr container =
|
||||
std::make_shared<QueryServer>(threadpool, ctx, client, stats);
|
||||
|
||||
// Subscribe to entitlement data
|
||||
container->entitlement_sub =
|
||||
container->admin_client.subscribe_md_entitlement_data().subscribe(
|
||||
[=](const api::admin::Md_entitlement_dataUpdate &update) {
|
||||
const auto &all_updates = update.all();
|
||||
utils::Dispatcher::background().post([all_updates, container]() {
|
||||
auto entitlements = std::make_shared<
|
||||
std::unordered_map<uint32_t, Entitlements>>();
|
||||
for (const auto &row : all_updates) {
|
||||
if (row && row->enabled()) {
|
||||
auto iter = entitlements->find(row->user_id());
|
||||
if (iter == entitlements->end()) {
|
||||
Entitlements ents;
|
||||
ents.insert(row->mic());
|
||||
entitlements->insert(
|
||||
iter,
|
||||
std::make_pair(row->user_id(), std::move(ents)));
|
||||
} else {
|
||||
iter->second.insert(row->mic());
|
||||
}
|
||||
}
|
||||
}
|
||||
container->relay.update_entitlements(entitlements);
|
||||
});
|
||||
});
|
||||
|
||||
// Add a few threads to keep md latency low
|
||||
for (int x = 1; x < std::min(4, os::num_cores()); ++x) {
|
||||
threadpool.add_thread();
|
||||
}
|
||||
|
||||
return container;
|
||||
}
|
||||
|
||||
std::unique_ptr<IQueryServer> create_query_server(
|
||||
MDClient &mdc, metrics::MetricsContainer &mets) {
|
||||
return std::make_unique<QueryServerImpl>(mdc, mets);
|
||||
}
|
||||
|
||||
} // namespace md
|
||||
} // namespace bts
|
||||
#endif
|
||||
@@ -1,43 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/md/ISyntheticSecurityService.h"
|
||||
#include "spark/md/UserSymbol.h"
|
||||
#include "spark/utils/bts_export.h"
|
||||
#include "spark/utils/bts_no_copy.h"
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
|
||||
#include "spark/md/protos/co2.pb.h"
|
||||
|
||||
namespace bts {
|
||||
namespace api {
|
||||
class ServiceContext;
|
||||
}
|
||||
namespace metrics {
|
||||
class MetricsContainer;
|
||||
}
|
||||
namespace md {
|
||||
class MDClient;
|
||||
typedef boost::container::flat_set<std::string> Entitlements;
|
||||
|
||||
// provides a polling interface on top of market data
|
||||
|
||||
struct IQueryServer {
|
||||
virtual ~IQueryServer() {}
|
||||
virtual void query(
|
||||
const co2::MDRequest &request,
|
||||
co2::MDReply *reply,
|
||||
const Entitlements *,
|
||||
bool) const = 0;
|
||||
};
|
||||
std::unique_ptr<IQueryServer> create_query_server(
|
||||
MDClient &, metrics::MetricsContainer &);
|
||||
|
||||
struct QueryServer;
|
||||
typedef std::shared_ptr<QueryServer> QueryServerPtr;
|
||||
QueryServerPtr run_query_server(
|
||||
const bts::api::ServiceContext &, MDClient &, metrics::MetricsContainer &);
|
||||
|
||||
} // namespace md
|
||||
} // namespace bts
|
||||
@@ -0,0 +1,10 @@
|
||||
#pragma once
|
||||
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <string>
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
typedef boost::container::flat_set<std::string> Entitlements;
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -0,0 +1,7 @@
|
||||
#include "i_query_server.h"
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
IQueryServer::~IQueryServer() = default;
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/md/protos/co2.pb.h"
|
||||
#include "spark/md/relay/entitlements.h"
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
struct IQueryServer {
|
||||
virtual ~IQueryServer();
|
||||
|
||||
virtual void query(
|
||||
const co2::MDRequest &request,
|
||||
co2::MDReply *reply,
|
||||
const Entitlements *entitlements,
|
||||
bool bypass_entitlement) const = 0;
|
||||
};
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -0,0 +1,13 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/md/MDConsumerHandle.h"
|
||||
#include "spark/utils/TimeStamp.h"
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
struct DatedHandle {
|
||||
utils::TimeStamp discovered;
|
||||
MDConsumerHandle handle;
|
||||
};
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -0,0 +1,141 @@
|
||||
#include "md_consumer_group.h"
|
||||
#include "spark/config/protos/cme_reference_data.pb.h"
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
// probably just have to send the consumer in from outside
|
||||
MDConsumerGroup::MDConsumerGroup(
|
||||
const UserSymbol &symbol,
|
||||
const MDConsumer &consumer,
|
||||
metrics::MetricsContainer &stats)
|
||||
: symbol(symbol), channel(consumer.get_channel(symbol)), handles(), mic() {
|
||||
set_consumer(consumer);
|
||||
|
||||
// create and publish a stat for this group
|
||||
#ifndef _WIN32
|
||||
if (symbol.type() != SecurityType::SECURITYTYPE_SYNTHETIC) {
|
||||
std::vector<metrics::Label> labels;
|
||||
labels.push_back({"symbol", symbol.symbol()});
|
||||
labels.push_back(
|
||||
{"type",
|
||||
SecurityType_Name(symbol.type()).substr(13) /*SECURITYTYPE_*/});
|
||||
labels.push_back(
|
||||
{"exch", Venue_Name(symbol.exchange()).substr(6) /*VENUE_*/});
|
||||
stats.publish(std::make_shared<metrics::LazyCounterMetricL>(
|
||||
"spark_md_book_updates",
|
||||
[=, this]() {
|
||||
return channel.direct_count() + channel.implied_count();
|
||||
},
|
||||
labels));
|
||||
stats.publish(std::make_shared<metrics::LazyCounterMetricL>(
|
||||
"spark_md_security_count",
|
||||
[=, this]() { return channel.security_count(); }, labels));
|
||||
stats.publish(std::make_shared<metrics::LazyCounterMetricL>(
|
||||
"spark_md_trade_count", [=, this]() { return channel.trade_count(); },
|
||||
labels));
|
||||
stats.publish(std::make_shared<metrics::LazyCounterMetricD>(
|
||||
"spark_md_gap_count",
|
||||
[=, this]() {
|
||||
auto g = channel.gap_count();
|
||||
return g == 0 ? os::nan() : g;
|
||||
},
|
||||
labels));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// recovery has occured and we need to clear our our collection of handles
|
||||
// and wait for new ones
|
||||
void MDConsumerGroup::set_consumer(const MDConsumer &consumer) {
|
||||
// create a new set of handles and swap with the old.
|
||||
// If the new consumer is bad we still have to swap our handles because
|
||||
// the consumer that owns our handles will be scheduled for deletion
|
||||
auto temphandles =
|
||||
std::make_shared<ConcurrentAppendCollection<DatedHandle>>();
|
||||
handles.swap(temphandles);
|
||||
channel = consumer.get_channel(symbol);
|
||||
}
|
||||
|
||||
void MDConsumerGroup::add_security(MDConsumerHandle &handle) {
|
||||
bts_assert(handles);
|
||||
bts_assert_d(handle.is_initialized());
|
||||
bts_assert(handle.get_usersymbol() == symbol);
|
||||
DatedHandle h = {TimeStamp::now(), handle};
|
||||
handles->push_back(h);
|
||||
if (this->mic == "")
|
||||
set_mic(h);
|
||||
}
|
||||
|
||||
void MDConsumerGroup::set_mic(const DatedHandle &h) {
|
||||
switch (channel.exchange()) {
|
||||
case api::VENUE_ICE:
|
||||
case api::VENUE_SIMICE:
|
||||
this->mic = h.handle.get_normalized_security().ice().mic_code();
|
||||
break;
|
||||
case api::VENUE_GLBX:
|
||||
case api::VENUE_SIM: {
|
||||
const auto &cme_symb_mics =
|
||||
config::get<config::CmeSymbolMics>().cme_mic();
|
||||
for (auto mic = cme_symb_mics.begin(); mic != cme_symb_mics.end();
|
||||
mic++) {
|
||||
const auto &micsym = mic->first;
|
||||
if (micsym == symbol.symbol()) {
|
||||
this->mic = api::CMEMics_Name(mic->second);
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case api::VENUE_BTS:
|
||||
case api::VENUE_UNSET:
|
||||
this->mic = "BTS";
|
||||
break;
|
||||
case api::VENUE_DERIBIT:
|
||||
this->mic = "DERIBIT";
|
||||
break;
|
||||
case api::VENUE_FTX:
|
||||
this->mic = "FTX";
|
||||
break;
|
||||
case api::VENUE_BINANCE:
|
||||
this->mic = "BINANCE";
|
||||
break;
|
||||
case api::VENUE_BINANCE_COIN_FUTURES:
|
||||
this->mic = "BINANCE_COIN_FUTURES";
|
||||
break;
|
||||
case api::VENUE_BINANCE_US:
|
||||
this->mic = "BINANCE_US";
|
||||
break;
|
||||
case api::VENUE_COINBASE:
|
||||
this->mic = "COINBASE";
|
||||
break;
|
||||
case api::VENUE_BINANCE_USDS_FUTURES:
|
||||
this->mic = "BINANCE_USDS_FUTURES";
|
||||
break;
|
||||
case api::VENUE_OKX:
|
||||
this->mic = "OKX";
|
||||
break;
|
||||
case api::VENUE_CBOE:
|
||||
this->mic = "CBOE";
|
||||
break;
|
||||
case api::VENUE_TWSE:
|
||||
this->mic = "TWSE";
|
||||
break;
|
||||
case api::VENUE_TAIFEX:
|
||||
this->mic = "TAIFEX";
|
||||
break;
|
||||
case api::VENUE_BYBIT:
|
||||
this->mic = "BYBIT";
|
||||
break;
|
||||
case api::VENUE_GATEIO:
|
||||
this->mic = "GATEIO";
|
||||
break;
|
||||
case api::VENUE_CBOECFE:
|
||||
case api::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
|
||||
case api::Venue_INT_MAX_SENTINEL_DO_NOT_USE_:
|
||||
bts_assert(false);
|
||||
this->mic = "UNKNOWN";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -0,0 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/md/MDConsumer.h"
|
||||
#include "spark/md/UserSymbol.h"
|
||||
#include "spark/md/relay/internal/dated_handle.h"
|
||||
#include "spark/metrics/metrics.h"
|
||||
#include "spark/utils/ConcurrentAppendCollection.h"
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
struct MDConsumerGroup {
|
||||
// probably just have to send the consumer in from outside
|
||||
MDConsumerGroup(
|
||||
const UserSymbol &symbol,
|
||||
const MDConsumer &consumer,
|
||||
metrics::MetricsContainer &stats);
|
||||
MDConsumerGroup(const MDConsumerGroup &other) = delete;
|
||||
|
||||
// recovery has occured and we need to clear our our collection of handles
|
||||
// and wait for new ones
|
||||
void set_consumer(const MDConsumer &consumer);
|
||||
|
||||
void add_security(MDConsumerHandle &handle);
|
||||
|
||||
void set_mic(const DatedHandle &h);
|
||||
|
||||
UserSymbol symbol;
|
||||
MDConsumerChannel channel;
|
||||
|
||||
// Can add to handles without invalidating readers (no lock needed)
|
||||
// When we recover we swap a new vector into this unique_ptr and
|
||||
// delayed-delete
|
||||
std::shared_ptr<ConcurrentAppendCollection<DatedHandle>> handles;
|
||||
std::string mic;
|
||||
};
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -0,0 +1,130 @@
|
||||
#include "md_relay_server.h"
|
||||
#include "spark/api/public/ServiceContext.h"
|
||||
#include "spark/api/services/AuthService.pb.h"
|
||||
#include "spark/md/SymbolLoader.h"
|
||||
#include "spark/utils/AuthToken.h"
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
using bts::api::auth::decode_token;
|
||||
using bts::co2::ExpirationNameListReply;
|
||||
using bts::co2::MDReply;
|
||||
using bts::co2::MDRequest;
|
||||
using bts::co2::ProductListReply;
|
||||
using bts::co2::SecurityKeyList;
|
||||
|
||||
MDRelayServer::MDRelayServer(
|
||||
const QueryServerImpl &db,
|
||||
utils::Dispatcher &threadpool,
|
||||
bts::api::ServiceContext ctx,
|
||||
metrics::MetricsContainer &stats)
|
||||
: bts::co2::AuthenticatedMarketDataRelayBase(ctx), _db(db),
|
||||
_threadpool(threadpool), _entitlements()
|
||||
#ifndef _WIN32
|
||||
,
|
||||
_time(std::make_shared<metrics::HistogramMetric<5>>(
|
||||
"spark_qserver_response_time_ms",
|
||||
std::array<double, 5>({5.0, 10.0, 20.0, 50.0, 100.0}))),
|
||||
_size(std::make_shared<metrics::HistogramMetric<5>>(
|
||||
"spark_qserver_size_bytes",
|
||||
std::array<double, 5>(
|
||||
{10240 /*10k*/, 102400 /*100k*/, 1024 * 1024 /*1mb*/,
|
||||
1024 * 1024 * 10 /*10mb*/, 1024 * 1024 * 100 /*100mb*/}))) {
|
||||
stats.publish(_time);
|
||||
stats.publish(_size);
|
||||
}
|
||||
#else
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
void MDRelayServer::update_entitlements(
|
||||
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>> &entitlements) {
|
||||
_entitlements = entitlements;
|
||||
}
|
||||
|
||||
const Entitlements *MDRelayServer::get_entitlements(
|
||||
uint32_t user_id,
|
||||
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>> &entitlements) {
|
||||
if (entitlements) {
|
||||
auto iter = entitlements->find(user_id);
|
||||
if (iter != entitlements->end()) {
|
||||
return &iter->second;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void MDRelayServer::requestmd(
|
||||
const api::AuthToken &token,
|
||||
const MDRequest &request,
|
||||
const std::function<void(const MDReply &)> &reply) {
|
||||
TimeStamp start = TimeStamp::now();
|
||||
_threadpool.post([=, this]() {
|
||||
google::protobuf::Arena arena;
|
||||
MDReply *preply = google::protobuf::Arena::CreateMessage<MDReply>(&arena);
|
||||
// addref to keep shared ptr alive
|
||||
api::user::TokenPayload payload;
|
||||
auto ents = _entitlements;
|
||||
const Entitlements *user_entitlement = nullptr;
|
||||
bool bypass_entitlement = false;
|
||||
if (decode_token(token, &payload)) {
|
||||
bypass_entitlement = payload.wheel();
|
||||
if (!bypass_entitlement) {
|
||||
user_entitlement = get_entitlements(payload.id(), ents);
|
||||
}
|
||||
}
|
||||
// disabling md entitlements
|
||||
_db.query(request, preply, user_entitlement, true || bypass_entitlement);
|
||||
reply(*preply);
|
||||
#ifndef _WIN32
|
||||
auto elapsed = (TimeStamp::now() - start);
|
||||
_time->observe(elapsed.milliseconds());
|
||||
_size->observe(preply->ByteSizeLong());
|
||||
#endif
|
||||
});
|
||||
}
|
||||
|
||||
void MDRelayServer::request_products(
|
||||
const api::AuthToken &,
|
||||
const api::VoidMessage &,
|
||||
const std::function<void(const ProductListReply &)> &reply) {
|
||||
ProductListReply list;
|
||||
const auto syms = SymbolLoader::load();
|
||||
for (const auto &i : syms) {
|
||||
auto prod = list.add_products();
|
||||
prod->set_exchange(i.exchange());
|
||||
prod->set_type(i.type());
|
||||
prod->set_symbol(i.symbol());
|
||||
}
|
||||
reply(list);
|
||||
}
|
||||
|
||||
void MDRelayServer::request_names(
|
||||
const api::AuthToken &,
|
||||
const api::VoidMessage &,
|
||||
const std::function<void(const ExpirationNameListReply &)> &reply) {
|
||||
_threadpool.post([=, this]() {
|
||||
google::protobuf::Arena arena;
|
||||
ExpirationNameListReply *preply =
|
||||
google::protobuf::Arena::CreateMessage<ExpirationNameListReply>(
|
||||
&arena);
|
||||
_db.get_names(preply);
|
||||
reply(*preply);
|
||||
});
|
||||
}
|
||||
|
||||
void MDRelayServer::get_securities(
|
||||
const api::AuthToken &,
|
||||
const SecurityKeyList &keys,
|
||||
const std::function<void(const api::SecurityList &)> &reply) {
|
||||
_threadpool.post([=, this]() {
|
||||
google::protobuf::Arena arena;
|
||||
api::SecurityList *preply =
|
||||
google::protobuf::Arena::CreateMessage<api::SecurityList>(&arena);
|
||||
_db.get_securities(keys.keys(), preply);
|
||||
reply(*preply);
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -0,0 +1,54 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/api/services/MarketDataRelay.server.pb.h"
|
||||
#include "spark/md/relay/internal/query_server_impl.h"
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase {
|
||||
const QueryServerImpl &_db;
|
||||
utils::Dispatcher &_threadpool;
|
||||
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>> _entitlements;
|
||||
#ifndef _WIN32
|
||||
std::shared_ptr<metrics::HistogramMetric<5>> _time, _size;
|
||||
#endif
|
||||
|
||||
MDRelayServer(
|
||||
const QueryServerImpl &db,
|
||||
utils::Dispatcher &threadpool,
|
||||
bts::api::ServiceContext ctx,
|
||||
metrics::MetricsContainer &stats);
|
||||
|
||||
void update_entitlements(
|
||||
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>>
|
||||
&entitlements);
|
||||
|
||||
static const Entitlements *get_entitlements(
|
||||
uint32_t user_id,
|
||||
std::shared_ptr<std::unordered_map<uint32_t, Entitlements>>
|
||||
&entitlements);
|
||||
|
||||
virtual void requestmd(
|
||||
const api::AuthToken &token,
|
||||
const co2::MDRequest &request,
|
||||
const std::function<void(const co2::MDReply &)> &reply) override;
|
||||
|
||||
virtual void request_products(
|
||||
const api::AuthToken &,
|
||||
const api::VoidMessage &,
|
||||
const std::function<void(const co2::ProductListReply &)> &reply)
|
||||
override;
|
||||
|
||||
virtual void request_names(
|
||||
const api::AuthToken &,
|
||||
const api::VoidMessage &,
|
||||
const std::function<void(const co2::ExpirationNameListReply &)> &reply)
|
||||
override;
|
||||
|
||||
virtual void get_securities(
|
||||
const api::AuthToken &,
|
||||
const co2::SecurityKeyList &keys,
|
||||
const std::function<void(const api::SecurityList &)> &reply) override;
|
||||
};
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -0,0 +1,360 @@
|
||||
#include "query_server_impl.h"
|
||||
#include "spark/domain/DomainSecurityUtils.h"
|
||||
#include "spark/domain_model/Security.h"
|
||||
#include "spark/md/relay/internal/md_consumer_group.h"
|
||||
#include "spark/utils/helper_macros.h"
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
using bts::co2::ChannelReply;
|
||||
using bts::co2::ChannelUpdate;
|
||||
using bts::co2::ExpirationNameList;
|
||||
using bts::co2::ExpirationNameListReply;
|
||||
using bts::co2::MDReply;
|
||||
using bts::co2::MDRequest;
|
||||
using bts::co2::PriceSizeMessage;
|
||||
using bts::co2::SecurityChannel;
|
||||
using bts::domain::DomainSecurityUtils;
|
||||
|
||||
static void convert(PriceSize ps, PriceSizeMessage *pps) {
|
||||
pps->set_mantissa(ps.mantissa());
|
||||
pps->set_exponent(ps.exponent());
|
||||
pps->set_size(ps.size());
|
||||
}
|
||||
|
||||
static void set_price(PriceSize ps, co2::TradeMessage *ptrade) {
|
||||
ptrade->set_mantissa(ps.mantissa());
|
||||
ptrade->set_exponent(ps.exponent());
|
||||
ptrade->set_size(ps.size());
|
||||
}
|
||||
|
||||
static void set_price(const IntPrice &price, co2::PriceMessage *dest) {
|
||||
dest->set_mantissa(price.mantissa());
|
||||
dest->set_exponent(price.exponent());
|
||||
}
|
||||
|
||||
static void copy_misc(
|
||||
IntPrice settlement, MiscData misc, co2::MiscMessage *pmisc) {
|
||||
set_price(settlement, pmisc->mutable_settlement());
|
||||
pmisc->set_settlement_date(misc.settlement_date.to_short());
|
||||
set_price(
|
||||
misc.previous_settlement_price,
|
||||
pmisc->mutable_previous_settlement_price());
|
||||
pmisc->set_previous_settlement_date(
|
||||
misc.previous_settlement_date.to_short());
|
||||
pmisc->set_open_price(misc.open_price);
|
||||
pmisc->set_open_date(misc.open_date.to_short());
|
||||
pmisc->set_close_price(misc.close_price);
|
||||
pmisc->set_close_date(misc.close_date.to_short());
|
||||
pmisc->set_fixing_price(misc.fixing_price);
|
||||
pmisc->set_session_high(misc.session_high);
|
||||
pmisc->set_session_low(misc.session_low);
|
||||
// pmisc->set_session_high_bid(misc.session_high_bid);
|
||||
// pmisc->set_session_low_offer(misc.session_low_offer);
|
||||
}
|
||||
|
||||
// Handles market data queries and responds
|
||||
QueryServerImpl::QueryServerImpl(
|
||||
MDClient &client, metrics::MetricsContainer &stats)
|
||||
: _client(client), _consumer(client.current_consumer()), _sub(0),
|
||||
_groups() {
|
||||
for (const auto &sym : client.symbols()) {
|
||||
_groups.emplace(std::make_pair(
|
||||
sym, std::make_unique<MDConsumerGroup>(sym, _consumer, stats)));
|
||||
}
|
||||
_groups.shrink_to_fit();
|
||||
|
||||
_sub = client.on_consumer().subscribe(
|
||||
[=, this](const MDConsumer &c) { handle_recovery(c); });
|
||||
subscribe_securities();
|
||||
}
|
||||
|
||||
QueryServerImpl::~QueryServerImpl() { _client.on_consumer().unsubscribe(_sub); }
|
||||
|
||||
void QueryServerImpl::subscribe_securities() {
|
||||
LOG(md, Info, "QueryServer::subscribe_securities");
|
||||
auto id = _consumer.id();
|
||||
_consumer.on_security(
|
||||
[this, id](const SecurityPtr &sec, MDConsumerHandle handle) {
|
||||
if (_consumer.id() == id) {
|
||||
auto iter = _groups.find(handle.get_usersymbol());
|
||||
bts_assert(iter != _groups.end());
|
||||
const auto &pgroup = iter->second;
|
||||
bts_assert(pgroup);
|
||||
LOG(md, Info,
|
||||
"QueryServer::_consumer.on_security symbol:{} "
|
||||
"sec:{}",
|
||||
handle.get_usersymbol().symbol(),
|
||||
bts::securities::descriptor(*sec));
|
||||
pgroup->add_security(handle);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// not-reentrant
|
||||
void QueryServerImpl::handle_recovery(const MDConsumer &consumer) {
|
||||
MDConsumer temp(consumer);
|
||||
if (temp.all_heartbeats_ok()) {
|
||||
_consumer.swap(temp);
|
||||
LOG(md, Info,
|
||||
"Recovering from bad MD. Old consumer {} new consumer "
|
||||
"{}",
|
||||
_consumer.id(), temp.id());
|
||||
for (auto &pair : _groups) {
|
||||
pair.second->set_consumer(_consumer);
|
||||
}
|
||||
// _handle_index.clear();
|
||||
subscribe_securities();
|
||||
utils::Dispatcher::background().post(
|
||||
[temp]() {
|
||||
LOG(md, Info, "QServer destroying consumer {}", temp.id());
|
||||
},
|
||||
60000);
|
||||
}
|
||||
}
|
||||
|
||||
// the hot part
|
||||
void QueryServerImpl::fill(
|
||||
const SecurityChannel &chan,
|
||||
MDConsumerGroup &group,
|
||||
ChannelReply *reply,
|
||||
const Entitlements *entitlements,
|
||||
bool bypass_entitlement) const {
|
||||
bts_assert(reply);
|
||||
|
||||
reply->mutable_channel()->CopyFrom(chan);
|
||||
reply->mutable_channel()->set_time(TimeStamp::now().to_uint64());
|
||||
|
||||
bool entitled = bypass_entitlement || chan.exchange() == api::VENUE_BTS;
|
||||
if (entitlements && !entitled)
|
||||
entitled = entitlements->count(group.mic);
|
||||
reply->mutable_channel()->set_entitled(entitled);
|
||||
auto flags = (MDRequest::Flags)chan.flags();
|
||||
|
||||
// Get a copy of handles. If recovery happens during this call
|
||||
// we can safely continue using this pointer for 60 seconds
|
||||
auto phandles = group.handles;
|
||||
if (!phandles)
|
||||
return;
|
||||
|
||||
ChannelUpdate temp_update;
|
||||
for (const auto &instrument : *phandles) {
|
||||
bool snapshot = false;
|
||||
MDConsumerHandle handle = instrument.handle;
|
||||
temp_update.Clear();
|
||||
bts_assert_d(handle.is_initialized());
|
||||
if (!handle.is_initialized())
|
||||
continue;
|
||||
ChannelUpdate *pupd = &temp_update;
|
||||
pupd->set_key(instrument.handle.key());
|
||||
// If the security was just discovered try to ensure all
|
||||
// data is sent
|
||||
if (instrument.discovered > TimeStamp(chan.time())) {
|
||||
auto security = handle.get_normalized_security();
|
||||
pupd->mutable_security()->CopyFrom(security);
|
||||
snapshot = true;
|
||||
}
|
||||
// if we are not entitled AND this is the first request
|
||||
// since we lost our entitlement, zero out books
|
||||
if (!entitled && chan.entitled()) {
|
||||
if (HASFLAG(flags, MDRequest::BBO) ||
|
||||
HASFLAG(flags, MDRequest::DIRECT)) {
|
||||
auto *p = pupd->mutable_direct();
|
||||
for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) {
|
||||
convert(NO_MARKET, p->add_bids());
|
||||
convert(NO_MARKET, p->add_asks());
|
||||
}
|
||||
}
|
||||
if ((HASFLAG(flags, MDRequest::IMPLIED) ||
|
||||
HASFLAG(flags, MDRequest::BBO)) &&
|
||||
!handle.is_synthetic()) {
|
||||
auto *p = pupd->mutable_implied();
|
||||
for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) {
|
||||
convert(NO_MARKET, p->add_bids());
|
||||
convert(NO_MARKET, p->add_asks());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (entitled) {
|
||||
// If we are entitled now but this is the first request
|
||||
// where we are ensure we send all values
|
||||
if (!chan.entitled())
|
||||
snapshot = true;
|
||||
|
||||
if (HASFLAG(flags, MDRequest::BBO) ||
|
||||
HASFLAG(flags, MDRequest::DIRECT)) {
|
||||
std::array<PriceSize, COCONUT_MAX_DEPTH> bids, asks;
|
||||
TimeStamp time(snapshot ? 0 : chan.time());
|
||||
size_t copy_depth =
|
||||
HASFLAG(flags, MDRequest::DIRECT) ? COCONUT_MAX_DEPTH : 1;
|
||||
auto result = handle.get_direct_book(
|
||||
time, copy_depth, bids.data(), asks.data());
|
||||
if (result == HandleReadResult::SUCCESS) {
|
||||
auto *p = pupd->mutable_direct();
|
||||
for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) {
|
||||
PriceSize b = bids[x];
|
||||
if (b != NO_MARKET) {
|
||||
convert(b, p->add_bids());
|
||||
}
|
||||
PriceSize a = asks[x];
|
||||
if (a != NO_MARKET) {
|
||||
convert(a, p->add_asks());
|
||||
}
|
||||
}
|
||||
} else if (result == HandleReadResult::BUSY) {
|
||||
LOG(md, Info,
|
||||
"Busy read on handle {}. Retry on next "
|
||||
"request",
|
||||
instrument.handle.key());
|
||||
// If we get a busy reading BBO (which happens
|
||||
// 1-2x per day for a single relay client) We move
|
||||
// the timestamp back so the next request will try
|
||||
// to read this value again. ALTERNATIVE: We could
|
||||
// spin until the value was not busy but it could
|
||||
// be that the writer was context switched out and
|
||||
// we will have to wait multiple ms to unblock the
|
||||
// request. This fix assumes it is better to send
|
||||
// the update now with 1 stale value than to block
|
||||
// the call and allow all the values to be stale
|
||||
// until this one can be read.
|
||||
reply->mutable_channel()->set_time(chan.time());
|
||||
}
|
||||
}
|
||||
if ((HASFLAG(flags, MDRequest::IMPLIED) ||
|
||||
HASFLAG(flags, MDRequest::BBO)) &&
|
||||
!handle.is_synthetic()) {
|
||||
std::array<PriceSize, COCONUT_MAX_IMPLIED_DEPTH> bids, asks;
|
||||
TimeStamp time(snapshot ? 0 : chan.time());
|
||||
size_t copy_depth = HASFLAG(flags, MDRequest::IMPLIED)
|
||||
? COCONUT_MAX_IMPLIED_DEPTH
|
||||
: 1;
|
||||
auto result = handle.get_implied_book(
|
||||
time, copy_depth, bids.data(), asks.data());
|
||||
if (result == HandleReadResult::SUCCESS) {
|
||||
auto *p = pupd->mutable_implied();
|
||||
for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) {
|
||||
PriceSize b = bids[x];
|
||||
if (b != NO_MARKET) {
|
||||
convert(b, p->add_bids());
|
||||
}
|
||||
PriceSize a = asks[x];
|
||||
if (a != NO_MARKET) {
|
||||
convert(a, p->add_asks());
|
||||
}
|
||||
}
|
||||
} else if (result == HandleReadResult::BUSY) {
|
||||
LOG(md, Info,
|
||||
"Busy read on handle (implied) {}. Retry on "
|
||||
"next request",
|
||||
instrument.handle.key());
|
||||
reply->mutable_channel()->set_time(chan.time());
|
||||
}
|
||||
}
|
||||
if (HASFLAG(flags, MDRequest::LAST_TRADE)) {
|
||||
TimeStamp time(snapshot ? 0 : chan.time());
|
||||
PriceSize price;
|
||||
datetime::LLDateTime trade_time;
|
||||
size_t volume;
|
||||
int from_spread;
|
||||
if (handle.get_last_trade(
|
||||
time, price, trade_time, &volume, &from_spread)) {
|
||||
auto *trade = pupd->mutable_last_trade();
|
||||
trade->set_key(handle.key());
|
||||
trade->set_time(trade_time);
|
||||
trade->set_total_volume(volume);
|
||||
trade->set_from_spread(from_spread);
|
||||
set_price(price, trade);
|
||||
}
|
||||
}
|
||||
if (HASFLAG(flags, MDRequest::MISC)) {
|
||||
TimeStamp time(snapshot ? 0 : chan.time());
|
||||
if (handle.misc_timestamp() > time) {
|
||||
copy_misc(
|
||||
handle.settlement_price(), handle.misc(),
|
||||
pupd->mutable_misc());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (pupd->has_direct() || pupd->has_implied() || pupd->has_last_trade() ||
|
||||
pupd->has_misc() || pupd->has_security()) {
|
||||
reply->add_updates()->CopyFrom(*pupd);
|
||||
}
|
||||
}
|
||||
|
||||
reply->mutable_channel()->set_segment_id(
|
||||
group.channel.create_time().to_uint64());
|
||||
bool is_same_segment =
|
||||
(chan.segment_id() == group.channel.create_time().to_uint64());
|
||||
|
||||
if (HASFLAG(flags, MDRequest::TRADE_LOG)) {
|
||||
int64_t request_cursor = is_same_segment ? chan.trade_cursor() : 0;
|
||||
auto cursor = group.channel.read_trades(
|
||||
*reply->mutable_trade_log(), request_cursor);
|
||||
if (cursor != chan.trade_cursor()) {
|
||||
reply->mutable_channel()->set_trade_cursor(cursor);
|
||||
}
|
||||
}
|
||||
if (HASFLAG(flags, MDRequest::RFQ)) {
|
||||
int64_t request_cursor = is_same_segment ? chan.rfq_cursor() : 0;
|
||||
auto cursor =
|
||||
group.channel.read_rfqs(*reply->mutable_rfq_log(), request_cursor);
|
||||
if (cursor != chan.rfq_cursor()) {
|
||||
reply->mutable_channel()->set_rfq_cursor(cursor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void QueryServerImpl::query(
|
||||
const MDRequest &request,
|
||||
MDReply *reply,
|
||||
const Entitlements *entitlements,
|
||||
bool bypass_entitlement) const {
|
||||
bts_assert(reply != nullptr);
|
||||
for (auto &ch : request.channels()) {
|
||||
auto groupiter =
|
||||
_groups.find(UserSymbol(ch.symbol(), ch.exchange(), ch.type()));
|
||||
if (groupiter != _groups.end()) {
|
||||
auto &group = groupiter->second;
|
||||
fill(
|
||||
ch, *group, reply->add_channels(), entitlements,
|
||||
bypass_entitlement);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void QueryServerImpl::get_names(ExpirationNameListReply *reply) const {
|
||||
bts_assert(reply != nullptr);
|
||||
Security security;
|
||||
for (auto &groupiter : _groups) {
|
||||
const auto &group = groupiter.second;
|
||||
ExpirationNameList *psec = reply->add_securities();
|
||||
psec->mutable_product()->CopyFrom(group->symbol.to_product());
|
||||
for (const auto &h : *(group->handles)) {
|
||||
security.CopyFrom(h.handle.get_normalized_security());
|
||||
std::string desc;
|
||||
if (!DomainSecurityUtils::get_security_desc(security, desc)) {
|
||||
LOG(md, Error,
|
||||
"QueryServerImpl::get_names exchange: {} not handled",
|
||||
Venue_Name(security.exchange()));
|
||||
continue;
|
||||
}
|
||||
psec->mutable_names()->insert({h.handle.key(), desc});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void QueryServerImpl::get_securities(
|
||||
google::protobuf::RepeatedField<security_key> keys,
|
||||
api::SecurityList *result) const {
|
||||
bts_assert(result);
|
||||
// Ensure keys are unique
|
||||
std::set<security_key> unique_keys(keys.begin(), keys.end());
|
||||
for (auto k : unique_keys) {
|
||||
auto ptr = _consumer.get_security(SecurityKey(k));
|
||||
if (ptr) {
|
||||
result->add_security()->CopyFrom(*ptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -0,0 +1,52 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/api/services/MarketDataRelay.pb.h"
|
||||
#include "spark/md/MDClient.h"
|
||||
#include "spark/md/relay/i_query_server.h"
|
||||
#include "spark/md/relay/internal/md_consumer_group.h"
|
||||
#include "spark/metrics/metrics.h"
|
||||
|
||||
#include <boost/container/flat_map.hpp>
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
class QueryServerImpl : public IQueryServer {
|
||||
MDClient &_client;
|
||||
MDConsumer _consumer;
|
||||
int _sub;
|
||||
boost::container::flat_map<UserSymbol, std::shared_ptr<MDConsumerGroup>>
|
||||
_groups;
|
||||
|
||||
public:
|
||||
QueryServerImpl(MDClient &client, metrics::MetricsContainer &stats);
|
||||
|
||||
QueryServerImpl(const QueryServerImpl &) = delete;
|
||||
~QueryServerImpl();
|
||||
|
||||
void subscribe_securities();
|
||||
|
||||
// not-reentrant
|
||||
void handle_recovery(const MDConsumer &consumer);
|
||||
|
||||
// the hot part
|
||||
void fill(
|
||||
const co2::SecurityChannel &chan,
|
||||
MDConsumerGroup &group,
|
||||
co2::ChannelReply *reply,
|
||||
const Entitlements *entitlements,
|
||||
bool bypass_entitlement) const;
|
||||
|
||||
void query(
|
||||
const co2::MDRequest &request,
|
||||
co2::MDReply *reply,
|
||||
const Entitlements *entitlements,
|
||||
bool bypass_entitlement) const;
|
||||
|
||||
void get_names(co2::ExpirationNameListReply *reply) const;
|
||||
|
||||
void get_securities(
|
||||
google::protobuf::RepeatedField<security_key> keys,
|
||||
api::SecurityList *result) const;
|
||||
};
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -0,0 +1,114 @@
|
||||
#ifndef _WIN32
|
||||
#include "query_server.h"
|
||||
#include "spark/utils/Dispatcher.h"
|
||||
|
||||
#include "../MDClient.h"
|
||||
|
||||
#include "spark/api/public/ServiceContext.h"
|
||||
#include "spark/api/services/AdminService.client.pb.h"
|
||||
#include "spark/api/services/MarketDataRelay.server.pb.h"
|
||||
#include "spark/md/relay/internal/md_relay_server.h"
|
||||
#ifndef _WIN32
|
||||
#include "spark/metrics/metrics.h"
|
||||
#endif
|
||||
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <boost/container/flat_map.hpp>
|
||||
#include <google/protobuf/arena.h>
|
||||
|
||||
using namespace bts::co2;
|
||||
|
||||
using std::string;
|
||||
using std::unique_ptr;
|
||||
|
||||
namespace bts {
|
||||
namespace md {
|
||||
|
||||
/*
|
||||
* NORMAL OPERATION:
|
||||
* Client sends time=0 to server
|
||||
* Server sends complete snapshot back (time=502)
|
||||
* client sends time=502 to server
|
||||
* Server sends any changes that have happened after time=502 and any securities
|
||||
* discovered since time=502
|
||||
*
|
||||
* ERROR RECOVERY:
|
||||
* Client sends time=0 to server
|
||||
* Server sends complete snapshot back (time=502)
|
||||
* Server detects heartbeat failure and recovers. All timestamps > 502
|
||||
* Client sends time=502 to server
|
||||
* Server sends complete snapshot back (time=605)
|
||||
* - Since the server shared memory has been recreated the client will
|
||||
* automatically overwrite all stale data.
|
||||
* - If a security has been deleted the client will leave it frozen (OK)
|
||||
*
|
||||
* We scan for bad heartbeats on a different time
|
||||
* If a bad heartbeat is detected we try to construct a new Consumer
|
||||
* When a good consumer is created we std::move the old consumer and handles to
|
||||
* delayed queue for deletion This way we don't have to lock anything during the
|
||||
* recovery process because all pointers will remain valid for the duration of
|
||||
* existing calls.
|
||||
*
|
||||
*/
|
||||
|
||||
QueryServer::QueryServer(
|
||||
utils::Dispatcher &threadpool,
|
||||
const bts::api::ServiceContext &ctx,
|
||||
MDClient &client,
|
||||
metrics::MetricsContainer &stats)
|
||||
: db(client, stats), relay(db, threadpool, ctx, stats),
|
||||
admin_client(ctx.locate<api::admin::AdminService>()) {}
|
||||
|
||||
QueryServerPtr run_query_server(
|
||||
const bts::api::ServiceContext &ctx,
|
||||
MDClient &client,
|
||||
metrics::MetricsContainer &stats) {
|
||||
auto &threadpool = utils::Dispatcher::get_named("MDQuery");
|
||||
|
||||
QueryServerPtr container =
|
||||
std::make_shared<QueryServer>(threadpool, ctx, client, stats);
|
||||
|
||||
// Subscribe to entitlement data
|
||||
container->entitlement_sub =
|
||||
container->admin_client.subscribe_md_entitlement_data().subscribe(
|
||||
[=](const api::admin::Md_entitlement_dataUpdate &update) {
|
||||
const auto &all_updates = update.all();
|
||||
utils::Dispatcher::background().post([all_updates, container]() {
|
||||
auto entitlements = std::make_shared<
|
||||
std::unordered_map<uint32_t, Entitlements>>();
|
||||
for (const auto &row : all_updates) {
|
||||
if (row && row->enabled()) {
|
||||
auto iter = entitlements->find(row->user_id());
|
||||
if (iter == entitlements->end()) {
|
||||
Entitlements ents;
|
||||
ents.insert(row->mic());
|
||||
entitlements->insert(
|
||||
iter,
|
||||
std::make_pair(row->user_id(), std::move(ents)));
|
||||
} else {
|
||||
iter->second.insert(row->mic());
|
||||
}
|
||||
}
|
||||
}
|
||||
container->relay.update_entitlements(entitlements);
|
||||
});
|
||||
});
|
||||
|
||||
// Add a few threads to keep md latency low
|
||||
for (int x = 1; x < std::min(4, os::num_cores()); ++x) {
|
||||
threadpool.add_thread();
|
||||
}
|
||||
|
||||
return container;
|
||||
}
|
||||
|
||||
std::unique_ptr<IQueryServer> create_query_server(
|
||||
MDClient &mdc, metrics::MetricsContainer &mets) {
|
||||
return std::make_unique<QueryServerImpl>(mdc, mets);
|
||||
}
|
||||
|
||||
} // namespace md
|
||||
} // namespace bts
|
||||
#endif
|
||||
@@ -0,0 +1,32 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/api/services/AdminService.client.pb.h"
|
||||
#include "spark/md/relay/internal/md_relay_server.h"
|
||||
#include "spark/md/relay/internal/query_server_impl.h"
|
||||
#include "spark/metrics/metrics.h"
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
struct QueryServer {
|
||||
QueryServer(
|
||||
utils::Dispatcher &threadpool,
|
||||
const api::ServiceContext &ctx,
|
||||
MDClient &client,
|
||||
metrics::MetricsContainer &stats);
|
||||
|
||||
QueryServerImpl db;
|
||||
MDRelayServer relay;
|
||||
api::admin::AdminService admin_client;
|
||||
rxcpp::composite_subscription entitlement_sub;
|
||||
};
|
||||
typedef std::shared_ptr<QueryServer> QueryServerPtr;
|
||||
|
||||
class MDClient;
|
||||
|
||||
std::unique_ptr<IQueryServer> create_query_server(
|
||||
MDClient &, metrics::MetricsContainer &);
|
||||
|
||||
QueryServerPtr run_query_server(
|
||||
const bts::api::ServiceContext &, MDClient &, metrics::MetricsContainer &);
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -3,9 +3,9 @@
|
||||
#include "../SecurityCache.h"
|
||||
#include "../SettlementRecorder.h"
|
||||
#include "../coconut2.h"
|
||||
#include "../relay/QueryServer.h"
|
||||
#include "Co2TestInstrument.h"
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/md/relay/query_server.h"
|
||||
#include "spark/metrics/metrics.h"
|
||||
#include "spark/os/os.h"
|
||||
#include "spark/persistence/SqliteStore.h"
|
||||
|
||||
Reference in new issue
Block a user