Files
spark/src/lib/md/relay/QueryServer.cpp
T
2019-09-17 16:15:17 -04:00

612 lines
25 KiB
C++

#ifndef _WIN32
#include "QueryServer.h"
#include "lib/config/config.h"
#include "lib/utils/Dispatcher.h"
#include "lib/utils/TimeStamp.h"
#include "lib/utils/helper_macros.h"
#include "lib/utils/AuthToken.h"
#include "../MDConsumer.h"
#include "../MDClient.h"
#include "../PlatformLoader.h"
#include "../MBOProcessor.h"
#include "lib/logging/logging.h"
#include "lib/config/protos/system.pb.h"
#include "lib/md/protos/co2.pb.h"
#include "lib/md/SecurityCache.h"
#include "lib/api/apicore.pb.h"
#include "lib/api/services/MarketDataRelay.server.pb.h"
#include "lib/api/public/ServiceContext.h"
#include "lib/api/services/AdminService.client.pb.h"
#include "lib/api/service_helpers/AdminServiceUtils.h"
#include "lib/config/protos/cme_reference_data.pb.h"
#include "lib/utils/ConcurrentAppendCollection.h"
#ifndef _WIN32
#include "lib/metrics/metrics.h"
#endif
#include <vector>
#include <list>
#include <memory>
#include <set>
#include <unordered_map>
#include <optional>
#include <google/protobuf/arena.h>
#include <boost/container/flat_map.hpp>
using namespace bts::co2;
using std::unique_ptr;
using std::shared_ptr;
namespace bts {
namespace md {
/*
* NORMAL OPERATION:
* Client sends time=0 to server
* Server sends complete snapshot back (time=502)
* client sends time=502 to server
* Server sends any changes that have happened after time=502 and any securities discovered since time=502
*
* ERROR RECOVERY:
* Client sends time=0 to server
* Server sends complete snapshot back (time=502)
* Server detects heartbeat failure and recovers. All timestamps > 502
* Client sends time=502 to server
* Server sends complete snapshot back (time=605)
* - Since the server shared memory has been recreated the client will automatically overwrite all stale data.
* - If a security has been deleted the client will leave it frozen (OK)
*
* We scan for bad heartbeats on a different time
* If a bad heartbeat is detected we try to construct a new Consumer
* When a good consumer is created we std::move the old consumer and handles to delayed queue for deletion
* This way we don't have to lock anything during the recovery process because all pointers will remain valid
* for the duration of existing calls.
*
*/
struct DatedHandle {
TimeStamp discovered;
MDConsumerHandle handle;
};
struct MDConsumerGroup {
// probably just have to send the consumer in from outside
MDConsumerGroup(const UserSymbol& symbol, const MDConsumer& consumer, metrics::MetricsContainer& stats) :
symbol(symbol),
platform(platform_for_exchange(symbol.exchange())),
channel(consumer.get_channel(symbol)),
handles(),
mic()
{
set_consumer(consumer);
// create and publish a stat for this group
#ifndef _WIN32
if (symbol.type() != SecurityType::SECURITYTYPE_SYNTHETIC) {
std::vector<metrics::Label> labels;
labels.push_back({ "symbol", symbol.symbol() });
labels.push_back({ "type", SecurityType_Name(symbol.type()).substr(13) /*SECURITYTYPE_*/ });
labels.push_back({ "exch", Venue_Name(symbol.exchange()).substr(6) /*VENUE_*/ });
stats.publish(
std::make_shared<metrics::LazyCounterMetricL>("spark_md_book_updates", [=]() {
return channel.direct_count() + channel.implied_count();
}, labels));
stats.publish(
std::make_shared<metrics::LazyCounterMetricL>("spark_md_security_count", [=]() { return channel.security_count(); }, labels));
stats.publish(
std::make_shared<metrics::LazyCounterMetricL>("spark_md_trade_count", [=]() { return channel.trade_count(); }, labels));
stats.publish(
std::make_shared<metrics::LazyCounterMetricD>("spark_md_gap_count", [=]() { auto g = channel.gap_count(); return g == 0 ? os::nan() : g; }, labels));
}
#endif
}
MDConsumerGroup(const MDConsumerGroup& other) = delete;
// recovery has occured and we need to clear our our collection of handles and wait for new ones
void set_consumer(const MDConsumer& consumer) {
// create a new set of handles and swap with the old.
// If the new consumer is bad we still have to swap our handles because the consumer that owns our handles will be scheduled for deletion
auto temphandles = std::make_shared<ConcurrentAppendCollection<DatedHandle>>();
handles.swap(temphandles);
channel = consumer.get_channel(symbol);
}
void add_security(MDConsumerHandle& handle) {
bts_assert(handles);
bts_assert_d(handle.is_initialized());
bts_assert(handle.get_usersymbol() == symbol);
DatedHandle h = { TimeStamp::now(), handle };
handles->push_back(h);
if(this->mic == "")
set_mic(h);
}
void set_mic(const DatedHandle& h)
{
if(channel.exchange() == api::VENUE_ICE || channel.exchange() == api::VENUE_SIMICE)
{
this->mic = h.handle.get_normalized_security().ice().mic_code();
}
else if(channel.exchange() == api::VENUE_GLBX || channel.exchange() == api::VENUE_SIM)
{
const auto& cme_symb_mics = config::get<config::CmeSymbolMics>().cme_mic();
for(auto mic=cme_symb_mics.begin();mic!=cme_symb_mics.end();mic++)
{
const auto& micsym = mic->first;
if(micsym == symbol.symbol())
{
this->mic = api::CMEMics_Name(mic->second);
break;
}
}
}
else if(channel.exchange() == api::VENUE_BTS || channel.exchange() == api::VENUE_UNSET)
{
this->mic = "BTS";
}
else
bts_assert_msg(false, "unknown venue in QueryServer");
}
UserSymbol symbol;
Platform platform;
MDConsumerChannel channel;
// Can add to handles without invalidating readers (no lock needed)
// When we recover we swap a new vector into this unique_ptr and delayed-delete
shared_ptr<ConcurrentAppendCollection<DatedHandle>> handles;
std::string mic;
};
void convert(PriceSize ps, PriceSizeMessage* pps) {
pps->set_mantissa(ps.mantissa());
pps->set_exponent(ps.exponent());
pps->set_size(ps.size());
}
void set_price(PriceSize ps, co2::TradeMessage* ptrade) {
ptrade->set_mantissa(ps.mantissa());
ptrade->set_exponent(ps.exponent());
ptrade->set_size(ps.size());
}
void set_price(const IntPrice& price, co2::PriceMessage* dest) {
dest->set_mantissa(price.mantissa());
dest->set_exponent(price.exponent());
}
void copy_misc(IntPrice settlement, MiscData misc, co2::MiscMessage* pmisc) {
set_price(settlement, pmisc->mutable_settlement());
pmisc->set_settlement_date(misc.settlement_date.to_short());
set_price(misc.previous_settlement_price, pmisc->mutable_previous_settlement_price());
pmisc->set_previous_settlement_date(misc.previous_settlement_date.to_short());
pmisc->set_open_price(misc.open_price);
pmisc->set_open_date(misc.open_date.to_short());
pmisc->set_close_price(misc.close_price);
pmisc->set_close_date(misc.close_date.to_short());
//pmisc->set_fixing_price(misc.fixing_price);
pmisc->set_session_high(misc.session_high);
pmisc->set_session_low(misc.session_low);
//pmisc->set_session_high_bid(misc.session_high_bid);
//pmisc->set_session_low_offer(misc.session_low_offer);
}
// Handles market data queries and responds
class QueryServerImpl : public IQueryServer {
MDClient& _client;
MDConsumer _consumer;
int _sub;
boost::container::flat_map<UserSymbol,shared_ptr<MDConsumerGroup>> _groups;
public:
QueryServerImpl(MDClient& client, metrics::MetricsContainer& stats) :
_client(client),
_consumer(client.current_consumer()),
_sub(0),
_groups()
{
for (const auto& sym : client.symbols()) {
_groups.emplace(std::make_pair(sym, std::make_unique<MDConsumerGroup>(sym, _consumer, stats)));
}
_groups.shrink_to_fit();
_sub = client.on_consumer().subscribe([=](const MDConsumer& c) { handle_recovery(c); });
subscribe_securities();
}
QueryServerImpl(const QueryServerImpl&)=delete;
~QueryServerImpl() {
_client.on_consumer().unsubscribe(_sub);
}
void subscribe_securities() {
auto id = _consumer.id();
_consumer.on_security([this,id](const SecurityPtr&, MDConsumerHandle handle) {
if (_consumer.id() == id) {
auto iter = _groups.find(handle.get_usersymbol());
bts_assert(iter != _groups.end());
const auto& pgroup = iter->second;
bts_assert(pgroup);
pgroup->add_security(handle);
}
});
}
// not-reentrant
void handle_recovery(const MDConsumer& consumer) {
MDConsumer temp(consumer);
if (temp.all_heartbeats_ok()) {
_consumer.swap(temp);
LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id());
for (auto& pair : _groups) {
pair.second->set_consumer(_consumer);
}
// _handle_index.clear();
subscribe_securities();
utils::Dispatcher::background().post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000);
}
}
// the hot part
void fill(const SecurityChannel& chan, MDConsumerGroup& group, ChannelReply* reply,
const Entitlements* entitlements, bool bypass_entitlement) const {
bts_assert(reply);
reply->mutable_channel()->CopyFrom(chan);
reply->set_platform(group.platform);
reply->mutable_channel()->set_time(TimeStamp::now().to_uint64());
bool entitled = bypass_entitlement || chan.exchange() == api::VENUE_BTS;
if(entitlements && !entitled)
entitled = entitlements->count(group.mic);
reply->mutable_channel()->set_entitled(entitled);
auto flags = (MDRequest::Flags)chan.flags();
// Get a copy of handles. If recovery happens during this call
// we can safely continue using this pointer for 60 seconds
auto phandles = group.handles;
if (!phandles) return;
ChannelUpdate temp_update;
for (const auto& instrument : *phandles) {
bool snapshot = false;
MDConsumerHandle handle = instrument.handle;
temp_update.Clear();
bts_assert_d(handle.is_initialized());
if (!handle.is_initialized()) continue;
ChannelUpdate* pupd = &temp_update;
pupd->set_key(instrument.handle.key());
// If the security was just discovered try to ensure all data is sent
if (instrument.discovered > TimeStamp(chan.time()))
{
auto security = handle.get_normalized_security();
pupd->mutable_security()->CopyFrom(security);
snapshot = true;
}
//if we are not entitled AND this is the first request since we lost our entitlement, zero out books
if (!entitled && chan.entitled() ) {
if (HASFLAG(flags, MDRequest::BBO) || HASFLAG(flags, MDRequest::DIRECT)) {
auto* p = pupd->mutable_direct();
for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) {
convert(NO_MARKET, p->add_bids());
convert(NO_MARKET, p->add_asks());
}
}
if ((HASFLAG(flags, MDRequest::IMPLIED) || HASFLAG(flags, MDRequest::BBO)) && !handle.is_synthetic()) {
auto* p = pupd->mutable_implied();
for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) {
convert(NO_MARKET, p->add_bids());
convert(NO_MARKET, p->add_asks());
}
}
}
if(entitled) {
// If we are entitled now but this is the first request where we are ensure we send all values
if(!chan.entitled()) snapshot = true;
if (HASFLAG(flags, MDRequest::BBO) || HASFLAG(flags, MDRequest::DIRECT)) {
std::array<PriceSize, COCONUT_MAX_DEPTH> bids, asks;
TimeStamp time(snapshot ? 0 : chan.time());
size_t copy_depth = HASFLAG(flags, MDRequest::DIRECT) ? COCONUT_MAX_DEPTH : 1;
auto result = handle.get_direct_book(time, copy_depth, bids.data(), asks.data());
if (result == HandleReadResult::SUCCESS) {
auto* p = pupd->mutable_direct();
for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) {
PriceSize b = bids[x];
if (b != NO_MARKET) {
convert(b, p->add_bids());
}
PriceSize a = asks[x];
if (a != NO_MARKET) {
convert(a, p->add_asks());
}
}
}
else if (result == HandleReadResult::BUSY) {
LOG(md,Info, "Busy read on handle {}. Retry on next request", instrument.handle.key());
// If we get a busy reading BBO (which happens 1-2x per day for a single relay client)
// We move the timestamp back so the next request will try to read this value again.
// ALTERNATIVE: We could spin until the value was not busy but it could be that the writer
// was context switched out and we will have to wait multiple ms to unblock the request.
// This fix assumes it is better to send the update now with 1 stale value than to block the call
// and allow all the values to be stale until this one can be read.
reply->mutable_channel()->set_time(chan.time());
}
}
if ((HASFLAG(flags, MDRequest::IMPLIED) || HASFLAG(flags, MDRequest::BBO)) && !handle.is_synthetic()) {
std::array<PriceSize, COCONUT_MAX_IMPLIED_DEPTH> bids, asks;
TimeStamp time(snapshot ? 0 : chan.time());
size_t copy_depth = HASFLAG(flags, MDRequest::IMPLIED) ? COCONUT_MAX_IMPLIED_DEPTH : 1;
auto result = handle.get_implied_book(time, copy_depth, bids.data(), asks.data());
if (result == HandleReadResult::SUCCESS) {
auto* p = pupd->mutable_implied();
for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) {
PriceSize b = bids[x];
if (b != NO_MARKET) {
convert(b, p->add_bids());
}
PriceSize a = asks[x];
if (a != NO_MARKET) {
convert(a, p->add_asks());
}
}
}
else if (result == HandleReadResult::BUSY) {
LOG(md,Info, "Busy read on handle (implied) {}. Retry on next request", instrument.handle.key());
reply->mutable_channel()->set_time(chan.time());
}
}
if (HASFLAG(flags, MDRequest::LAST_TRADE)) {
TimeStamp time(snapshot ? 0 : chan.time());
PriceSize price;
datetime::LLDateTime trade_time;
size_t volume; int from_spread;
if (handle.get_last_trade(time, price, trade_time, &volume, &from_spread)) {
auto* trade = pupd->mutable_last_trade();
trade->set_key(handle.key());
trade->set_time(trade_time);
trade->set_total_volume(volume);
trade->set_from_spread(from_spread);
set_price(price, trade);
}
}
if (HASFLAG(flags, MDRequest::MISC)) {
TimeStamp time(snapshot ? 0 : chan.time());
if (handle.misc_timestamp() > time) {
copy_misc(handle.settlement_price(), handle.misc(), pupd->mutable_misc());
}
}
}
if (pupd->has_direct() ||
pupd->has_implied() ||
pupd->has_last_trade() ||
pupd->has_misc() ||
pupd->has_security()) {
reply->add_updates()->CopyFrom(*pupd);
}
}
reply->mutable_channel()->set_segment_id(group.channel.create_time().to_uint64());
bool is_same_segment = (chan.segment_id() == group.channel.create_time().to_uint64());
if (HASFLAG(flags, MDRequest::TRADE_LOG)) {
int64_t request_cursor = is_same_segment ? chan.trade_cursor() : 0;
auto cursor = group.channel.read_trades(*reply->mutable_trade_log(), request_cursor);
if (cursor != chan.trade_cursor()) {
reply->mutable_channel()->set_trade_cursor(cursor);
}
}
if (HASFLAG(flags, MDRequest::RFQ)) {
int64_t request_cursor = is_same_segment ? chan.rfq_cursor() : 0;
auto cursor = group.channel.read_rfqs(*reply->mutable_rfq_log(), request_cursor);
if (cursor != chan.rfq_cursor()) {
reply->mutable_channel()->set_rfq_cursor(cursor);
}
}
}
void query(const MDRequest& request, MDReply* reply, const Entitlements* entitlements, bool bypass_entitlement) const {
bts_assert(reply != nullptr);
for (auto& ch : request.channels()) {
auto groupiter = _groups.find(UserSymbol(ch.symbol(), ch.exchange(), ch.type()));
if (groupiter != _groups.end()) {
auto& group = groupiter->second;
fill(ch, *group, reply->add_channels(), entitlements, bypass_entitlement);
}
}
}
void get_names(ExpirationNameListReply* reply) const {
bts_assert(reply != nullptr);
Security security;
for (auto& groupiter : _groups) {
const auto& group = groupiter.second;
ExpirationNameList* psec = reply->add_securities();
psec->mutable_product()->CopyFrom(group->symbol.to_product());
for(const auto& h : *(group->handles)) {
security.CopyFrom(h.handle.get_normalized_security());
auto x = psec->mutable_names();
if (security.ExchangeData_case() == Security::kGlbx) {
google::protobuf::MapPair<uint32_t,std::string> v(h.handle.key(), security.glbx().security_desc());
x->insert(v);
}
else if (security.ExchangeData_case() == Security::kIce) {
google::protobuf::MapPair<uint32_t,std::string> v(h.handle.key(), security.ice().security_desc());
x->insert(v);
}
// There is currently no ExchangeData for synthetics
}
}
}
void get_securities(google::protobuf::RepeatedField<security_key> keys, api::SecurityList* result) const {
bts_assert(result);
// Ensure keys are unique
std::set<security_key> unique_keys(keys.begin(), keys.end());
for (auto k : unique_keys) {
auto ptr = _consumer.get_security(SecurityKey(k));
if (ptr) {
result->add_security()->CopyFrom(*ptr);
}
}
}
};
struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase {
const QueryServerImpl& _db;
utils::Dispatcher& _threadpool;
std::shared_ptr<std::unordered_map<uint32_t,Entitlements>> _entitlements;
#ifndef _WIN32
std::shared_ptr<metrics::HistogramMetric<5>> _time,_size;
#endif
MDRelayServer(const QueryServerImpl& db, utils::Dispatcher& threadpool, bts::api::ServiceContext ctx, metrics::MetricsContainer& stats) :
bts::co2::AuthenticatedMarketDataRelayBase(ctx),
_db(db), _threadpool(threadpool), _entitlements()
#ifndef _WIN32
, _time(std::make_shared<metrics::HistogramMetric<5>>("spark_qserver_response_time_ms", std::array<double,5>({ 5.0, 10.0, 20.0, 50.0, 100.0 }))),
_size(std::make_shared<metrics::HistogramMetric<5>>("spark_qserver_size_bytes", std::array<double,5>({ 10240 /*10k*/, 102400 /*100k*/, 1024*1024 /*1mb*/, 1024*1024*10 /*10mb*/, 1024*1024*100 /*100mb*/ })))
{
stats.publish(_time);
stats.publish(_size);
}
#else
{}
#endif
void update_entitlements(std::shared_ptr<std::unordered_map<uint32_t, Entitlements>>& entitlements) {
_entitlements = entitlements;
}
static const Entitlements* get_entitlements(uint32_t user_id, std::shared_ptr<std::unordered_map<uint32_t,Entitlements>>& entitlements) {
if (entitlements) {
auto iter = entitlements->find(user_id);
if (iter != entitlements->end()) {
return &iter->second;
}
}
return nullptr;
}
virtual void requestmd(const api::AuthToken& token, const MDRequest& request, const std::function<void(const MDReply&)>& reply) override {
TimeStamp start = TimeStamp::now();
_threadpool.post([=]() {
google::protobuf::Arena arena;
MDReply* preply = google::protobuf::Arena::CreateMessage<MDReply>(&arena);
// addref to keep shared ptr alive
api::user::TokenPayload payload;
auto ents = _entitlements;
const Entitlements* user_entitlement = nullptr;
bool bypass_entitlement = false;
if (api::auth::decode_token(token, &payload)) {
bypass_entitlement = payload.wheel();
if (!bypass_entitlement) {
user_entitlement = get_entitlements(payload.id(), ents);
}
}
_db.query(request, preply, user_entitlement, bypass_entitlement);
reply(*preply);
#ifndef _WIN32
auto elapsed = (TimeStamp::now() - start);
_time->observe(elapsed.milliseconds());
_size->observe(preply->ByteSize());
#endif
});
}
virtual void request_products(const api::AuthToken&, const api::VoidMessage&, const std::function<void(const ProductListReply&)>& reply) override {
ProductListReply list;
std::set<UserSymbol> syms;
get_system_md_symbols(syms);
for(const auto& i : syms)
{
auto prod = list.add_products();
prod->set_exchange(i.exchange());
prod->set_type(i.type());
prod->set_symbol(i.symbol());
}
reply(list);
}
virtual void request_names(const api::AuthToken&, const api::VoidMessage&, const std::function<void(const ExpirationNameListReply&)>& reply) override {
_threadpool.post([=]() {
google::protobuf::Arena arena;
ExpirationNameListReply* preply = google::protobuf::Arena::CreateMessage<ExpirationNameListReply>(&arena);
_db.get_names(preply);
reply(*preply);
});
}
virtual void get_securities(const api::AuthToken&, const SecurityKeyList& keys, const std::function<void(const api::SecurityList&)>& reply) override {
_threadpool.post([=]() {
google::protobuf::Arena arena;
api::SecurityList* preply = google::protobuf::Arena::CreateMessage<api::SecurityList>(&arena);
_db.get_securities(keys.keys(), preply);
reply(*preply);
});
}
};
struct QueryServer {
QueryServer(utils::Dispatcher& threadpool,
const bts::api::ServiceContext& ctx,
MDClient& client,
metrics::MetricsContainer& stats):
db(client, stats),
relay(db, threadpool, ctx, stats),
admin_client(ctx.locate<api::admin::AdminService>())
{ }
QueryServerImpl db;
MDRelayServer relay;
api::admin::AdminService admin_client;
rxcpp::composite_subscription entitlement_sub;
};
QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, MDClient& client, metrics::MetricsContainer& stats) {
auto& threadpool = utils::Dispatcher::get_named("MDQuery");
QueryServerPtr container = std::make_shared<QueryServer>(threadpool, ctx, client, stats);
// Subscribe to entitlement data
container->entitlement_sub =
container->admin_client.subscribe_md_entitlement_data().subscribe([=](const api::admin::Md_entitlement_dataUpdate& update) {
const auto& all_updates = update.all();
utils::Dispatcher::background().post([all_updates, container]()
{
auto entitlements = std::make_shared<std::unordered_map<uint32_t, Entitlements>>();
for (const auto& row : all_updates)
{
if (row && row->enabled())
{
auto iter = entitlements->find(row->user_id());
if (iter == entitlements->end()) {
Entitlements ents;
ents.insert(row->mic());
entitlements->insert(iter, std::make_pair(row->user_id(), std::move(ents)));
}
else {
iter->second.insert(row->mic());
}
}
}
container->relay.update_entitlements(entitlements);
});
});
// Add a few threads to keep md latency low
for (int x = 1; x < std::min(4, os::num_cores()); ++x) {
threadpool.add_thread();
}
return container;
}
std::unique_ptr<IQueryServer> create_query_server(MDClient& mdc, metrics::MetricsContainer& mets) {
return std::make_unique<QueryServerImpl>(mdc, mets);
}
}}//namespace
#endif