#ifndef _WIN32 #include "QueryServer.h" #include "lib/config/config.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/TimeStamp.h" #include "lib/utils/helper_macros.h" #include "lib/utils/AuthToken.h" #include "../MDConsumer.h" #include "../MDClient.h" #include "../PlatformLoader.h" #include "../MBOProcessor.h" #include "lib/logging/logging.h" #include "lib/config/protos/system.pb.h" #include "lib/md/protos/co2.pb.h" #include "lib/md/SecurityCache.h" #include "lib/api/apicore.pb.h" #include "lib/api/services/MarketDataRelay.server.pb.h" #include "lib/api/public/ServiceContext.h" #include "lib/api/services/AdminService.client.pb.h" #include "lib/api/service_helpers/AdminServiceUtils.h" #include "lib/config/protos/cme_reference_data.pb.h" #include "lib/utils/ConcurrentAppendCollection.h" #ifndef _WIN32 #include "lib/metrics/metrics.h" #endif #include #include #include #include #include #include #include #include using namespace bts::co2; using std::unique_ptr; using std::shared_ptr; namespace bts { namespace md { /* * NORMAL OPERATION: * Client sends time=0 to server * Server sends complete snapshot back (time=502) * client sends time=502 to server * Server sends any changes that have happened after time=502 and any securities discovered since time=502 * * ERROR RECOVERY: * Client sends time=0 to server * Server sends complete snapshot back (time=502) * Server detects heartbeat failure and recovers. All timestamps > 502 * Client sends time=502 to server * Server sends complete snapshot back (time=605) * - Since the server shared memory has been recreated the client will automatically overwrite all stale data. * - If a security has been deleted the client will leave it frozen (OK) * * We scan for bad heartbeats on a different time * If a bad heartbeat is detected we try to construct a new Consumer * When a good consumer is created we std::move the old consumer and handles to delayed queue for deletion * This way we don't have to lock anything during the recovery process because all pointers will remain valid * for the duration of existing calls. * */ struct DatedHandle { TimeStamp discovered; MDConsumerHandle handle; }; struct MDConsumerGroup { // probably just have to send the consumer in from outside MDConsumerGroup(const UserSymbol& symbol, const MDConsumer& consumer, metrics::MetricsContainer& stats) : symbol(symbol), platform(platform_for_exchange(symbol.exchange())), channel(consumer.get_channel(symbol)), handles(), mic() { set_consumer(consumer); // create and publish a stat for this group #ifndef _WIN32 if (symbol.type() != SecurityType::SECURITYTYPE_SYNTHETIC) { std::vector labels; labels.push_back({ "symbol", symbol.symbol() }); labels.push_back({ "type", SecurityType_Name(symbol.type()).substr(13) /*SECURITYTYPE_*/ }); labels.push_back({ "exch", Venue_Name(symbol.exchange()).substr(6) /*VENUE_*/ }); stats.publish( std::make_shared("spark_md_book_updates", [=]() { return channel.direct_count() + channel.implied_count(); }, labels)); stats.publish( std::make_shared("spark_md_security_count", [=]() { return channel.security_count(); }, labels)); stats.publish( std::make_shared("spark_md_trade_count", [=]() { return channel.trade_count(); }, labels)); stats.publish( std::make_shared("spark_md_gap_count", [=]() { auto g = channel.gap_count(); return g == 0 ? os::nan() : g; }, labels)); } #endif } MDConsumerGroup(const MDConsumerGroup& other) = delete; // recovery has occured and we need to clear our our collection of handles and wait for new ones void set_consumer(const MDConsumer& consumer) { // create a new set of handles and swap with the old. // If the new consumer is bad we still have to swap our handles because the consumer that owns our handles will be scheduled for deletion auto temphandles = std::make_shared>(); handles.swap(temphandles); channel = consumer.get_channel(symbol); } void add_security(MDConsumerHandle& handle) { bts_assert(handles); bts_assert_d(handle.is_initialized()); bts_assert(handle.get_usersymbol() == symbol); DatedHandle h = { TimeStamp::now(), handle }; handles->push_back(h); if(this->mic == "") set_mic(h); } void set_mic(const DatedHandle& h) { if(channel.exchange() == api::VENUE_ICE || channel.exchange() == api::VENUE_SIMICE) { this->mic = h.handle.get_normalized_security().ice().mic_code(); } else if(channel.exchange() == api::VENUE_GLBX || channel.exchange() == api::VENUE_SIM) { const auto& cme_symb_mics = config::get().cme_mic(); for(auto mic=cme_symb_mics.begin();mic!=cme_symb_mics.end();mic++) { const auto& micsym = mic->first; if(micsym == symbol.symbol()) { this->mic = api::CMEMics_Name(mic->second); break; } } } else if(channel.exchange() == api::VENUE_BTS || channel.exchange() == api::VENUE_UNSET) { this->mic = "BTS"; } else bts_assert_msg(false, "unknown venue in QueryServer"); } UserSymbol symbol; Platform platform; MDConsumerChannel channel; // Can add to handles without invalidating readers (no lock needed) // When we recover we swap a new vector into this unique_ptr and delayed-delete shared_ptr> handles; std::string mic; }; void convert(PriceSize ps, PriceSizeMessage* pps) { pps->set_mantissa(ps.mantissa()); pps->set_exponent(ps.exponent()); pps->set_size(ps.size()); } void set_price(PriceSize ps, co2::TradeMessage* ptrade) { ptrade->set_mantissa(ps.mantissa()); ptrade->set_exponent(ps.exponent()); ptrade->set_size(ps.size()); } void set_price(const IntPrice& price, co2::PriceMessage* dest) { dest->set_mantissa(price.mantissa()); dest->set_exponent(price.exponent()); } void copy_misc(IntPrice settlement, MiscData misc, co2::MiscMessage* pmisc) { set_price(settlement, pmisc->mutable_settlement()); pmisc->set_settlement_date(misc.settlement_date.to_short()); set_price(misc.previous_settlement_price, pmisc->mutable_previous_settlement_price()); pmisc->set_previous_settlement_date(misc.previous_settlement_date.to_short()); pmisc->set_open_price(misc.open_price); pmisc->set_open_date(misc.open_date.to_short()); pmisc->set_close_price(misc.close_price); pmisc->set_close_date(misc.close_date.to_short()); //pmisc->set_fixing_price(misc.fixing_price); pmisc->set_session_high(misc.session_high); pmisc->set_session_low(misc.session_low); //pmisc->set_session_high_bid(misc.session_high_bid); //pmisc->set_session_low_offer(misc.session_low_offer); } // Handles market data queries and responds class QueryServerImpl : public IQueryServer { MDClient& _client; MDConsumer _consumer; int _sub; boost::container::flat_map> _groups; public: QueryServerImpl(MDClient& client, metrics::MetricsContainer& stats) : _client(client), _consumer(client.current_consumer()), _sub(0), _groups() { for (const auto& sym : client.symbols()) { _groups.emplace(std::make_pair(sym, std::make_unique(sym, _consumer, stats))); } _groups.shrink_to_fit(); _sub = client.on_consumer().subscribe([=](const MDConsumer& c) { handle_recovery(c); }); subscribe_securities(); } QueryServerImpl(const QueryServerImpl&)=delete; ~QueryServerImpl() { _client.on_consumer().unsubscribe(_sub); } void subscribe_securities() { auto id = _consumer.id(); _consumer.on_security([this,id](const SecurityPtr&, MDConsumerHandle handle) { if (_consumer.id() == id) { auto iter = _groups.find(handle.get_usersymbol()); bts_assert(iter != _groups.end()); const auto& pgroup = iter->second; bts_assert(pgroup); pgroup->add_security(handle); } }); } // not-reentrant void handle_recovery(const MDConsumer& consumer) { MDConsumer temp(consumer); if (temp.all_heartbeats_ok()) { _consumer.swap(temp); LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); for (auto& pair : _groups) { pair.second->set_consumer(_consumer); } // _handle_index.clear(); subscribe_securities(); utils::Dispatcher::background().post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000); } } // the hot part void fill(const SecurityChannel& chan, MDConsumerGroup& group, ChannelReply* reply, const Entitlements* entitlements, bool bypass_entitlement) const { bts_assert(reply); reply->mutable_channel()->CopyFrom(chan); reply->set_platform(group.platform); reply->mutable_channel()->set_time(TimeStamp::now().to_uint64()); bool entitled = bypass_entitlement || chan.exchange() == api::VENUE_BTS; if(entitlements && !entitled) entitled = entitlements->count(group.mic); reply->mutable_channel()->set_entitled(entitled); auto flags = (MDRequest::Flags)chan.flags(); // Get a copy of handles. If recovery happens during this call // we can safely continue using this pointer for 60 seconds auto phandles = group.handles; if (!phandles) return; ChannelUpdate temp_update; for (const auto& instrument : *phandles) { bool snapshot = false; MDConsumerHandle handle = instrument.handle; temp_update.Clear(); bts_assert_d(handle.is_initialized()); if (!handle.is_initialized()) continue; ChannelUpdate* pupd = &temp_update; pupd->set_key(instrument.handle.key()); // If the security was just discovered try to ensure all data is sent if (instrument.discovered > TimeStamp(chan.time())) { auto security = handle.get_normalized_security(); pupd->mutable_security()->CopyFrom(security); snapshot = true; } //if we are not entitled AND this is the first request since we lost our entitlement, zero out books if (!entitled && chan.entitled() ) { if (HASFLAG(flags, MDRequest::BBO) || HASFLAG(flags, MDRequest::DIRECT)) { auto* p = pupd->mutable_direct(); for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { convert(NO_MARKET, p->add_bids()); convert(NO_MARKET, p->add_asks()); } } if ((HASFLAG(flags, MDRequest::IMPLIED) || HASFLAG(flags, MDRequest::BBO)) && !handle.is_synthetic()) { auto* p = pupd->mutable_implied(); for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) { convert(NO_MARKET, p->add_bids()); convert(NO_MARKET, p->add_asks()); } } } if(entitled) { // If we are entitled now but this is the first request where we are ensure we send all values if(!chan.entitled()) snapshot = true; if (HASFLAG(flags, MDRequest::BBO) || HASFLAG(flags, MDRequest::DIRECT)) { std::array bids, asks; TimeStamp time(snapshot ? 0 : chan.time()); size_t copy_depth = HASFLAG(flags, MDRequest::DIRECT) ? COCONUT_MAX_DEPTH : 1; auto result = handle.get_direct_book(time, copy_depth, bids.data(), asks.data()); if (result == HandleReadResult::SUCCESS) { auto* p = pupd->mutable_direct(); for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { PriceSize b = bids[x]; if (b != NO_MARKET) { convert(b, p->add_bids()); } PriceSize a = asks[x]; if (a != NO_MARKET) { convert(a, p->add_asks()); } } } else if (result == HandleReadResult::BUSY) { LOG(md,Info, "Busy read on handle {}. Retry on next request", instrument.handle.key()); // If we get a busy reading BBO (which happens 1-2x per day for a single relay client) // We move the timestamp back so the next request will try to read this value again. // ALTERNATIVE: We could spin until the value was not busy but it could be that the writer // was context switched out and we will have to wait multiple ms to unblock the request. // This fix assumes it is better to send the update now with 1 stale value than to block the call // and allow all the values to be stale until this one can be read. reply->mutable_channel()->set_time(chan.time()); } } if ((HASFLAG(flags, MDRequest::IMPLIED) || HASFLAG(flags, MDRequest::BBO)) && !handle.is_synthetic()) { std::array bids, asks; TimeStamp time(snapshot ? 0 : chan.time()); size_t copy_depth = HASFLAG(flags, MDRequest::IMPLIED) ? COCONUT_MAX_IMPLIED_DEPTH : 1; auto result = handle.get_implied_book(time, copy_depth, bids.data(), asks.data()); if (result == HandleReadResult::SUCCESS) { auto* p = pupd->mutable_implied(); for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) { PriceSize b = bids[x]; if (b != NO_MARKET) { convert(b, p->add_bids()); } PriceSize a = asks[x]; if (a != NO_MARKET) { convert(a, p->add_asks()); } } } else if (result == HandleReadResult::BUSY) { LOG(md,Info, "Busy read on handle (implied) {}. Retry on next request", instrument.handle.key()); reply->mutable_channel()->set_time(chan.time()); } } if (HASFLAG(flags, MDRequest::LAST_TRADE)) { TimeStamp time(snapshot ? 0 : chan.time()); PriceSize price; datetime::LLDateTime trade_time; size_t volume; int from_spread; if (handle.get_last_trade(time, price, trade_time, &volume, &from_spread)) { auto* trade = pupd->mutable_last_trade(); trade->set_key(handle.key()); trade->set_time(trade_time); trade->set_total_volume(volume); trade->set_from_spread(from_spread); set_price(price, trade); } } if (HASFLAG(flags, MDRequest::MISC)) { TimeStamp time(snapshot ? 0 : chan.time()); if (handle.misc_timestamp() > time) { copy_misc(handle.settlement_price(), handle.misc(), pupd->mutable_misc()); } } } if (pupd->has_direct() || pupd->has_implied() || pupd->has_last_trade() || pupd->has_misc() || pupd->has_security()) { reply->add_updates()->CopyFrom(*pupd); } } reply->mutable_channel()->set_segment_id(group.channel.create_time().to_uint64()); bool is_same_segment = (chan.segment_id() == group.channel.create_time().to_uint64()); if (HASFLAG(flags, MDRequest::TRADE_LOG)) { int64_t request_cursor = is_same_segment ? chan.trade_cursor() : 0; auto cursor = group.channel.read_trades(*reply->mutable_trade_log(), request_cursor); if (cursor != chan.trade_cursor()) { reply->mutable_channel()->set_trade_cursor(cursor); } } if (HASFLAG(flags, MDRequest::RFQ)) { int64_t request_cursor = is_same_segment ? chan.rfq_cursor() : 0; auto cursor = group.channel.read_rfqs(*reply->mutable_rfq_log(), request_cursor); if (cursor != chan.rfq_cursor()) { reply->mutable_channel()->set_rfq_cursor(cursor); } } } void query(const MDRequest& request, MDReply* reply, const Entitlements* entitlements, bool bypass_entitlement) const { bts_assert(reply != nullptr); for (auto& ch : request.channels()) { auto groupiter = _groups.find(UserSymbol(ch.symbol(), ch.exchange(), ch.type())); if (groupiter != _groups.end()) { auto& group = groupiter->second; fill(ch, *group, reply->add_channels(), entitlements, bypass_entitlement); } } } void get_names(ExpirationNameListReply* reply) const { bts_assert(reply != nullptr); Security security; for (auto& groupiter : _groups) { const auto& group = groupiter.second; ExpirationNameList* psec = reply->add_securities(); psec->mutable_product()->CopyFrom(group->symbol.to_product()); for(const auto& h : *(group->handles)) { security.CopyFrom(h.handle.get_normalized_security()); auto x = psec->mutable_names(); if (security.ExchangeData_case() == Security::kGlbx) { google::protobuf::MapPair v(h.handle.key(), security.glbx().security_desc()); x->insert(v); } else if (security.ExchangeData_case() == Security::kIce) { google::protobuf::MapPair v(h.handle.key(), security.ice().security_desc()); x->insert(v); } // There is currently no ExchangeData for synthetics } } } void get_securities(google::protobuf::RepeatedField keys, api::SecurityList* result) const { bts_assert(result); // Ensure keys are unique std::set unique_keys(keys.begin(), keys.end()); for (auto k : unique_keys) { auto ptr = _consumer.get_security(SecurityKey(k)); if (ptr) { result->add_security()->CopyFrom(*ptr); } } } }; struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { const QueryServerImpl& _db; utils::Dispatcher& _threadpool; std::shared_ptr> _entitlements; #ifndef _WIN32 std::shared_ptr> _time,_size; #endif MDRelayServer(const QueryServerImpl& db, utils::Dispatcher& threadpool, bts::api::ServiceContext ctx, metrics::MetricsContainer& stats) : bts::co2::AuthenticatedMarketDataRelayBase(ctx), _db(db), _threadpool(threadpool), _entitlements() #ifndef _WIN32 , _time(std::make_shared>("spark_qserver_response_time_ms", std::array({ 5.0, 10.0, 20.0, 50.0, 100.0 }))), _size(std::make_shared>("spark_qserver_size_bytes", std::array({ 10240 /*10k*/, 102400 /*100k*/, 1024*1024 /*1mb*/, 1024*1024*10 /*10mb*/, 1024*1024*100 /*100mb*/ }))) { stats.publish(_time); stats.publish(_size); } #else {} #endif void update_entitlements(std::shared_ptr>& entitlements) { _entitlements = entitlements; } static const Entitlements* get_entitlements(uint32_t user_id, std::shared_ptr>& entitlements) { if (entitlements) { auto iter = entitlements->find(user_id); if (iter != entitlements->end()) { return &iter->second; } } return nullptr; } virtual void requestmd(const api::AuthToken& token, const MDRequest& request, const std::function& reply) override { TimeStamp start = TimeStamp::now(); _threadpool.post([=]() { google::protobuf::Arena arena; MDReply* preply = google::protobuf::Arena::CreateMessage(&arena); // addref to keep shared ptr alive api::user::TokenPayload payload; auto ents = _entitlements; const Entitlements* user_entitlement = nullptr; bool bypass_entitlement = false; if (api::auth::decode_token(token, &payload)) { bypass_entitlement = payload.wheel(); if (!bypass_entitlement) { user_entitlement = get_entitlements(payload.id(), ents); } } _db.query(request, preply, user_entitlement, bypass_entitlement); reply(*preply); #ifndef _WIN32 auto elapsed = (TimeStamp::now() - start); _time->observe(elapsed.milliseconds()); _size->observe(preply->ByteSize()); #endif }); } virtual void request_products(const api::AuthToken&, const api::VoidMessage&, const std::function& reply) override { ProductListReply list; std::set syms; get_system_md_symbols(syms); for(const auto& i : syms) { auto prod = list.add_products(); prod->set_exchange(i.exchange()); prod->set_type(i.type()); prod->set_symbol(i.symbol()); } reply(list); } virtual void request_names(const api::AuthToken&, const api::VoidMessage&, const std::function& reply) override { _threadpool.post([=]() { google::protobuf::Arena arena; ExpirationNameListReply* preply = google::protobuf::Arena::CreateMessage(&arena); _db.get_names(preply); reply(*preply); }); } virtual void get_securities(const api::AuthToken&, const SecurityKeyList& keys, const std::function& reply) override { _threadpool.post([=]() { google::protobuf::Arena arena; api::SecurityList* preply = google::protobuf::Arena::CreateMessage(&arena); _db.get_securities(keys.keys(), preply); reply(*preply); }); } }; struct QueryServer { QueryServer(utils::Dispatcher& threadpool, const bts::api::ServiceContext& ctx, MDClient& client, metrics::MetricsContainer& stats): db(client, stats), relay(db, threadpool, ctx, stats), admin_client(ctx.locate()) { } QueryServerImpl db; MDRelayServer relay; api::admin::AdminService admin_client; rxcpp::composite_subscription entitlement_sub; }; QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, MDClient& client, metrics::MetricsContainer& stats) { auto& threadpool = utils::Dispatcher::get_named("MDQuery"); QueryServerPtr container = std::make_shared(threadpool, ctx, client, stats); // Subscribe to entitlement data container->entitlement_sub = container->admin_client.subscribe_md_entitlement_data().subscribe([=](const api::admin::Md_entitlement_dataUpdate& update) { const auto& all_updates = update.all(); utils::Dispatcher::background().post([all_updates, container]() { auto entitlements = std::make_shared>(); for (const auto& row : all_updates) { if (row && row->enabled()) { auto iter = entitlements->find(row->user_id()); if (iter == entitlements->end()) { Entitlements ents; ents.insert(row->mic()); entitlements->insert(iter, std::make_pair(row->user_id(), std::move(ents))); } else { iter->second.insert(row->mic()); } } } container->relay.update_entitlements(entitlements); }); }); // Add a few threads to keep md latency low for (int x = 1; x < std::min(4, os::num_cores()); ++x) { threadpool.add_thread(); } return container; } std::unique_ptr create_query_server(MDClient& mdc, metrics::MetricsContainer& mets) { return std::make_unique(mdc, mets); } }}//namespace #endif