diff --git a/compose.yaml b/compose.yaml index d1c779ec9..334fcf039 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,6 +2,7 @@ services: sparkd: volumes: - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data + - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording build: dockerfile: src/app_sparkd/Dockerfile args: @@ -161,3 +162,6 @@ volumes: name: "md_publisher_22019" historic_md: name: "historic_md" + md_recording: + name: "md_recording" + external: true diff --git a/src/app_sparkd/sparkd/cmds/Run.cpp b/src/app_sparkd/sparkd/cmds/Run.cpp index df1d05cc7..6488b8a5f 100644 --- a/src/app_sparkd/sparkd/cmds/Run.cpp +++ b/src/app_sparkd/sparkd/cmds/Run.cpp @@ -10,7 +10,7 @@ #include "spark/md/MBOServiceServer.h" #include "spark/md/MDClient.h" #include "spark/md/SyntheticSecurityService.h" -#include "spark/md/relay/QueryServer.h" +#include "spark/md/relay/query_server.h" #include "spark/md/service/MarketDataServiceServer.h" #include "spark/metrics/metrics.h" #include "spark/metrics/system_metrics.h" diff --git a/src/app_sparkd/sparkd/cmds/Web.cpp b/src/app_sparkd/sparkd/cmds/Web.cpp index 595be4d8f..f41a98e40 100644 --- a/src/app_sparkd/sparkd/cmds/Web.cpp +++ b/src/app_sparkd/sparkd/cmds/Web.cpp @@ -6,7 +6,7 @@ #include "spark/config/protos/quickbus_cfg.pb.h" #include "spark/md/MDClient.h" #include "spark/md/SyntheticSecurityService.h" -#include "spark/md/relay/QueryServer.h" +#include "spark/md/relay/query_server.h" #include "spark/metrics/metrics.h" #include "spark/metrics/system_metrics.h" #include "spark/quickbus/core/Quickbus.h" diff --git a/src/lib_spark/cmake/GetWindowsClientSources.cmake b/src/lib_spark/cmake/GetWindowsClientSources.cmake index c4a989232..281f44aad 100644 --- a/src/lib_spark/cmake/GetWindowsClientSources.cmake +++ b/src/lib_spark/cmake/GetWindowsClientSources.cmake @@ -91,7 +91,11 @@ function(get_cpp_sources dest) ${PROJECT_NAME}/md/cme/MDP3Utils.cpp ${PROJECT_NAME}/md/common.cpp ${PROJECT_NAME}/md/ice/ICEUtils.cpp - ${PROJECT_NAME}/md/relay/QueryServer.cpp + ${PROJECT_NAME}/md/relay/internal/md_consumer_group.cpp + ${PROJECT_NAME}/md/relay/internal/md_relay_server.cpp + ${PROJECT_NAME}/md/relay/internal/query_server_impl.cpp + ${PROJECT_NAME}/md/relay/i_query_server.cpp + ${PROJECT_NAME}/md/relay/query_server.cpp ${PROJECT_NAME}/md/relay/RelayClient.cpp ${PROJECT_NAME}/md/relay/RelayRecorder.cpp ${PROJECT_NAME}/md/service/MarketDataServiceServer.cpp diff --git a/src/lib_spark/spark/md/relay/QueryServer.cpp b/src/lib_spark/spark/md/relay/QueryServer.cpp deleted file mode 100644 index d7dca2bb0..000000000 --- a/src/lib_spark/spark/md/relay/QueryServer.cpp +++ /dev/null @@ -1,770 +0,0 @@ -#ifndef _WIN32 -#include "QueryServer.h" -#include "spark/config/config.h" -#include "spark/domain/DomainSecurityUtils.h" -#include "spark/domain_model/Security.h" -#include "spark/md/SymbolLoader.h" -#include "spark/utils/AuthToken.h" -#include "spark/utils/Dispatcher.h" -#include "spark/utils/TimeStamp.h" -#include "spark/utils/bts_assert.h" -#include "spark/utils/helper_macros.h" - -#include "../MBOProcessor.h" -#include "../MDClient.h" -#include "../MDConsumer.h" - -#include "spark/logging/logging.h" - -#include "spark/api/apicore.pb.h" -#include "spark/api/public/ServiceContext.h" -#include "spark/api/service_helpers/AdminServiceUtils.h" -#include "spark/api/services/AdminService.client.pb.h" -#include "spark/api/services/MarketDataRelay.server.pb.h" -#include "spark/config/protos/cme_reference_data.pb.h" -#include "spark/config/protos/system_config.pb.h" -#include "spark/md/SecurityCache.h" -#include "spark/md/protos/co2.pb.h" -#include "spark/utils/ConcurrentAppendCollection.h" -#ifndef _WIN32 -#include "spark/metrics/metrics.h" -#endif - -#include -#include -#include -#include -#include -#include - -#include -#include - -using namespace bts::co2; -using bts::domain::DomainSecurityUtils; -using std::shared_ptr; -using std::string; -using std::unique_ptr; - -namespace bts { -namespace md { - -/* - * NORMAL OPERATION: - * Client sends time=0 to server - * Server sends complete snapshot back (time=502) - * client sends time=502 to server - * Server sends any changes that have happened after time=502 and any securities - * discovered since time=502 - * - * ERROR RECOVERY: - * Client sends time=0 to server - * Server sends complete snapshot back (time=502) - * Server detects heartbeat failure and recovers. All timestamps > 502 - * Client sends time=502 to server - * Server sends complete snapshot back (time=605) - * - Since the server shared memory has been recreated the client will - * automatically overwrite all stale data. - * - If a security has been deleted the client will leave it frozen (OK) - * - * We scan for bad heartbeats on a different time - * If a bad heartbeat is detected we try to construct a new Consumer - * When a good consumer is created we std::move the old consumer and handles to - * delayed queue for deletion This way we don't have to lock anything during the - * recovery process because all pointers will remain valid for the duration of - * existing calls. - * - */ -struct DatedHandle { - TimeStamp discovered; - MDConsumerHandle handle; -}; - -struct MDConsumerGroup { - // probably just have to send the consumer in from outside - MDConsumerGroup( - const UserSymbol &symbol, - const MDConsumer &consumer, - metrics::MetricsContainer &stats) - : symbol(symbol), channel(consumer.get_channel(symbol)), handles(), - mic() { - set_consumer(consumer); - - // create and publish a stat for this group -#ifndef _WIN32 - if (symbol.type() != SecurityType::SECURITYTYPE_SYNTHETIC) { - std::vector labels; - labels.push_back({"symbol", symbol.symbol()}); - labels.push_back( - {"type", - SecurityType_Name(symbol.type()).substr(13) /*SECURITYTYPE_*/}); - labels.push_back( - {"exch", Venue_Name(symbol.exchange()).substr(6) /*VENUE_*/}); - stats.publish(std::make_shared( - "spark_md_book_updates", - [=, this]() { - return channel.direct_count() + channel.implied_count(); - }, - labels)); - stats.publish(std::make_shared( - "spark_md_security_count", - [=, this]() { return channel.security_count(); }, labels)); - stats.publish(std::make_shared( - "spark_md_trade_count", - [=, this]() { return channel.trade_count(); }, labels)); - stats.publish(std::make_shared( - "spark_md_gap_count", - [=, this]() { - auto g = channel.gap_count(); - return g == 0 ? os::nan() : g; - }, - labels)); - } -#endif - } - MDConsumerGroup(const MDConsumerGroup &other) = delete; - - // recovery has occured and we need to clear our our collection of handles - // and wait for new ones - void set_consumer(const MDConsumer &consumer) { - // create a new set of handles and swap with the old. - // If the new consumer is bad we still have to swap our handles because - // the consumer that owns our handles will be scheduled for deletion - auto temphandles = - std::make_shared>(); - handles.swap(temphandles); - channel = consumer.get_channel(symbol); - } - - void add_security(MDConsumerHandle &handle) { - bts_assert(handles); - bts_assert_d(handle.is_initialized()); - bts_assert(handle.get_usersymbol() == symbol); - DatedHandle h = {TimeStamp::now(), handle}; - handles->push_back(h); - if (this->mic == "") - set_mic(h); - } - - void set_mic(const DatedHandle &h) { - switch (channel.exchange()) { - case api::VENUE_ICE: - case api::VENUE_SIMICE: - this->mic = h.handle.get_normalized_security().ice().mic_code(); - break; - case api::VENUE_GLBX: - case api::VENUE_SIM: { - const auto &cme_symb_mics = - config::get().cme_mic(); - for (auto mic = cme_symb_mics.begin(); mic != cme_symb_mics.end(); - mic++) { - const auto &micsym = mic->first; - if (micsym == symbol.symbol()) { - this->mic = api::CMEMics_Name(mic->second); - break; - } - } - break; - } - case api::VENUE_BTS: - case api::VENUE_UNSET: - this->mic = "BTS"; - break; - case api::VENUE_DERIBIT: - this->mic = "DERIBIT"; - break; - case api::VENUE_FTX: - this->mic = "FTX"; - break; - case api::VENUE_BINANCE: - this->mic = "BINANCE"; - break; - case api::VENUE_BINANCE_COIN_FUTURES: - this->mic = "BINANCE_COIN_FUTURES"; - break; - case api::VENUE_BINANCE_US: - this->mic = "BINANCE_US"; - break; - case api::VENUE_COINBASE: - this->mic = "COINBASE"; - break; - case api::VENUE_BINANCE_USDS_FUTURES: - this->mic = "BINANCE_USDS_FUTURES"; - break; - case api::VENUE_OKX: - this->mic = "OKX"; - break; - case api::VENUE_CBOE: - this->mic = "CBOE"; - break; - case api::VENUE_TWSE: - this->mic = "TWSE"; - break; - case api::VENUE_TAIFEX: - this->mic = "TAIFEX"; - break; - case api::VENUE_BYBIT: - this->mic = "BYBIT"; - break; - case api::VENUE_GATEIO: - this->mic = "GATEIO"; - break; - case api::VENUE_CBOECFE: - case api::Venue_INT_MIN_SENTINEL_DO_NOT_USE_: - case api::Venue_INT_MAX_SENTINEL_DO_NOT_USE_: - bts_assert(false); - this->mic = "UNKNOWN"; - break; - } - } - - UserSymbol symbol; - MDConsumerChannel channel; - - // Can add to handles without invalidating readers (no lock needed) - // When we recover we swap a new vector into this unique_ptr and - // delayed-delete - shared_ptr> handles; - std::string mic; -}; - -void convert(PriceSize ps, PriceSizeMessage *pps) { - pps->set_mantissa(ps.mantissa()); - pps->set_exponent(ps.exponent()); - pps->set_size(ps.size()); -} -void set_price(PriceSize ps, co2::TradeMessage *ptrade) { - ptrade->set_mantissa(ps.mantissa()); - ptrade->set_exponent(ps.exponent()); - ptrade->set_size(ps.size()); -} - -void set_price(const IntPrice &price, co2::PriceMessage *dest) { - dest->set_mantissa(price.mantissa()); - dest->set_exponent(price.exponent()); -} - -void copy_misc(IntPrice settlement, MiscData misc, co2::MiscMessage *pmisc) { - set_price(settlement, pmisc->mutable_settlement()); - pmisc->set_settlement_date(misc.settlement_date.to_short()); - set_price( - misc.previous_settlement_price, - pmisc->mutable_previous_settlement_price()); - pmisc->set_previous_settlement_date( - misc.previous_settlement_date.to_short()); - pmisc->set_open_price(misc.open_price); - pmisc->set_open_date(misc.open_date.to_short()); - pmisc->set_close_price(misc.close_price); - pmisc->set_close_date(misc.close_date.to_short()); - pmisc->set_fixing_price(misc.fixing_price); - pmisc->set_session_high(misc.session_high); - pmisc->set_session_low(misc.session_low); - // pmisc->set_session_high_bid(misc.session_high_bid); - // pmisc->set_session_low_offer(misc.session_low_offer); -} - -// Handles market data queries and responds -class QueryServerImpl : public IQueryServer { - MDClient &_client; - MDConsumer _consumer; - int _sub; - boost::container::flat_map> _groups; - - public: - QueryServerImpl(MDClient &client, metrics::MetricsContainer &stats) - : _client(client), _consumer(client.current_consumer()), _sub(0), - _groups() { - for (const auto &sym : client.symbols()) { - _groups.emplace(std::make_pair( - sym, std::make_unique(sym, _consumer, stats))); - } - _groups.shrink_to_fit(); - - _sub = client.on_consumer().subscribe( - [=, this](const MDConsumer &c) { handle_recovery(c); }); - subscribe_securities(); - } - QueryServerImpl(const QueryServerImpl &) = delete; - ~QueryServerImpl() { _client.on_consumer().unsubscribe(_sub); } - - void subscribe_securities() { - LOG(md, Info, "QueryServer::subscribe_securities"); - auto id = _consumer.id(); - _consumer.on_security( - [this, id](const SecurityPtr &sec, MDConsumerHandle handle) { - if (_consumer.id() == id) { - auto iter = _groups.find(handle.get_usersymbol()); - bts_assert(iter != _groups.end()); - const auto &pgroup = iter->second; - bts_assert(pgroup); - LOG(md, Info, - "QueryServer::_consumer.on_security symbol:{} " - "sec:{}", - handle.get_usersymbol().symbol(), - bts::securities::descriptor(*sec)); - pgroup->add_security(handle); - } - }); - } - - // not-reentrant - void handle_recovery(const MDConsumer &consumer) { - MDConsumer temp(consumer); - if (temp.all_heartbeats_ok()) { - _consumer.swap(temp); - LOG(md, Info, - "Recovering from bad MD. Old consumer {} new consumer " - "{}", - _consumer.id(), temp.id()); - for (auto &pair : _groups) { - pair.second->set_consumer(_consumer); - } - // _handle_index.clear(); - subscribe_securities(); - utils::Dispatcher::background().post( - [temp]() { - LOG(md, Info, "QServer destroying consumer {}", temp.id()); - }, - 60000); - } - } - - // the hot part - void fill( - const SecurityChannel &chan, - MDConsumerGroup &group, - ChannelReply *reply, - const Entitlements *entitlements, - bool bypass_entitlement) const { - bts_assert(reply); - - reply->mutable_channel()->CopyFrom(chan); - reply->mutable_channel()->set_time(TimeStamp::now().to_uint64()); - - bool entitled = bypass_entitlement || chan.exchange() == api::VENUE_BTS; - if (entitlements && !entitled) - entitled = entitlements->count(group.mic); - reply->mutable_channel()->set_entitled(entitled); - auto flags = (MDRequest::Flags)chan.flags(); - - // Get a copy of handles. If recovery happens during this call - // we can safely continue using this pointer for 60 seconds - auto phandles = group.handles; - if (!phandles) - return; - - ChannelUpdate temp_update; - for (const auto &instrument : *phandles) { - bool snapshot = false; - MDConsumerHandle handle = instrument.handle; - temp_update.Clear(); - bts_assert_d(handle.is_initialized()); - if (!handle.is_initialized()) - continue; - ChannelUpdate *pupd = &temp_update; - pupd->set_key(instrument.handle.key()); - // If the security was just discovered try to ensure all - // data is sent - if (instrument.discovered > TimeStamp(chan.time())) { - auto security = handle.get_normalized_security(); - pupd->mutable_security()->CopyFrom(security); - snapshot = true; - } - // if we are not entitled AND this is the first request - // since we lost our entitlement, zero out books - if (!entitled && chan.entitled()) { - if (HASFLAG(flags, MDRequest::BBO) || - HASFLAG(flags, MDRequest::DIRECT)) { - auto *p = pupd->mutable_direct(); - for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { - convert(NO_MARKET, p->add_bids()); - convert(NO_MARKET, p->add_asks()); - } - } - if ((HASFLAG(flags, MDRequest::IMPLIED) || - HASFLAG(flags, MDRequest::BBO)) && - !handle.is_synthetic()) { - auto *p = pupd->mutable_implied(); - for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) { - convert(NO_MARKET, p->add_bids()); - convert(NO_MARKET, p->add_asks()); - } - } - } - if (entitled) { - // If we are entitled now but this is the first request - // where we are ensure we send all values - if (!chan.entitled()) - snapshot = true; - - if (HASFLAG(flags, MDRequest::BBO) || - HASFLAG(flags, MDRequest::DIRECT)) { - std::array bids, asks; - TimeStamp time(snapshot ? 0 : chan.time()); - size_t copy_depth = - HASFLAG(flags, MDRequest::DIRECT) ? COCONUT_MAX_DEPTH : 1; - auto result = handle.get_direct_book( - time, copy_depth, bids.data(), asks.data()); - if (result == HandleReadResult::SUCCESS) { - auto *p = pupd->mutable_direct(); - for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { - PriceSize b = bids[x]; - if (b != NO_MARKET) { - convert(b, p->add_bids()); - } - PriceSize a = asks[x]; - if (a != NO_MARKET) { - convert(a, p->add_asks()); - } - } - } else if (result == HandleReadResult::BUSY) { - LOG(md, Info, - "Busy read on handle {}. Retry on next " - "request", - instrument.handle.key()); - // If we get a busy reading BBO (which happens - // 1-2x per day for a single relay client) We move - // the timestamp back so the next request will try - // to read this value again. ALTERNATIVE: We could - // spin until the value was not busy but it could - // be that the writer was context switched out and - // we will have to wait multiple ms to unblock the - // request. This fix assumes it is better to send - // the update now with 1 stale value than to block - // the call and allow all the values to be stale - // until this one can be read. - reply->mutable_channel()->set_time(chan.time()); - } - } - if ((HASFLAG(flags, MDRequest::IMPLIED) || - HASFLAG(flags, MDRequest::BBO)) && - !handle.is_synthetic()) { - std::array bids, asks; - TimeStamp time(snapshot ? 0 : chan.time()); - size_t copy_depth = HASFLAG(flags, MDRequest::IMPLIED) - ? COCONUT_MAX_IMPLIED_DEPTH - : 1; - auto result = handle.get_implied_book( - time, copy_depth, bids.data(), asks.data()); - if (result == HandleReadResult::SUCCESS) { - auto *p = pupd->mutable_implied(); - for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) { - PriceSize b = bids[x]; - if (b != NO_MARKET) { - convert(b, p->add_bids()); - } - PriceSize a = asks[x]; - if (a != NO_MARKET) { - convert(a, p->add_asks()); - } - } - } else if (result == HandleReadResult::BUSY) { - LOG(md, Info, - "Busy read on handle (implied) {}. Retry on " - "next request", - instrument.handle.key()); - reply->mutable_channel()->set_time(chan.time()); - } - } - if (HASFLAG(flags, MDRequest::LAST_TRADE)) { - TimeStamp time(snapshot ? 0 : chan.time()); - PriceSize price; - datetime::LLDateTime trade_time; - size_t volume; - int from_spread; - if (handle.get_last_trade( - time, price, trade_time, &volume, &from_spread)) { - auto *trade = pupd->mutable_last_trade(); - trade->set_key(handle.key()); - trade->set_time(trade_time); - trade->set_total_volume(volume); - trade->set_from_spread(from_spread); - set_price(price, trade); - } - } - if (HASFLAG(flags, MDRequest::MISC)) { - TimeStamp time(snapshot ? 0 : chan.time()); - if (handle.misc_timestamp() > time) { - copy_misc( - handle.settlement_price(), handle.misc(), - pupd->mutable_misc()); - } - } - } - if (pupd->has_direct() || pupd->has_implied() || - pupd->has_last_trade() || pupd->has_misc() || - pupd->has_security()) { - reply->add_updates()->CopyFrom(*pupd); - } - } - - reply->mutable_channel()->set_segment_id( - group.channel.create_time().to_uint64()); - bool is_same_segment = - (chan.segment_id() == group.channel.create_time().to_uint64()); - - if (HASFLAG(flags, MDRequest::TRADE_LOG)) { - int64_t request_cursor = is_same_segment ? chan.trade_cursor() : 0; - auto cursor = group.channel.read_trades( - *reply->mutable_trade_log(), request_cursor); - if (cursor != chan.trade_cursor()) { - reply->mutable_channel()->set_trade_cursor(cursor); - } - } - if (HASFLAG(flags, MDRequest::RFQ)) { - int64_t request_cursor = is_same_segment ? chan.rfq_cursor() : 0; - auto cursor = - group.channel.read_rfqs(*reply->mutable_rfq_log(), request_cursor); - if (cursor != chan.rfq_cursor()) { - reply->mutable_channel()->set_rfq_cursor(cursor); - } - } - } - - void query( - const MDRequest &request, - MDReply *reply, - const Entitlements *entitlements, - bool bypass_entitlement) const { - bts_assert(reply != nullptr); - for (auto &ch : request.channels()) { - auto groupiter = - _groups.find(UserSymbol(ch.symbol(), ch.exchange(), ch.type())); - if (groupiter != _groups.end()) { - auto &group = groupiter->second; - fill( - ch, *group, reply->add_channels(), entitlements, - bypass_entitlement); - } - } - } - - void get_names(ExpirationNameListReply *reply) const { - bts_assert(reply != nullptr); - Security security; - for (auto &groupiter : _groups) { - const auto &group = groupiter.second; - ExpirationNameList *psec = reply->add_securities(); - psec->mutable_product()->CopyFrom(group->symbol.to_product()); - for (const auto &h : *(group->handles)) { - security.CopyFrom(h.handle.get_normalized_security()); - std::string desc; - if (!DomainSecurityUtils::get_security_desc(security, desc)) { - LOG(md, Error, - "QueryServerImpl::get_names exchange: {} not handled", - Venue_Name(security.exchange())); - continue; - } - psec->mutable_names()->insert({h.handle.key(), desc}); - } - } - } - - void get_securities( - google::protobuf::RepeatedField keys, - api::SecurityList *result) const { - bts_assert(result); - // Ensure keys are unique - std::set unique_keys(keys.begin(), keys.end()); - for (auto k : unique_keys) { - auto ptr = _consumer.get_security(SecurityKey(k)); - if (ptr) { - result->add_security()->CopyFrom(*ptr); - } - } - } -}; - -struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { - const QueryServerImpl &_db; - utils::Dispatcher &_threadpool; - std::shared_ptr> _entitlements; -#ifndef _WIN32 - std::shared_ptr> _time, _size; -#endif - - MDRelayServer( - const QueryServerImpl &db, - utils::Dispatcher &threadpool, - bts::api::ServiceContext ctx, - metrics::MetricsContainer &stats) - : bts::co2::AuthenticatedMarketDataRelayBase(ctx), _db(db), - _threadpool(threadpool), _entitlements() -#ifndef _WIN32 - , - _time(std::make_shared>( - "spark_qserver_response_time_ms", - std::array({5.0, 10.0, 20.0, 50.0, 100.0}))), - _size(std::make_shared>( - "spark_qserver_size_bytes", - std::array( - {10240 /*10k*/, 102400 /*100k*/, 1024 * 1024 /*1mb*/, - 1024 * 1024 * 10 /*10mb*/, 1024 * 1024 * 100 /*100mb*/}))) { - stats.publish(_time); - stats.publish(_size); - } -#else - { - } -#endif - - void update_entitlements( - std::shared_ptr> - &entitlements) { - _entitlements = entitlements; - } - - static const Entitlements *get_entitlements( - uint32_t user_id, - std::shared_ptr> - &entitlements) { - if (entitlements) { - auto iter = entitlements->find(user_id); - if (iter != entitlements->end()) { - return &iter->second; - } - } - return nullptr; - } - - virtual void requestmd( - const api::AuthToken &token, - const MDRequest &request, - const std::function &reply) override { - TimeStamp start = TimeStamp::now(); - _threadpool.post([=, this]() { - google::protobuf::Arena arena; - MDReply *preply = - google::protobuf::Arena::CreateMessage(&arena); - // addref to keep shared ptr alive - api::user::TokenPayload payload; - auto ents = _entitlements; - const Entitlements *user_entitlement = nullptr; - bool bypass_entitlement = false; - if (api::auth::decode_token(token, &payload)) { - bypass_entitlement = payload.wheel(); - if (!bypass_entitlement) { - user_entitlement = get_entitlements(payload.id(), ents); - } - } - // disabling md entitlements - _db.query( - request, preply, user_entitlement, true || bypass_entitlement); - reply(*preply); -#ifndef _WIN32 - auto elapsed = (TimeStamp::now() - start); - _time->observe(elapsed.milliseconds()); - _size->observe(preply->ByteSizeLong()); -#endif - }); - } - - virtual void request_products( - const api::AuthToken &, - const api::VoidMessage &, - const std::function &reply) override { - ProductListReply list; - const auto syms = SymbolLoader::load(); - for (const auto &i : syms) { - auto prod = list.add_products(); - prod->set_exchange(i.exchange()); - prod->set_type(i.type()); - prod->set_symbol(i.symbol()); - } - reply(list); - } - - virtual void request_names( - const api::AuthToken &, - const api::VoidMessage &, - const std::function &reply) - override { - _threadpool.post([=, this]() { - google::protobuf::Arena arena; - ExpirationNameListReply *preply = - google::protobuf::Arena::CreateMessage( - &arena); - _db.get_names(preply); - reply(*preply); - }); - } - - virtual void get_securities( - const api::AuthToken &, - const SecurityKeyList &keys, - const std::function &reply) override { - _threadpool.post([=, this]() { - google::protobuf::Arena arena; - api::SecurityList *preply = - google::protobuf::Arena::CreateMessage(&arena); - _db.get_securities(keys.keys(), preply); - reply(*preply); - }); - } -}; - -struct QueryServer { - QueryServer( - utils::Dispatcher &threadpool, - const bts::api::ServiceContext &ctx, - MDClient &client, - metrics::MetricsContainer &stats) - : db(client, stats), relay(db, threadpool, ctx, stats), - admin_client(ctx.locate()) {} - QueryServerImpl db; - MDRelayServer relay; - api::admin::AdminService admin_client; - rxcpp::composite_subscription entitlement_sub; -}; - -QueryServerPtr run_query_server( - const bts::api::ServiceContext &ctx, - MDClient &client, - metrics::MetricsContainer &stats) { - auto &threadpool = utils::Dispatcher::get_named("MDQuery"); - - QueryServerPtr container = - std::make_shared(threadpool, ctx, client, stats); - - // Subscribe to entitlement data - container->entitlement_sub = - container->admin_client.subscribe_md_entitlement_data().subscribe( - [=](const api::admin::Md_entitlement_dataUpdate &update) { - const auto &all_updates = update.all(); - utils::Dispatcher::background().post([all_updates, container]() { - auto entitlements = std::make_shared< - std::unordered_map>(); - for (const auto &row : all_updates) { - if (row && row->enabled()) { - auto iter = entitlements->find(row->user_id()); - if (iter == entitlements->end()) { - Entitlements ents; - ents.insert(row->mic()); - entitlements->insert( - iter, - std::make_pair(row->user_id(), std::move(ents))); - } else { - iter->second.insert(row->mic()); - } - } - } - container->relay.update_entitlements(entitlements); - }); - }); - - // Add a few threads to keep md latency low - for (int x = 1; x < std::min(4, os::num_cores()); ++x) { - threadpool.add_thread(); - } - - return container; -} - -std::unique_ptr create_query_server( - MDClient &mdc, metrics::MetricsContainer &mets) { - return std::make_unique(mdc, mets); -} - -} // namespace md -} // namespace bts -#endif diff --git a/src/lib_spark/spark/md/relay/QueryServer.h b/src/lib_spark/spark/md/relay/QueryServer.h deleted file mode 100644 index 5bb9f17e1..000000000 --- a/src/lib_spark/spark/md/relay/QueryServer.h +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once - -#include "spark/md/ISyntheticSecurityService.h" -#include "spark/md/UserSymbol.h" -#include "spark/utils/bts_export.h" -#include "spark/utils/bts_no_copy.h" -#include -#include -#include - -#include "spark/md/protos/co2.pb.h" - -namespace bts { -namespace api { -class ServiceContext; -} -namespace metrics { -class MetricsContainer; -} -namespace md { -class MDClient; -typedef boost::container::flat_set Entitlements; - -// provides a polling interface on top of market data - -struct IQueryServer { - virtual ~IQueryServer() {} - virtual void query( - const co2::MDRequest &request, - co2::MDReply *reply, - const Entitlements *, - bool) const = 0; -}; -std::unique_ptr create_query_server( - MDClient &, metrics::MetricsContainer &); - -struct QueryServer; -typedef std::shared_ptr QueryServerPtr; -QueryServerPtr run_query_server( - const bts::api::ServiceContext &, MDClient &, metrics::MetricsContainer &); - -} // namespace md -} // namespace bts diff --git a/src/lib_spark/spark/md/relay/entitlements.h b/src/lib_spark/spark/md/relay/entitlements.h new file mode 100644 index 000000000..fb7d136e3 --- /dev/null +++ b/src/lib_spark/spark/md/relay/entitlements.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +namespace bts::md { + +typedef boost::container::flat_set Entitlements; + +} // namespace bts::md diff --git a/src/lib_spark/spark/md/relay/i_query_server.cpp b/src/lib_spark/spark/md/relay/i_query_server.cpp new file mode 100644 index 000000000..dc4d717c4 --- /dev/null +++ b/src/lib_spark/spark/md/relay/i_query_server.cpp @@ -0,0 +1,7 @@ +#include "i_query_server.h" + +namespace bts::md { + +IQueryServer::~IQueryServer() = default; + +} // namespace bts::md diff --git a/src/lib_spark/spark/md/relay/i_query_server.h b/src/lib_spark/spark/md/relay/i_query_server.h new file mode 100644 index 000000000..11155634a --- /dev/null +++ b/src/lib_spark/spark/md/relay/i_query_server.h @@ -0,0 +1,18 @@ +#pragma once + +#include "spark/md/protos/co2.pb.h" +#include "spark/md/relay/entitlements.h" + +namespace bts::md { + +struct IQueryServer { + virtual ~IQueryServer(); + + virtual void query( + const co2::MDRequest &request, + co2::MDReply *reply, + const Entitlements *entitlements, + bool bypass_entitlement) const = 0; +}; + +} // namespace bts::md diff --git a/src/lib_spark/spark/md/relay/internal/dated_handle.h b/src/lib_spark/spark/md/relay/internal/dated_handle.h new file mode 100644 index 000000000..6927e2d79 --- /dev/null +++ b/src/lib_spark/spark/md/relay/internal/dated_handle.h @@ -0,0 +1,13 @@ +#pragma once + +#include "spark/md/MDConsumerHandle.h" +#include "spark/utils/TimeStamp.h" + +namespace bts::md { + +struct DatedHandle { + utils::TimeStamp discovered; + MDConsumerHandle handle; +}; + +} // namespace bts::md diff --git a/src/lib_spark/spark/md/relay/internal/md_consumer_group.cpp b/src/lib_spark/spark/md/relay/internal/md_consumer_group.cpp new file mode 100644 index 000000000..94ea163dc --- /dev/null +++ b/src/lib_spark/spark/md/relay/internal/md_consumer_group.cpp @@ -0,0 +1,141 @@ +#include "md_consumer_group.h" +#include "spark/config/protos/cme_reference_data.pb.h" + +namespace bts::md { + +// probably just have to send the consumer in from outside +MDConsumerGroup::MDConsumerGroup( + const UserSymbol &symbol, + const MDConsumer &consumer, + metrics::MetricsContainer &stats) + : symbol(symbol), channel(consumer.get_channel(symbol)), handles(), mic() { + set_consumer(consumer); + + // create and publish a stat for this group +#ifndef _WIN32 + if (symbol.type() != SecurityType::SECURITYTYPE_SYNTHETIC) { + std::vector labels; + labels.push_back({"symbol", symbol.symbol()}); + labels.push_back( + {"type", + SecurityType_Name(symbol.type()).substr(13) /*SECURITYTYPE_*/}); + labels.push_back( + {"exch", Venue_Name(symbol.exchange()).substr(6) /*VENUE_*/}); + stats.publish(std::make_shared( + "spark_md_book_updates", + [=, this]() { + return channel.direct_count() + channel.implied_count(); + }, + labels)); + stats.publish(std::make_shared( + "spark_md_security_count", + [=, this]() { return channel.security_count(); }, labels)); + stats.publish(std::make_shared( + "spark_md_trade_count", [=, this]() { return channel.trade_count(); }, + labels)); + stats.publish(std::make_shared( + "spark_md_gap_count", + [=, this]() { + auto g = channel.gap_count(); + return g == 0 ? os::nan() : g; + }, + labels)); + } +#endif +} + +// recovery has occured and we need to clear our our collection of handles +// and wait for new ones +void MDConsumerGroup::set_consumer(const MDConsumer &consumer) { + // create a new set of handles and swap with the old. + // If the new consumer is bad we still have to swap our handles because + // the consumer that owns our handles will be scheduled for deletion + auto temphandles = + std::make_shared>(); + handles.swap(temphandles); + channel = consumer.get_channel(symbol); +} + +void MDConsumerGroup::add_security(MDConsumerHandle &handle) { + bts_assert(handles); + bts_assert_d(handle.is_initialized()); + bts_assert(handle.get_usersymbol() == symbol); + DatedHandle h = {TimeStamp::now(), handle}; + handles->push_back(h); + if (this->mic == "") + set_mic(h); +} + +void MDConsumerGroup::set_mic(const DatedHandle &h) { + switch (channel.exchange()) { + case api::VENUE_ICE: + case api::VENUE_SIMICE: + this->mic = h.handle.get_normalized_security().ice().mic_code(); + break; + case api::VENUE_GLBX: + case api::VENUE_SIM: { + const auto &cme_symb_mics = + config::get().cme_mic(); + for (auto mic = cme_symb_mics.begin(); mic != cme_symb_mics.end(); + mic++) { + const auto &micsym = mic->first; + if (micsym == symbol.symbol()) { + this->mic = api::CMEMics_Name(mic->second); + break; + } + } + break; + } + case api::VENUE_BTS: + case api::VENUE_UNSET: + this->mic = "BTS"; + break; + case api::VENUE_DERIBIT: + this->mic = "DERIBIT"; + break; + case api::VENUE_FTX: + this->mic = "FTX"; + break; + case api::VENUE_BINANCE: + this->mic = "BINANCE"; + break; + case api::VENUE_BINANCE_COIN_FUTURES: + this->mic = "BINANCE_COIN_FUTURES"; + break; + case api::VENUE_BINANCE_US: + this->mic = "BINANCE_US"; + break; + case api::VENUE_COINBASE: + this->mic = "COINBASE"; + break; + case api::VENUE_BINANCE_USDS_FUTURES: + this->mic = "BINANCE_USDS_FUTURES"; + break; + case api::VENUE_OKX: + this->mic = "OKX"; + break; + case api::VENUE_CBOE: + this->mic = "CBOE"; + break; + case api::VENUE_TWSE: + this->mic = "TWSE"; + break; + case api::VENUE_TAIFEX: + this->mic = "TAIFEX"; + break; + case api::VENUE_BYBIT: + this->mic = "BYBIT"; + break; + case api::VENUE_GATEIO: + this->mic = "GATEIO"; + break; + case api::VENUE_CBOECFE: + case api::Venue_INT_MIN_SENTINEL_DO_NOT_USE_: + case api::Venue_INT_MAX_SENTINEL_DO_NOT_USE_: + bts_assert(false); + this->mic = "UNKNOWN"; + break; + } +} + +} // namespace bts::md diff --git a/src/lib_spark/spark/md/relay/internal/md_consumer_group.h b/src/lib_spark/spark/md/relay/internal/md_consumer_group.h new file mode 100644 index 000000000..4fe25ae46 --- /dev/null +++ b/src/lib_spark/spark/md/relay/internal/md_consumer_group.h @@ -0,0 +1,37 @@ +#pragma once + +#include "spark/md/MDConsumer.h" +#include "spark/md/UserSymbol.h" +#include "spark/md/relay/internal/dated_handle.h" +#include "spark/metrics/metrics.h" +#include "spark/utils/ConcurrentAppendCollection.h" + +namespace bts::md { + +struct MDConsumerGroup { + // probably just have to send the consumer in from outside + MDConsumerGroup( + const UserSymbol &symbol, + const MDConsumer &consumer, + metrics::MetricsContainer &stats); + MDConsumerGroup(const MDConsumerGroup &other) = delete; + + // recovery has occured and we need to clear our our collection of handles + // and wait for new ones + void set_consumer(const MDConsumer &consumer); + + void add_security(MDConsumerHandle &handle); + + void set_mic(const DatedHandle &h); + + UserSymbol symbol; + MDConsumerChannel channel; + + // Can add to handles without invalidating readers (no lock needed) + // When we recover we swap a new vector into this unique_ptr and + // delayed-delete + std::shared_ptr> handles; + std::string mic; +}; + +} // namespace bts::md diff --git a/src/lib_spark/spark/md/relay/internal/md_relay_server.cpp b/src/lib_spark/spark/md/relay/internal/md_relay_server.cpp new file mode 100644 index 000000000..36fc5ad7f --- /dev/null +++ b/src/lib_spark/spark/md/relay/internal/md_relay_server.cpp @@ -0,0 +1,130 @@ +#include "md_relay_server.h" +#include "spark/api/public/ServiceContext.h" +#include "spark/api/services/AuthService.pb.h" +#include "spark/md/SymbolLoader.h" +#include "spark/utils/AuthToken.h" + +namespace bts::md { + +using bts::api::auth::decode_token; +using bts::co2::ExpirationNameListReply; +using bts::co2::MDReply; +using bts::co2::MDRequest; +using bts::co2::ProductListReply; +using bts::co2::SecurityKeyList; + +MDRelayServer::MDRelayServer( + const QueryServerImpl &db, + utils::Dispatcher &threadpool, + bts::api::ServiceContext ctx, + metrics::MetricsContainer &stats) + : bts::co2::AuthenticatedMarketDataRelayBase(ctx), _db(db), + _threadpool(threadpool), _entitlements() +#ifndef _WIN32 + , + _time(std::make_shared>( + "spark_qserver_response_time_ms", + std::array({5.0, 10.0, 20.0, 50.0, 100.0}))), + _size(std::make_shared>( + "spark_qserver_size_bytes", + std::array( + {10240 /*10k*/, 102400 /*100k*/, 1024 * 1024 /*1mb*/, + 1024 * 1024 * 10 /*10mb*/, 1024 * 1024 * 100 /*100mb*/}))) { + stats.publish(_time); + stats.publish(_size); +} +#else +{ +} +#endif + +void MDRelayServer::update_entitlements( + std::shared_ptr> &entitlements) { + _entitlements = entitlements; +} + +const Entitlements *MDRelayServer::get_entitlements( + uint32_t user_id, + std::shared_ptr> &entitlements) { + if (entitlements) { + auto iter = entitlements->find(user_id); + if (iter != entitlements->end()) { + return &iter->second; + } + } + return nullptr; +} + +void MDRelayServer::requestmd( + const api::AuthToken &token, + const MDRequest &request, + const std::function &reply) { + TimeStamp start = TimeStamp::now(); + _threadpool.post([=, this]() { + google::protobuf::Arena arena; + MDReply *preply = google::protobuf::Arena::CreateMessage(&arena); + // addref to keep shared ptr alive + api::user::TokenPayload payload; + auto ents = _entitlements; + const Entitlements *user_entitlement = nullptr; + bool bypass_entitlement = false; + if (decode_token(token, &payload)) { + bypass_entitlement = payload.wheel(); + if (!bypass_entitlement) { + user_entitlement = get_entitlements(payload.id(), ents); + } + } + // disabling md entitlements + _db.query(request, preply, user_entitlement, true || bypass_entitlement); + reply(*preply); +#ifndef _WIN32 + auto elapsed = (TimeStamp::now() - start); + _time->observe(elapsed.milliseconds()); + _size->observe(preply->ByteSizeLong()); +#endif + }); +} + +void MDRelayServer::request_products( + const api::AuthToken &, + const api::VoidMessage &, + const std::function &reply) { + ProductListReply list; + const auto syms = SymbolLoader::load(); + for (const auto &i : syms) { + auto prod = list.add_products(); + prod->set_exchange(i.exchange()); + prod->set_type(i.type()); + prod->set_symbol(i.symbol()); + } + reply(list); +} + +void MDRelayServer::request_names( + const api::AuthToken &, + const api::VoidMessage &, + const std::function &reply) { + _threadpool.post([=, this]() { + google::protobuf::Arena arena; + ExpirationNameListReply *preply = + google::protobuf::Arena::CreateMessage( + &arena); + _db.get_names(preply); + reply(*preply); + }); +} + +void MDRelayServer::get_securities( + const api::AuthToken &, + const SecurityKeyList &keys, + const std::function &reply) { + _threadpool.post([=, this]() { + google::protobuf::Arena arena; + api::SecurityList *preply = + google::protobuf::Arena::CreateMessage(&arena); + _db.get_securities(keys.keys(), preply); + reply(*preply); + }); +} + +} // namespace bts::md diff --git a/src/lib_spark/spark/md/relay/internal/md_relay_server.h b/src/lib_spark/spark/md/relay/internal/md_relay_server.h new file mode 100644 index 000000000..5402908c9 --- /dev/null +++ b/src/lib_spark/spark/md/relay/internal/md_relay_server.h @@ -0,0 +1,54 @@ +#pragma once + +#include "spark/api/services/MarketDataRelay.server.pb.h" +#include "spark/md/relay/internal/query_server_impl.h" + +namespace bts::md { + +struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { + const QueryServerImpl &_db; + utils::Dispatcher &_threadpool; + std::shared_ptr> _entitlements; +#ifndef _WIN32 + std::shared_ptr> _time, _size; +#endif + + MDRelayServer( + const QueryServerImpl &db, + utils::Dispatcher &threadpool, + bts::api::ServiceContext ctx, + metrics::MetricsContainer &stats); + + void update_entitlements( + std::shared_ptr> + &entitlements); + + static const Entitlements *get_entitlements( + uint32_t user_id, + std::shared_ptr> + &entitlements); + + virtual void requestmd( + const api::AuthToken &token, + const co2::MDRequest &request, + const std::function &reply) override; + + virtual void request_products( + const api::AuthToken &, + const api::VoidMessage &, + const std::function &reply) + override; + + virtual void request_names( + const api::AuthToken &, + const api::VoidMessage &, + const std::function &reply) + override; + + virtual void get_securities( + const api::AuthToken &, + const co2::SecurityKeyList &keys, + const std::function &reply) override; +}; + +} // namespace bts::md diff --git a/src/lib_spark/spark/md/relay/internal/query_server_impl.cpp b/src/lib_spark/spark/md/relay/internal/query_server_impl.cpp new file mode 100644 index 000000000..0866cedda --- /dev/null +++ b/src/lib_spark/spark/md/relay/internal/query_server_impl.cpp @@ -0,0 +1,360 @@ +#include "query_server_impl.h" +#include "spark/domain/DomainSecurityUtils.h" +#include "spark/domain_model/Security.h" +#include "spark/md/relay/internal/md_consumer_group.h" +#include "spark/utils/helper_macros.h" + +namespace bts::md { + +using bts::co2::ChannelReply; +using bts::co2::ChannelUpdate; +using bts::co2::ExpirationNameList; +using bts::co2::ExpirationNameListReply; +using bts::co2::MDReply; +using bts::co2::MDRequest; +using bts::co2::PriceSizeMessage; +using bts::co2::SecurityChannel; +using bts::domain::DomainSecurityUtils; + +static void convert(PriceSize ps, PriceSizeMessage *pps) { + pps->set_mantissa(ps.mantissa()); + pps->set_exponent(ps.exponent()); + pps->set_size(ps.size()); +} + +static void set_price(PriceSize ps, co2::TradeMessage *ptrade) { + ptrade->set_mantissa(ps.mantissa()); + ptrade->set_exponent(ps.exponent()); + ptrade->set_size(ps.size()); +} + +static void set_price(const IntPrice &price, co2::PriceMessage *dest) { + dest->set_mantissa(price.mantissa()); + dest->set_exponent(price.exponent()); +} + +static void copy_misc( + IntPrice settlement, MiscData misc, co2::MiscMessage *pmisc) { + set_price(settlement, pmisc->mutable_settlement()); + pmisc->set_settlement_date(misc.settlement_date.to_short()); + set_price( + misc.previous_settlement_price, + pmisc->mutable_previous_settlement_price()); + pmisc->set_previous_settlement_date( + misc.previous_settlement_date.to_short()); + pmisc->set_open_price(misc.open_price); + pmisc->set_open_date(misc.open_date.to_short()); + pmisc->set_close_price(misc.close_price); + pmisc->set_close_date(misc.close_date.to_short()); + pmisc->set_fixing_price(misc.fixing_price); + pmisc->set_session_high(misc.session_high); + pmisc->set_session_low(misc.session_low); + // pmisc->set_session_high_bid(misc.session_high_bid); + // pmisc->set_session_low_offer(misc.session_low_offer); +} + +// Handles market data queries and responds +QueryServerImpl::QueryServerImpl( + MDClient &client, metrics::MetricsContainer &stats) + : _client(client), _consumer(client.current_consumer()), _sub(0), + _groups() { + for (const auto &sym : client.symbols()) { + _groups.emplace(std::make_pair( + sym, std::make_unique(sym, _consumer, stats))); + } + _groups.shrink_to_fit(); + + _sub = client.on_consumer().subscribe( + [=, this](const MDConsumer &c) { handle_recovery(c); }); + subscribe_securities(); +} + +QueryServerImpl::~QueryServerImpl() { _client.on_consumer().unsubscribe(_sub); } + +void QueryServerImpl::subscribe_securities() { + LOG(md, Info, "QueryServer::subscribe_securities"); + auto id = _consumer.id(); + _consumer.on_security( + [this, id](const SecurityPtr &sec, MDConsumerHandle handle) { + if (_consumer.id() == id) { + auto iter = _groups.find(handle.get_usersymbol()); + bts_assert(iter != _groups.end()); + const auto &pgroup = iter->second; + bts_assert(pgroup); + LOG(md, Info, + "QueryServer::_consumer.on_security symbol:{} " + "sec:{}", + handle.get_usersymbol().symbol(), + bts::securities::descriptor(*sec)); + pgroup->add_security(handle); + } + }); +} + +// not-reentrant +void QueryServerImpl::handle_recovery(const MDConsumer &consumer) { + MDConsumer temp(consumer); + if (temp.all_heartbeats_ok()) { + _consumer.swap(temp); + LOG(md, Info, + "Recovering from bad MD. Old consumer {} new consumer " + "{}", + _consumer.id(), temp.id()); + for (auto &pair : _groups) { + pair.second->set_consumer(_consumer); + } + // _handle_index.clear(); + subscribe_securities(); + utils::Dispatcher::background().post( + [temp]() { + LOG(md, Info, "QServer destroying consumer {}", temp.id()); + }, + 60000); + } +} + +// the hot part +void QueryServerImpl::fill( + const SecurityChannel &chan, + MDConsumerGroup &group, + ChannelReply *reply, + const Entitlements *entitlements, + bool bypass_entitlement) const { + bts_assert(reply); + + reply->mutable_channel()->CopyFrom(chan); + reply->mutable_channel()->set_time(TimeStamp::now().to_uint64()); + + bool entitled = bypass_entitlement || chan.exchange() == api::VENUE_BTS; + if (entitlements && !entitled) + entitled = entitlements->count(group.mic); + reply->mutable_channel()->set_entitled(entitled); + auto flags = (MDRequest::Flags)chan.flags(); + + // Get a copy of handles. If recovery happens during this call + // we can safely continue using this pointer for 60 seconds + auto phandles = group.handles; + if (!phandles) + return; + + ChannelUpdate temp_update; + for (const auto &instrument : *phandles) { + bool snapshot = false; + MDConsumerHandle handle = instrument.handle; + temp_update.Clear(); + bts_assert_d(handle.is_initialized()); + if (!handle.is_initialized()) + continue; + ChannelUpdate *pupd = &temp_update; + pupd->set_key(instrument.handle.key()); + // If the security was just discovered try to ensure all + // data is sent + if (instrument.discovered > TimeStamp(chan.time())) { + auto security = handle.get_normalized_security(); + pupd->mutable_security()->CopyFrom(security); + snapshot = true; + } + // if we are not entitled AND this is the first request + // since we lost our entitlement, zero out books + if (!entitled && chan.entitled()) { + if (HASFLAG(flags, MDRequest::BBO) || + HASFLAG(flags, MDRequest::DIRECT)) { + auto *p = pupd->mutable_direct(); + for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + convert(NO_MARKET, p->add_bids()); + convert(NO_MARKET, p->add_asks()); + } + } + if ((HASFLAG(flags, MDRequest::IMPLIED) || + HASFLAG(flags, MDRequest::BBO)) && + !handle.is_synthetic()) { + auto *p = pupd->mutable_implied(); + for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) { + convert(NO_MARKET, p->add_bids()); + convert(NO_MARKET, p->add_asks()); + } + } + } + if (entitled) { + // If we are entitled now but this is the first request + // where we are ensure we send all values + if (!chan.entitled()) + snapshot = true; + + if (HASFLAG(flags, MDRequest::BBO) || + HASFLAG(flags, MDRequest::DIRECT)) { + std::array bids, asks; + TimeStamp time(snapshot ? 0 : chan.time()); + size_t copy_depth = + HASFLAG(flags, MDRequest::DIRECT) ? COCONUT_MAX_DEPTH : 1; + auto result = handle.get_direct_book( + time, copy_depth, bids.data(), asks.data()); + if (result == HandleReadResult::SUCCESS) { + auto *p = pupd->mutable_direct(); + for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + PriceSize b = bids[x]; + if (b != NO_MARKET) { + convert(b, p->add_bids()); + } + PriceSize a = asks[x]; + if (a != NO_MARKET) { + convert(a, p->add_asks()); + } + } + } else if (result == HandleReadResult::BUSY) { + LOG(md, Info, + "Busy read on handle {}. Retry on next " + "request", + instrument.handle.key()); + // If we get a busy reading BBO (which happens + // 1-2x per day for a single relay client) We move + // the timestamp back so the next request will try + // to read this value again. ALTERNATIVE: We could + // spin until the value was not busy but it could + // be that the writer was context switched out and + // we will have to wait multiple ms to unblock the + // request. This fix assumes it is better to send + // the update now with 1 stale value than to block + // the call and allow all the values to be stale + // until this one can be read. + reply->mutable_channel()->set_time(chan.time()); + } + } + if ((HASFLAG(flags, MDRequest::IMPLIED) || + HASFLAG(flags, MDRequest::BBO)) && + !handle.is_synthetic()) { + std::array bids, asks; + TimeStamp time(snapshot ? 0 : chan.time()); + size_t copy_depth = HASFLAG(flags, MDRequest::IMPLIED) + ? COCONUT_MAX_IMPLIED_DEPTH + : 1; + auto result = handle.get_implied_book( + time, copy_depth, bids.data(), asks.data()); + if (result == HandleReadResult::SUCCESS) { + auto *p = pupd->mutable_implied(); + for (size_t x = 0; x < COCONUT_MAX_IMPLIED_DEPTH; ++x) { + PriceSize b = bids[x]; + if (b != NO_MARKET) { + convert(b, p->add_bids()); + } + PriceSize a = asks[x]; + if (a != NO_MARKET) { + convert(a, p->add_asks()); + } + } + } else if (result == HandleReadResult::BUSY) { + LOG(md, Info, + "Busy read on handle (implied) {}. Retry on " + "next request", + instrument.handle.key()); + reply->mutable_channel()->set_time(chan.time()); + } + } + if (HASFLAG(flags, MDRequest::LAST_TRADE)) { + TimeStamp time(snapshot ? 0 : chan.time()); + PriceSize price; + datetime::LLDateTime trade_time; + size_t volume; + int from_spread; + if (handle.get_last_trade( + time, price, trade_time, &volume, &from_spread)) { + auto *trade = pupd->mutable_last_trade(); + trade->set_key(handle.key()); + trade->set_time(trade_time); + trade->set_total_volume(volume); + trade->set_from_spread(from_spread); + set_price(price, trade); + } + } + if (HASFLAG(flags, MDRequest::MISC)) { + TimeStamp time(snapshot ? 0 : chan.time()); + if (handle.misc_timestamp() > time) { + copy_misc( + handle.settlement_price(), handle.misc(), + pupd->mutable_misc()); + } + } + } + if (pupd->has_direct() || pupd->has_implied() || pupd->has_last_trade() || + pupd->has_misc() || pupd->has_security()) { + reply->add_updates()->CopyFrom(*pupd); + } + } + + reply->mutable_channel()->set_segment_id( + group.channel.create_time().to_uint64()); + bool is_same_segment = + (chan.segment_id() == group.channel.create_time().to_uint64()); + + if (HASFLAG(flags, MDRequest::TRADE_LOG)) { + int64_t request_cursor = is_same_segment ? chan.trade_cursor() : 0; + auto cursor = group.channel.read_trades( + *reply->mutable_trade_log(), request_cursor); + if (cursor != chan.trade_cursor()) { + reply->mutable_channel()->set_trade_cursor(cursor); + } + } + if (HASFLAG(flags, MDRequest::RFQ)) { + int64_t request_cursor = is_same_segment ? chan.rfq_cursor() : 0; + auto cursor = + group.channel.read_rfqs(*reply->mutable_rfq_log(), request_cursor); + if (cursor != chan.rfq_cursor()) { + reply->mutable_channel()->set_rfq_cursor(cursor); + } + } +} + +void QueryServerImpl::query( + const MDRequest &request, + MDReply *reply, + const Entitlements *entitlements, + bool bypass_entitlement) const { + bts_assert(reply != nullptr); + for (auto &ch : request.channels()) { + auto groupiter = + _groups.find(UserSymbol(ch.symbol(), ch.exchange(), ch.type())); + if (groupiter != _groups.end()) { + auto &group = groupiter->second; + fill( + ch, *group, reply->add_channels(), entitlements, + bypass_entitlement); + } + } +} + +void QueryServerImpl::get_names(ExpirationNameListReply *reply) const { + bts_assert(reply != nullptr); + Security security; + for (auto &groupiter : _groups) { + const auto &group = groupiter.second; + ExpirationNameList *psec = reply->add_securities(); + psec->mutable_product()->CopyFrom(group->symbol.to_product()); + for (const auto &h : *(group->handles)) { + security.CopyFrom(h.handle.get_normalized_security()); + std::string desc; + if (!DomainSecurityUtils::get_security_desc(security, desc)) { + LOG(md, Error, + "QueryServerImpl::get_names exchange: {} not handled", + Venue_Name(security.exchange())); + continue; + } + psec->mutable_names()->insert({h.handle.key(), desc}); + } + } +} + +void QueryServerImpl::get_securities( + google::protobuf::RepeatedField keys, + api::SecurityList *result) const { + bts_assert(result); + // Ensure keys are unique + std::set unique_keys(keys.begin(), keys.end()); + for (auto k : unique_keys) { + auto ptr = _consumer.get_security(SecurityKey(k)); + if (ptr) { + result->add_security()->CopyFrom(*ptr); + } + } +} + +} // namespace bts::md diff --git a/src/lib_spark/spark/md/relay/internal/query_server_impl.h b/src/lib_spark/spark/md/relay/internal/query_server_impl.h new file mode 100644 index 000000000..f42177ac9 --- /dev/null +++ b/src/lib_spark/spark/md/relay/internal/query_server_impl.h @@ -0,0 +1,52 @@ +#pragma once + +#include "spark/api/services/MarketDataRelay.pb.h" +#include "spark/md/MDClient.h" +#include "spark/md/relay/i_query_server.h" +#include "spark/md/relay/internal/md_consumer_group.h" +#include "spark/metrics/metrics.h" + +#include + +namespace bts::md { + +class QueryServerImpl : public IQueryServer { + MDClient &_client; + MDConsumer _consumer; + int _sub; + boost::container::flat_map> + _groups; + + public: + QueryServerImpl(MDClient &client, metrics::MetricsContainer &stats); + + QueryServerImpl(const QueryServerImpl &) = delete; + ~QueryServerImpl(); + + void subscribe_securities(); + + // not-reentrant + void handle_recovery(const MDConsumer &consumer); + + // the hot part + void fill( + const co2::SecurityChannel &chan, + MDConsumerGroup &group, + co2::ChannelReply *reply, + const Entitlements *entitlements, + bool bypass_entitlement) const; + + void query( + const co2::MDRequest &request, + co2::MDReply *reply, + const Entitlements *entitlements, + bool bypass_entitlement) const; + + void get_names(co2::ExpirationNameListReply *reply) const; + + void get_securities( + google::protobuf::RepeatedField keys, + api::SecurityList *result) const; +}; + +} // namespace bts::md diff --git a/src/lib_spark/spark/md/relay/query_server.cpp b/src/lib_spark/spark/md/relay/query_server.cpp new file mode 100644 index 000000000..1c5629642 --- /dev/null +++ b/src/lib_spark/spark/md/relay/query_server.cpp @@ -0,0 +1,114 @@ +#ifndef _WIN32 +#include "query_server.h" +#include "spark/utils/Dispatcher.h" + +#include "../MDClient.h" + +#include "spark/api/public/ServiceContext.h" +#include "spark/api/services/AdminService.client.pb.h" +#include "spark/api/services/MarketDataRelay.server.pb.h" +#include "spark/md/relay/internal/md_relay_server.h" +#ifndef _WIN32 +#include "spark/metrics/metrics.h" +#endif + +#include +#include + +#include +#include + +using namespace bts::co2; + +using std::string; +using std::unique_ptr; + +namespace bts { +namespace md { + +/* + * NORMAL OPERATION: + * Client sends time=0 to server + * Server sends complete snapshot back (time=502) + * client sends time=502 to server + * Server sends any changes that have happened after time=502 and any securities + * discovered since time=502 + * + * ERROR RECOVERY: + * Client sends time=0 to server + * Server sends complete snapshot back (time=502) + * Server detects heartbeat failure and recovers. All timestamps > 502 + * Client sends time=502 to server + * Server sends complete snapshot back (time=605) + * - Since the server shared memory has been recreated the client will + * automatically overwrite all stale data. + * - If a security has been deleted the client will leave it frozen (OK) + * + * We scan for bad heartbeats on a different time + * If a bad heartbeat is detected we try to construct a new Consumer + * When a good consumer is created we std::move the old consumer and handles to + * delayed queue for deletion This way we don't have to lock anything during the + * recovery process because all pointers will remain valid for the duration of + * existing calls. + * + */ + +QueryServer::QueryServer( + utils::Dispatcher &threadpool, + const bts::api::ServiceContext &ctx, + MDClient &client, + metrics::MetricsContainer &stats) + : db(client, stats), relay(db, threadpool, ctx, stats), + admin_client(ctx.locate()) {} + +QueryServerPtr run_query_server( + const bts::api::ServiceContext &ctx, + MDClient &client, + metrics::MetricsContainer &stats) { + auto &threadpool = utils::Dispatcher::get_named("MDQuery"); + + QueryServerPtr container = + std::make_shared(threadpool, ctx, client, stats); + + // Subscribe to entitlement data + container->entitlement_sub = + container->admin_client.subscribe_md_entitlement_data().subscribe( + [=](const api::admin::Md_entitlement_dataUpdate &update) { + const auto &all_updates = update.all(); + utils::Dispatcher::background().post([all_updates, container]() { + auto entitlements = std::make_shared< + std::unordered_map>(); + for (const auto &row : all_updates) { + if (row && row->enabled()) { + auto iter = entitlements->find(row->user_id()); + if (iter == entitlements->end()) { + Entitlements ents; + ents.insert(row->mic()); + entitlements->insert( + iter, + std::make_pair(row->user_id(), std::move(ents))); + } else { + iter->second.insert(row->mic()); + } + } + } + container->relay.update_entitlements(entitlements); + }); + }); + + // Add a few threads to keep md latency low + for (int x = 1; x < std::min(4, os::num_cores()); ++x) { + threadpool.add_thread(); + } + + return container; +} + +std::unique_ptr create_query_server( + MDClient &mdc, metrics::MetricsContainer &mets) { + return std::make_unique(mdc, mets); +} + +} // namespace md +} // namespace bts +#endif diff --git a/src/lib_spark/spark/md/relay/query_server.h b/src/lib_spark/spark/md/relay/query_server.h new file mode 100644 index 000000000..a548fc069 --- /dev/null +++ b/src/lib_spark/spark/md/relay/query_server.h @@ -0,0 +1,32 @@ +#pragma once + +#include "spark/api/services/AdminService.client.pb.h" +#include "spark/md/relay/internal/md_relay_server.h" +#include "spark/md/relay/internal/query_server_impl.h" +#include "spark/metrics/metrics.h" + +namespace bts::md { + +struct QueryServer { + QueryServer( + utils::Dispatcher &threadpool, + const api::ServiceContext &ctx, + MDClient &client, + metrics::MetricsContainer &stats); + + QueryServerImpl db; + MDRelayServer relay; + api::admin::AdminService admin_client; + rxcpp::composite_subscription entitlement_sub; +}; +typedef std::shared_ptr QueryServerPtr; + +class MDClient; + +std::unique_ptr create_query_server( + MDClient &, metrics::MetricsContainer &); + +QueryServerPtr run_query_server( + const bts::api::ServiceContext &, MDClient &, metrics::MetricsContainer &); + +} // namespace bts::md diff --git a/src/lib_spark/spark/md/test/md2_tests.cpp b/src/lib_spark/spark/md/test/md2_tests.cpp index 620f6d6f5..0be169ca3 100644 --- a/src/lib_spark/spark/md/test/md2_tests.cpp +++ b/src/lib_spark/spark/md/test/md2_tests.cpp @@ -3,9 +3,9 @@ #include "../SecurityCache.h" #include "../SettlementRecorder.h" #include "../coconut2.h" -#include "../relay/QueryServer.h" #include "Co2TestInstrument.h" #include "spark/config/config.h" +#include "spark/md/relay/query_server.h" #include "spark/metrics/metrics.h" #include "spark/os/os.h" #include "spark/persistence/SqliteStore.h"