#ifndef NO_CPP0X #include "QueryServer.h" #include "lib/config/config.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/TimeStamp.h" #include "lib/utils/helper_macros.h" #include "../MDConsumer.h" #include "../PlatformLoader.h" #include "lib/logging/logging.h" #include "lib/config/protos/system.pb.h" #include "lib/md/protos/co2.pb.h" #include "lib/api/services/MarketDataRelay.server.pb.h" #include "lib/api/public/ServiceContext.h" #include #include #include #include #include #include #include using namespace bts::md2; using namespace bts::co2; using std::unique_ptr; using std::shared_ptr; namespace bts { namespace md { /* * NORMAL OPERATION: * Client sends time=0 to server * Server sends complete snapshot back (time=502) * client sends time=502 to server * Server sends any changes that have happened after time=502 and any securities discovered since time=502 * * ERROR RECOVERY: * Client sends time=0 to server * Server sends complete snapshot back (time=502) * Server detects heartbeat failure and recovers. All timestamps > 502 * Client sends time=502 to server * Server sends complete snapshot back (time=605) * - Since the server shared memory has been recreated the client will automatically overwrite all stale data. * - If a security has been deleted the client will leave it frozen (OK) * * We scan for bad heartbeats on a different time * If a bad heartbeat is detected we try to construct a new Consumer * When a good consumer is created we std::move the old consumer and handles to delayed queue for deletion * This way we don't have to lock anything during the recovery process because all pointers will remain valid * for the duration of existing calls. * */ struct DatedHandle { TimeStamp discovered; MDConsumerHandle handle; }; struct MDConsumerGroup { MDConsumerGroup(const UserSymbol& symbol) : symbol(symbol), platform(platform_for_exchange(symbol.exchange())), consumer(symbol), channel(consumer.get_channel(symbol)), handles(new tbb::concurrent_vector()) { initialize_handles(); } // non-reentrant void try_recover(utils::Dispatcher& d) { // if hearbeat is ok just return if (consumer.all_heartbeats_ok()) return; // otherwise construct a new MDConsumer MDConsumer temp(symbol); if (temp.all_heartbeats_ok()) { consumer.swap(temp); auto temphandles = std::make_unique>(); handles.swap(temphandles); initialize_handles(); // Switch to shared pointer for lambda. shared_ptr> oldhandles(std::move(temphandles)); d.post([oldhandles,temp]() mutable { // After some time has passed all outstanding requests are guarunteed to be complete // Explicit reset of oldhandles ensures they are deleted before the shared memory is unmapped // (lambda capture object data members order is not specified) oldhandles.reset(); }, 60000); } } UserSymbol symbol; Platform::type platform; MDConsumer consumer; MDConsumerChannel channel; // Can add to handles without invalidating readers (no lock needed) // When we recover we swap a new vector into this unique_ptr and delayed-delete unique_ptr> handles; private: BTS_NO_COPY(MDConsumerGroup); void add_security(MDConsumerHandle handle) { bts_assert(handles); bts_assert_d(handle.is_initialized()); DatedHandle h = { TimeStamp::now(), handle }; handles->push_back(h); } // We can't copy or move this because initialize_handles implicitly captures 'this' void initialize_handles() { bts_assert(handles->empty()); auto id = consumer.id(); channel = consumer.get_channel(symbol.symbol(), symbol.type(), symbol.exchange()); consumer.on_security( [this,id](const Security&, MDConsumerHandle handle) { if (id == consumer.id()) { add_security(handle); } }, false); } }; // Wrap the flat_map to ensure immutability class Map { // Can't use unique_ptr because flat_map requires assignment. typedef boost::container::flat_map> InternalMap; InternalMap _map; public: Map(InternalMap map) : _map(move(map)) { } Map(Map&& other) : _map(move(other._map)) {} InternalMap::iterator begin() { return _map.begin(); } InternalMap::iterator end() { return _map.end(); } InternalMap::const_iterator begin() const { return _map.begin(); } InternalMap::const_iterator end() const { return _map.end(); } InternalMap::const_iterator find(const UserSymbol& symbol) const { return _map.find(symbol); } }; Map form_groups(const std::set& symbols) { boost::container::flat_map> groups; for (const auto& sym : symbols) { groups.emplace(std::make_pair(sym, std::make_unique(sym))); } groups.shrink_to_fit(); return Map(groups); } void convert(PriceSize ps, PriceSizeMessage* pps) { pps->set_mantissa(ps.mantissa()); pps->set_exponent(ps.exponent()); pps->set_size(ps.size()); } void set_price(PriceSize ps, co2::TradeMessage* ptrade) { ptrade->set_mantissa(ps.mantissa()); ptrade->set_exponent(ps.exponent()); ptrade->set_size(ps.size()); } void copy_misc(MiscData misc, co2::MiscMessage* pmisc) { pmisc->set_settlement_price(misc.settlement_price); pmisc->set_settlement_date(misc.settlement_date.to_short()); pmisc->set_previous_settlement_price(misc.previous_settlement_price); pmisc->set_previous_settlement_date(misc.previous_settlement_date.to_short()); pmisc->set_open_price(misc.open_price); pmisc->set_open_date(misc.open_date.to_short()); pmisc->set_close_price(misc.close_price); pmisc->set_close_date(misc.close_date.to_short()); //pmisc->set_fixing_price(misc.fixing_price); pmisc->set_session_high(misc.session_high); pmisc->set_session_low(misc.session_low); //pmisc->set_open_interest(misc.open_interest); //pmisc->set_session_high_bid(misc.session_high_bid); //pmisc->set_session_low_offer(misc.session_low_offer); } // Handles market data queries and responds struct QueryServerImpl : public IQueryServer { QueryServerImpl(const std::set& symbols) : groups(form_groups(symbols)) { } // Scan for new securities and get their handles+times // not-reentrant void scan_securities() const { for (auto& group : groups) { if (group.second->consumer.all_heartbeats_ok()) { TimeStamp start = TimeStamp::now(); size_t count = group.second->consumer.scan_securities(false); auto delta = (TimeStamp::now() - start); if (delta.milliseconds() > 10) { LOG(MarketData,Diag,"Scanned handles: {}ms found: {} securities. " , delta.milliseconds() ,count ,group.second->symbol ); } } } } // not-reentrant void check_heartbeats(utils::Dispatcher& dispatcher) { for (auto& group : groups) { if (!group.second->consumer.all_heartbeats_ok()) { group.second->try_recover(dispatcher); } } } // the hot part void fill(const SecurityChannel& chan, MDConsumerGroup& group, MDReply::ChannelReply* reply) const { bts_assert(reply); reply->mutable_channel()->CopyFrom(chan); reply->set_platform(group.platform); reply->mutable_channel()->set_time(TimeStamp::now().to_uint64()); auto flags = (MDRequest::Flags)chan.flags(); // Get a copy of handles. If recovery happens during this call // we can safely continue using this pointer for 60 seconds auto phandles = group.handles.get(); if (!phandles) return; MDReply::Update temp_update; for (const auto& instrument : *phandles) { bool snapshot = false; MDConsumerHandle handle = instrument.handle; temp_update.Clear(); MDReply::Update* pupd = &temp_update; pupd->set_key(instrument.handle.key()); bts_assert_d(handle.is_initialized()); if (!handle.is_initialized()) continue; // If the security was just discovered try to ensure all data is sent if (instrument.discovered > TimeStamp(chan.time())) { pupd->mutable_security()->CopyFrom(handle.get_normalized_security()); snapshot = true; } if (HASFLAG(flags, MDRequest::BBO) || HASFLAG(flags, MDRequest::DEPTH)) { PriceSize bid, ask; TimeStamp time(snapshot ? 0 : chan.time()); if (handle.get_bbo(time, bid, ask)) { auto* p = pupd->mutable_book(); convert(bid, p->add_bids()); convert(ask, p->add_asks()); if (HASFLAG(flags, MDRequest::DEPTH) && handle.has_depth()) { for (size_t x = 1; x < COCONUT_MAX_DEPTH; ++x) { PriceSize bid = handle.get_bid(x); if (bid != PriceSize::NO_MARKET) { convert(bid, p->add_bids()); } PriceSize ask = handle.get_ask(x); if (ask != PriceSize::NO_MARKET) { convert(ask, p->add_asks()); } } } } } if (HASFLAG(flags, MDRequest::LAST_TRADE)) { TimeStamp time(snapshot ? 0 : chan.time()); PriceSize price; datetime::LLDateTime trade_time; size_t volume; if (handle.get_last_trade(time, price, trade_time, &volume)) { auto* trade = pupd->mutable_last_trade(); trade->set_key(handle.key()); trade->set_time(trade_time); trade->set_total_volume(volume); set_price(price, trade); } } if (HASFLAG(flags, MDRequest::CUSTOMER_BBO)) { TimeStamp time(snapshot ? 0 : chan.time()); PriceSize bid, ask; if (handle.get_customer_bbo(time, bid, ask)) { auto* p = pupd->mutable_customer_bbo(); convert(bid, p->mutable_bid()); convert(ask, p->mutable_ask()); } } if (HASFLAG(flags, MDRequest::MISC)) { TimeStamp time(snapshot ? 0 : chan.time()); if (handle.misc_timestamp() > time) { copy_misc(handle.misc(), pupd->mutable_misc()); } } if (pupd->has_book() || pupd->has_last_trade() || pupd->has_customer_bbo() || pupd->has_misc() || pupd->has_security()) { reply->add_updates()->CopyFrom(*pupd); } } reply->mutable_channel()->set_segment_id(group.channel.create_time().to_uint64()); bool is_same_segment = (chan.segment_id() == group.channel.create_time().to_uint64()); if (HASFLAG(flags, MDRequest::TRADE_LOG)) { int64_t request_cursor = is_same_segment ? chan.trade_cursor() : 0; auto cursor = group.channel.read_trades(*reply->mutable_trade_log(), request_cursor); if (cursor != chan.trade_cursor()) { reply->mutable_channel()->set_trade_cursor(cursor); } } if (HASFLAG(flags, MDRequest::RFQ)) { int64_t request_cursor = is_same_segment ? chan.rfq_cursor() : 0; auto cursor = group.channel.read_rfqs(*reply->mutable_rfq_log(), request_cursor); if (cursor != chan.rfq_cursor()) { reply->mutable_channel()->set_rfq_cursor(cursor); } } } void query(const MDRequest& request, MDReply* reply) const { bts_assert(reply != nullptr); for (auto& ch : request.channels()) { auto groupiter = groups.find(UserSymbol(ch.symbol(), ch.exchange(), ch.type())); if (groupiter != groups.end()) { auto group = groupiter->second; fill(ch, *group, reply->add_channels()); } } } private: Map groups; }; struct MDRelayServer : bts::co2::MarketDataRelayBase { const QueryServerImpl& _db; utils::Dispatcher& _threadpool; MDRelayServer(const QueryServerImpl& db, utils::Dispatcher& threadpool, bts::api::ServiceContext ctx) : bts::co2::MarketDataRelayBase(ctx), _db(db), _threadpool(threadpool) { } virtual void requestmd(const MDRequest & request, std::function reply) override { _threadpool.post([=]() { google::protobuf::Arena arena; MDReply* preply = google::protobuf::Arena::CreateMessage(&arena); TimeStamp start = TimeStamp::now(); _db.query(request, preply); auto elapsed = (TimeStamp::now() - start); LOG(MarketData, Diag, "Request for: {} took {}us and returned {} bytes before compression", request.host(), elapsed.microseconds(), preply->ByteSize()); reply(*preply); }); } }; struct QueryServer { QueryServer(std::set symbols, utils::Dispatcher& threadpool, const bts::api::ServiceContext& ctx): db(symbols), relay(db, threadpool, ctx), security_timer(), health_timer() { } QueryServerImpl db; MDRelayServer relay; std::unique_ptr security_timer; std::unique_ptr health_timer; }; QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx) { auto& threadpool = utils::Dispatcher::get_named("MDQuery"); std::set all_symbols; get_system_md_symbols(config::get().exchange(), config::get().type(), all_symbols); QueryServerPtr container = std::make_shared(all_symbols, threadpool, ctx); auto& db = container->db; // Scan for new securities every half second container->security_timer = std::make_unique(threadpool, config::get().security_scan_interval(), [&db]() { db.scan_securities(); }); container->security_timer->start(); // Scan for dead producers every 30 seconds container->health_timer = std::make_unique(threadpool, config::get().recovery_interval(), [&db, &threadpool]() { db.check_heartbeats(threadpool); }); container->health_timer->start(); // Add a few threads to keep md latency low for (int x = 1; x < std::min(4, os::num_cores()); ++x) { threadpool.add_thread(); } return container; } std::unique_ptr create_query_server(const std::set& symbols) { return std::make_unique(symbols); } }}//namespace #endif