#ifndef NO_CPP0X #include "RelayClient.h" #include "RelayRecorder.h" #include "lib/config/config.h" #include "lib/utils/TimeStamp.h" #include "lib/utils/bts_chrono.h" #include "lib/utils/AuthToken.h" #include "../MDProducer.h" #include "../get_symbols.h" #include "lib/logging/logging.h" #include "lib/config/protos/system.pb.h" #include "lib/config/protos/MarketData.pb.h" #include "lib/md/protos/co2.pb.h" #include "lib/api/services/MarketDataRelay.client.pb.h" #include "lib/api/public/ServiceContext.h" #include "lib/utils/base64.hpp" #include #include #include #include #include #include #include #include #include using namespace std; namespace bts { namespace md { using namespace co2; PriceSize convert(const PriceSizeMessage& psm) { return psm.size() > 0 ? PriceSize({psm.mantissa(), (int8_t)psm.exponent()}, psm.size()) : NO_MARKET; } PriceSize trade_pricesize(const co2::TradeMessage& trade) { return trade.size() > 0 ? PriceSize({trade.mantissa(), (int8_t)trade.exponent()}, trade.size()) : NO_MARKET; } PriceSize book_at(const google::protobuf::RepeatedPtrField& side, int index) { if (side.size() > index) { return convert(side.Get(index)); } return NO_MARKET; } IntPrice convert(const PriceMessage& price) { return { price.mantissa(), (int8_t)price.exponent() }; } class RelayProducer : public IRelayWriter { bool _shared; std::map _producers; unordered_map _handles; void record_update(MDProducerHandle& handle, const ChannelUpdate& update) { utils::TimeStamp updated = utils::TimeStamp::now(); bool snapshot = update.has_security(); // If this is a snapshot an empty book() means "no market" rather than "no change" if (update.has_direct() || update.has_implied() || snapshot) { std::array bids, asks; std::array implied_bids, implied_asks; for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { bids[x] = book_at(update.direct().bids(), x); asks[x] = book_at(update.direct().asks(), x); } for (size_t x = 0; x second; } } return find_result->second; } void write(const co2::MDReply& data, bool* phas_new_securities) override { for (const auto& ch : data.channels()) { // Per-security data int added_security_count = 0; for (const auto& update : ch.updates()) { auto find_result = _handles.find(update.key()); if (update.has_security() && find_result == _handles.end()) { if (phas_new_securities) *phas_new_securities = true; added_security_count++; MDProducer& producer = ensure_producer(ch.platform()); with_security_like(update.security(), [&](FixedSecurity& fs) { convert_security(update.security(), fs, true); MDProducerHandle handle = producer.add_security(fs); _handles.insert(make_pair(handle.key(), handle)); record_update(handle, update); }); } else if (find_result != _handles.end()) { MDProducerHandle& handle = find_result->second; record_update(handle, update); } } // Heartbeat auto iter = _producers.find(ch.platform()); if (iter != _producers.end()) { auto& producer = iter->second; auto channel = producer.get_channel(ch.channel().symbol().c_str(), ch.channel().type(), ch.channel().exchange()); channel.heartbeat(); // RFQ and TRADE LOG if (ch.trade_log_size() > 0 || ch.rfq_log_size() > 0) { if (ch.trade_log_size() > 0) { LOG(md,Diag, "Added {} trades", ch.trade_log_size()); } for (const auto& trade : ch.trade_log()) { if (trade.key()) { channel.add_trade(trade); } } for (const auto& rfq : ch.rfq_log()) { channel.add_rfq(rfq); } } } if (added_security_count > 0) { LOG(md,Diag,"Added {} {} - {}" , added_security_count , ch.channel().symbol() ,SecurityType_Name(ch.channel().type())); } } } }; shared_ptr mkrelayproducer(bool shared) { return make_shared(shared); } // Single Threaded class RelayClient { MDRequest _request; MarketDataRelay& _rpc; MDConsumer* _pconsumer; IRelayWriter& _writer; utils::TimeStamp _request_log; public: RelayClient( MarketDataRelay& rpc, MDConsumer* pconsumer, const std::set& symbols, IRelayWriter& writer) : _request(), _rpc(rpc), _pconsumer(pconsumer), _writer(writer), _request_log(utils::TimeStamp::now()) { ostringstream fmt; fmt << config::get().host_name() << "(" << os::process_id() << ")"; _request.set_host(fmt.str()); for (const auto& s : symbols) { SecurityChannel ch; ch.set_symbol(s.symbol()); ch.set_type(s.type()); ch.set_exchange(s.exchange()); ch.set_flags(0xff); add_symbol(ch); } } // Can be called safely with the same symbol void add_symbol(const SecurityChannel& ch) { auto find_result = std::find_if(_request.mutable_channels()->begin(), _request.mutable_channels()->end(), [&](const SecurityChannel& c) { return c.symbol() == ch.symbol() && c.type() == ch.type() && c.exchange() == ch.exchange(); }); if (find_result == _request.mutable_channels()->end()) { auto* new_channel = _request.add_channels(); new_channel->CopyFrom(ch); new_channel->set_time(0); } else { // update flags auto flags = find_result->flags() | ch.flags(); if (flags != find_result->flags()) { find_result->set_flags(flags); // After changing flags request a new snapshot find_result->set_time(0); } } } std::string compact_log_format() const { ostringstream log; log << "{"; for (const auto& ch : _request.channels()) { log << std::endl; log << ch.symbol() << " "; switch(ch.type()){ case SecurityType::SECURITYTYPE_FUTURE: log << "Fut "; break; case SecurityType::SECURITYTYPE_SPREAD: log << "Spr "; break; case SecurityType::SECURITYTYPE_OPTION: log << "Opt "; break; case SecurityType::SECURITYTYPE_SYNTHETIC: log << "Syn "; break; default: break; } log << Venue_Name(ch.exchange()).substr(sizeof("venue")); log << " " << (uint32_t)ch.time(); log << " " << (uint16_t)ch.segment_id(); } log << "\n}"; auto encoded = websocketpp::base64_encode(log.str()); return encoded; } void request_md(std::function cont) { auto now = utils::TimeStamp::now(); if ((now - _request_log).seconds() > 60) { _request_log = now; LOG(md, Info, "Requesting MD (b64): {}", compact_log_format()); } _rpc.requestmd(_request, [=](const api::Result& r) { receive_md(r); cont(); }, 500u); } void receive_md(const api::Result& result) { if (result.success()) { // Write data to shared mem bool has_new_security = false; _writer.write(*result.value(), &has_new_security); // If the reply included and we have a consumer have it scan them if (has_new_security && _pconsumer) { scan_securities_thread().post([=]() { auto count = _pconsumer->scan_securities(true); LOG(md, Info, "Discovered {} securities.", count); }); } // Update reply timetamps for (auto& req : *_request.mutable_channels()) { for (auto& reply : result.value()->channels()) { if (req.type() == reply.channel().type() && req.exchange() == reply.channel().exchange() && req.symbol() == reply.channel().symbol()) { req.set_entitled(reply.channel().entitled()); req.set_time(reply.channel().time()); req.set_trade_cursor(reply.channel().trade_cursor()); req.set_rfq_cursor(reply.channel().rfq_cursor()); if (req.segment_id() != reply.channel().segment_id()) { LOG(md, Diag, "New segment_id = {}", reply.channel().segment_id()); req.set_segment_id(reply.channel().segment_id()); } } } } } } }; struct RelayScheduler { uint32_t _interval; api::ServiceContext& _ctx; RelayClient& _client; bool running; double speed_factor; RelayScheduler(api::ServiceContext& context, uint32_t interval, RelayClient& client) : _interval(interval), _ctx(context), _client(client), running(true), speed_factor(config::get().speed()) { } void run() { if (running) { utils::TimeStamp start = utils::TimeStamp::now(); _client.request_md([=]() { int64_t elapsed_ms = (utils::TimeStamp::now() - start).milliseconds(); int64_t delay = (_interval - elapsed_ms) / speed_factor; _ctx.post([=]() { run(); }, std::max(delay, (int64_t)0)); }); } } }; api::ServiceContext elevated_relay_context() { api::ServiceContext ctx; ctx.login(api::auth::generate_wheel_token()); return ctx; } void relay_selected(api::ServiceContext& ctx, const std::set& symbols, IRelayWriter& writer) { auto relay = ctx.locate(); RelayClient client(relay, nullptr, symbols, writer); RelayScheduler scheduler(ctx, config::get().naggle_ms(), client); ctx.post([&]() { scheduler.run(); }); // block forever, I guess utils::Dispatcher::current().run(); } void relay_everything() { api::ServiceContext ctx = elevated_relay_context(); std::set system_symbols; get_system_md_symbols(system_symbols); RelayProducer producer(true); relay_selected(ctx, system_symbols, producer); } std::set get_relay_symbols(bool record_sim) { std::set symbols; if (record_sim) { get_system_md_symbols(symbols); } else { std::set system_symbols; get_system_md_symbols(system_symbols); // Filter out VENUE_SIM for (const auto& s : system_symbols) { if (s.exchange() != api::VENUE_SIM) { symbols.insert(s); } } } return symbols; } void local_relay(IRelayWriter& writer) { api::ServiceContext ctx = elevated_relay_context(); ctx.login(api::auth::generate_wheel_token()); auto relay = ctx.locate(); auto symbols = get_relay_symbols(false); relay_selected(ctx, symbols, writer); } void record_everything(std::string path, std::string filename) { auto options = config::get(); if (filename.empty()) { ostringstream file; file << "marketdata" << "-"; int year, month, day, hour, minute, unused; datetime::ll_to_parts(datetime::ll_local_now(), year, month, day, hour, minute, unused, unused); file << year << setw(2) << setfill('0') << month << setw(2) << setfill('0') << day << "-" << setw(2) << setfill('0') << hour << setw(2) << setfill('0') << minute << ".coconut"; filename = file.str(); } if (path.empty()) { path = os::sparkdata_path("record"); } auto naggle_ms = config::get().naggle_ms(); RelayRecorder recorder(path, filename, naggle_ms); api::ServiceContext ctx = elevated_relay_context(); auto relay = ctx.locate(); std::set system_symbols = get_relay_symbols(options.record_sim()); RelayClient client(relay, nullptr, system_symbols, recorder); RelayScheduler scheduler(ctx, naggle_ms, client); scheduler.run(); utils::Dispatcher::current().run(); } /* * extern api for c# */ struct RelayControl { RelayControl(MDConsumer* pconsumer, uint32_t interval) : context(), relay(context.locate()), producer(false), client(relay, pconsumer, std::set(), producer), scheduler(context, interval, client) { scheduler.run(); } api::ServiceContext context; MarketDataRelay relay; RelayProducer producer; RelayClient client; RelayScheduler scheduler; }; RelayControl* start_private_relay(MDConsumer* pconsumer, uint32_t interval) { bts_assert(pconsumer); return new RelayControl(pconsumer, interval); } void stop_relay(RelayControl* relay) { if (relay) { relay->scheduler.running = false; bool terminated = false; relay->context.post([&]() { terminated = true; }, 500); } } void add_relay_symbol(RelayControl* relay, const SecurityChannel& chan) { bts_assert(relay); relay->context.post([=]() { relay->client.add_symbol(chan); }); } void set_relay_token(RelayControl* relay, const std::string& token) { bts_assert(relay); relay->context.post([=]() { api::AuthToken auth_token; auth_token.set_token(token); relay->context.login(auth_token); }); } utils::Dispatcher& scan_securities_thread() { return utils::Dispatcher::background(); } } }//namespace #endif