400 lines
15 KiB
C++
400 lines
15 KiB
C++
#ifndef NO_CPP0X
|
|
#include "QueryServer.h"
|
|
#include "lib/config/config.h"
|
|
#include "lib/utils/Dispatcher.h"
|
|
#include "lib/utils/TimeStamp.h"
|
|
#include "lib/utils/helper_macros.h"
|
|
|
|
#include "../MDConsumer.h"
|
|
#include "../PlatformLoader.h"
|
|
|
|
#include "lib/logging/logging.h"
|
|
|
|
#include "lib/config/protos/system.pb.h"
|
|
#include "lib/md/protos/co2.pb.h"
|
|
#include "lib/api/services/MarketDataRelay.server.pb.h"
|
|
#include "lib/api/public/ServiceContext.h"
|
|
|
|
#include <vector>
|
|
#include <list>
|
|
#include <memory>
|
|
#include <set>
|
|
#include <google/protobuf/arena.h>
|
|
#include <boost/container/flat_map.hpp>
|
|
#include <tbb/concurrent_vector.h>
|
|
|
|
using namespace bts::md2;
|
|
using namespace bts::co2;
|
|
using std::unique_ptr;
|
|
using std::shared_ptr;
|
|
|
|
namespace bts {
|
|
namespace md {
|
|
|
|
/*
|
|
* NORMAL OPERATION:
|
|
* Client sends time=0 to server
|
|
* Server sends complete snapshot back (time=502)
|
|
* client sends time=502 to server
|
|
* Server sends any changes that have happened after time=502 and any securities discovered since time=502
|
|
*
|
|
* ERROR RECOVERY:
|
|
* Client sends time=0 to server
|
|
* Server sends complete snapshot back (time=502)
|
|
* Server detects heartbeat failure and recovers. All timestamps > 502
|
|
* Client sends time=502 to server
|
|
* Server sends complete snapshot back (time=605)
|
|
* - Since the server shared memory has been recreated the client will automatically overwrite all stale data.
|
|
* - If a security has been deleted the client will leave it frozen (OK)
|
|
*
|
|
* We scan for bad heartbeats on a different time
|
|
* If a bad heartbeat is detected we try to construct a new Consumer
|
|
* When a good consumer is created we std::move the old consumer and handles to delayed queue for deletion
|
|
* This way we don't have to lock anything during the recovery process because all pointers will remain valid
|
|
* for the duration of existing calls.
|
|
*
|
|
*/
|
|
struct DatedHandle {
|
|
TimeStamp discovered;
|
|
MDConsumerHandle handle;
|
|
};
|
|
|
|
struct MDConsumerGroup {
|
|
MDConsumerGroup(const UserSymbol& symbol) :
|
|
symbol(symbol),
|
|
platform(platform_for_exchange(symbol.exchange())),
|
|
consumer(symbol),
|
|
channel(consumer.get_channel(symbol)),
|
|
handles(new tbb::concurrent_vector<DatedHandle>())
|
|
{
|
|
initialize_handles();
|
|
}
|
|
// non-reentrant
|
|
void try_recover(utils::Dispatcher& d) {
|
|
// if hearbeat is ok just return
|
|
if (consumer.all_heartbeats_ok()) return;
|
|
// otherwise construct a new MDConsumer
|
|
MDConsumer temp(symbol);
|
|
if (temp.all_heartbeats_ok()) {
|
|
consumer.swap(temp);
|
|
auto temphandles = std::make_unique<tbb::concurrent_vector<DatedHandle>>();
|
|
handles.swap(temphandles);
|
|
initialize_handles();
|
|
// Switch to shared pointer for lambda.
|
|
shared_ptr<tbb::concurrent_vector<DatedHandle>> oldhandles(std::move(temphandles));
|
|
d.post([oldhandles,temp]() mutable {
|
|
// After some time has passed all outstanding requests are guarunteed to be complete
|
|
// Explicit reset of oldhandles ensures they are deleted before the shared memory is unmapped
|
|
// (lambda capture object data members order is not specified)
|
|
oldhandles.reset();
|
|
}, 60000);
|
|
}
|
|
}
|
|
|
|
UserSymbol symbol;
|
|
Platform::type platform;
|
|
MDConsumer consumer;
|
|
MDConsumerChannel channel;
|
|
|
|
// Can add to handles without invalidating readers (no lock needed)
|
|
// When we recover we swap a new vector into this unique_ptr and delayed-delete
|
|
unique_ptr<tbb::concurrent_vector<DatedHandle>> handles;
|
|
|
|
private:
|
|
BTS_NO_COPY(MDConsumerGroup);
|
|
void add_security(MDConsumerHandle handle) {
|
|
bts_assert(handles);
|
|
bts_assert_d(handle.is_initialized());
|
|
DatedHandle h = { TimeStamp::now(), handle };
|
|
handles->push_back(h);
|
|
}
|
|
// We can't copy or move this because initialize_handles implicitly captures 'this'
|
|
void initialize_handles() {
|
|
bts_assert(handles->empty());
|
|
auto id = consumer.id();
|
|
channel = consumer.get_channel(symbol.symbol(), symbol.type(), symbol.exchange());
|
|
consumer.on_security(
|
|
[this,id](const Security&, MDConsumerHandle handle) {
|
|
if (id == consumer.id()) {
|
|
add_security(handle);
|
|
}
|
|
}, false);
|
|
}
|
|
};
|
|
|
|
// Wrap the flat_map to ensure immutability
|
|
class Map {
|
|
// Can't use unique_ptr because flat_map requires assignment.
|
|
typedef boost::container::flat_map<UserSymbol, shared_ptr<MDConsumerGroup>> InternalMap;
|
|
InternalMap _map;
|
|
public:
|
|
Map(InternalMap map) : _map(move(map)) { }
|
|
Map(Map&& other) : _map(move(other._map)) {}
|
|
|
|
InternalMap::iterator begin() { return _map.begin(); }
|
|
InternalMap::iterator end() { return _map.end(); }
|
|
InternalMap::const_iterator begin() const { return _map.begin(); }
|
|
InternalMap::const_iterator end() const { return _map.end(); }
|
|
InternalMap::const_iterator find(const UserSymbol& symbol) const { return _map.find(symbol); }
|
|
};
|
|
|
|
Map form_groups(const std::set<UserSymbol>& symbols) {
|
|
boost::container::flat_map<UserSymbol, shared_ptr<MDConsumerGroup>> groups;
|
|
for (const auto& sym : symbols) {
|
|
groups.emplace(std::make_pair(sym, std::make_unique<MDConsumerGroup>(sym)));
|
|
}
|
|
groups.shrink_to_fit();
|
|
return Map(groups);
|
|
}
|
|
|
|
void convert(PriceSize ps, PriceSizeMessage* pps) {
|
|
pps->set_mantissa(ps.mantissa());
|
|
pps->set_exponent(ps.exponent());
|
|
pps->set_size(ps.size());
|
|
}
|
|
void set_price(PriceSize ps, co2::TradeMessage* ptrade) {
|
|
ptrade->set_mantissa(ps.mantissa());
|
|
ptrade->set_exponent(ps.exponent());
|
|
ptrade->set_size(ps.size());
|
|
}
|
|
void copy_misc(MiscData misc, co2::MiscMessage* pmisc) {
|
|
pmisc->set_settlement_price(misc.settlement_price);
|
|
pmisc->set_settlement_date(misc.settlement_date.to_short());
|
|
pmisc->set_previous_settlement_price(misc.previous_settlement_price);
|
|
pmisc->set_previous_settlement_date(misc.previous_settlement_date.to_short());
|
|
pmisc->set_open_price(misc.open_price);
|
|
pmisc->set_open_date(misc.open_date.to_short());
|
|
pmisc->set_close_price(misc.close_price);
|
|
pmisc->set_close_date(misc.close_date.to_short());
|
|
//pmisc->set_fixing_price(misc.fixing_price);
|
|
pmisc->set_session_high(misc.session_high);
|
|
pmisc->set_session_low(misc.session_low);
|
|
//pmisc->set_open_interest(misc.open_interest);
|
|
//pmisc->set_session_high_bid(misc.session_high_bid);
|
|
//pmisc->set_session_low_offer(misc.session_low_offer);
|
|
}
|
|
|
|
// Handles market data queries and responds
|
|
struct QueryServerImpl : public IQueryServer {
|
|
QueryServerImpl(const std::set<UserSymbol>& symbols) :
|
|
groups(form_groups(symbols))
|
|
{ }
|
|
|
|
// Scan for new securities and get their handles+times
|
|
// not-reentrant
|
|
void scan_securities() const {
|
|
for (auto& group : groups) {
|
|
if (group.second->consumer.all_heartbeats_ok()) {
|
|
TimeStamp start = TimeStamp::now();
|
|
size_t count = group.second->consumer.scan_securities(false);
|
|
auto delta = (TimeStamp::now() - start);
|
|
if (delta.milliseconds() > 10) {
|
|
LOG(MarketData,Diag,"Scanned handles: {}ms found: {} securities. " ,
|
|
delta.milliseconds() ,count ,group.second->symbol );
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// not-reentrant
|
|
void check_heartbeats(utils::Dispatcher& dispatcher) {
|
|
for (auto& group : groups) {
|
|
if (!group.second->consumer.all_heartbeats_ok()) {
|
|
group.second->try_recover(dispatcher);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
// the hot part
|
|
void fill(const SecurityChannel& chan, MDConsumerGroup& group, MDReply::ChannelReply* reply) const {
|
|
bts_assert(reply);
|
|
reply->mutable_channel()->CopyFrom(chan);
|
|
reply->set_platform(group.platform);
|
|
reply->mutable_channel()->set_time(TimeStamp::now().to_uint64());
|
|
auto flags = (MDRequest::Flags)chan.flags();
|
|
|
|
// Get a copy of handles. If recovery happens during this call
|
|
// we can safely continue using this pointer for 60 seconds
|
|
auto phandles = group.handles.get();
|
|
if (!phandles) return;
|
|
|
|
MDReply::Update temp_update;
|
|
for (const auto& instrument : *phandles) {
|
|
bool snapshot = false;
|
|
MDConsumerHandle handle = instrument.handle;
|
|
temp_update.Clear();
|
|
MDReply::Update* pupd = &temp_update;
|
|
pupd->set_key(instrument.handle.key());
|
|
bts_assert_d(handle.is_initialized());
|
|
if (!handle.is_initialized()) continue;
|
|
// If the security was just discovered try to ensure all data is sent
|
|
if (instrument.discovered > TimeStamp(chan.time())) {
|
|
pupd->mutable_security()->CopyFrom(handle.get_normalized_security());
|
|
snapshot = true;
|
|
}
|
|
if (HASFLAG(flags, MDRequest::BBO) || HASFLAG(flags, MDRequest::DEPTH)) {
|
|
PriceSize bid, ask;
|
|
TimeStamp time(snapshot ? 0 : chan.time());
|
|
if (handle.get_bbo(time, bid, ask)) {
|
|
auto* p = pupd->mutable_book();
|
|
convert(bid, p->add_bids());
|
|
convert(ask, p->add_asks());
|
|
if (HASFLAG(flags, MDRequest::DEPTH) && handle.has_depth()) {
|
|
for (size_t x = 1; x < COCONUT_MAX_DEPTH; ++x) {
|
|
PriceSize bid = handle.get_bid(x);
|
|
if (bid != PriceSize::NO_MARKET) {
|
|
convert(bid, p->add_bids());
|
|
}
|
|
PriceSize ask = handle.get_ask(x);
|
|
if (ask != PriceSize::NO_MARKET) {
|
|
convert(ask, p->add_asks());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (HASFLAG(flags, MDRequest::LAST_TRADE)) {
|
|
TimeStamp time(snapshot ? 0 : chan.time());
|
|
PriceSize price;
|
|
datetime::LLDateTime trade_time;
|
|
size_t volume;
|
|
if (handle.get_last_trade(time, price, trade_time, &volume)) {
|
|
auto* trade = pupd->mutable_last_trade();
|
|
trade->set_key(handle.key());
|
|
trade->set_time(trade_time);
|
|
trade->set_total_volume(volume);
|
|
set_price(price, trade);
|
|
}
|
|
}
|
|
if (HASFLAG(flags, MDRequest::CUSTOMER_BBO)) {
|
|
TimeStamp time(snapshot ? 0 : chan.time());
|
|
PriceSize bid, ask;
|
|
if (handle.get_customer_bbo(time, bid, ask)) {
|
|
auto* p = pupd->mutable_customer_bbo();
|
|
convert(bid, p->mutable_bid());
|
|
convert(ask, p->mutable_ask());
|
|
}
|
|
}
|
|
if (HASFLAG(flags, MDRequest::MISC)) {
|
|
TimeStamp time(snapshot ? 0 : chan.time());
|
|
if (handle.misc_timestamp() > time) {
|
|
copy_misc(handle.misc(), pupd->mutable_misc());
|
|
}
|
|
}
|
|
if (pupd->has_book() ||
|
|
pupd->has_last_trade() ||
|
|
pupd->has_customer_bbo() ||
|
|
pupd->has_misc() ||
|
|
pupd->has_security()) {
|
|
reply->add_updates()->CopyFrom(*pupd);
|
|
}
|
|
}
|
|
|
|
reply->mutable_channel()->set_segment_id(group.channel.create_time().to_uint64());
|
|
bool is_same_segment = (chan.segment_id() == group.channel.create_time().to_uint64());
|
|
|
|
if (HASFLAG(flags, MDRequest::TRADE_LOG)) {
|
|
int64_t request_cursor = is_same_segment ? chan.trade_cursor() : 0;
|
|
auto cursor = group.channel.read_trades(*reply->mutable_trade_log(), request_cursor);
|
|
if (cursor != chan.trade_cursor()) {
|
|
reply->mutable_channel()->set_trade_cursor(cursor);
|
|
}
|
|
}
|
|
if (HASFLAG(flags, MDRequest::RFQ)) {
|
|
int64_t request_cursor = is_same_segment ? chan.rfq_cursor() : 0;
|
|
auto cursor = group.channel.read_rfqs(*reply->mutable_rfq_log(), request_cursor);
|
|
if (cursor != chan.rfq_cursor()) {
|
|
reply->mutable_channel()->set_rfq_cursor(cursor);
|
|
}
|
|
}
|
|
}
|
|
|
|
void query(const MDRequest& request, MDReply* reply) const {
|
|
bts_assert(reply != nullptr);
|
|
for (auto& ch : request.channels()) {
|
|
auto groupiter = groups.find(UserSymbol(ch.symbol(), ch.exchange(), ch.type()));
|
|
if (groupiter != groups.end()) {
|
|
auto group = groupiter->second;
|
|
fill(ch, *group, reply->add_channels());
|
|
}
|
|
}
|
|
}
|
|
|
|
private:
|
|
Map groups;
|
|
};
|
|
|
|
struct MDRelayServer : bts::co2::MarketDataRelayBase {
|
|
const QueryServerImpl& _db;
|
|
utils::Dispatcher& _threadpool;
|
|
MDRelayServer(const QueryServerImpl& db, utils::Dispatcher& threadpool, bts::api::ServiceContext ctx) :
|
|
bts::co2::MarketDataRelayBase(ctx),
|
|
_db(db), _threadpool(threadpool)
|
|
{ }
|
|
|
|
virtual void requestmd(const MDRequest & request, std::function<void(const MDReply&)> reply) override {
|
|
_threadpool.post([=]() {
|
|
google::protobuf::Arena arena;
|
|
MDReply* preply = google::protobuf::Arena::CreateMessage<MDReply>(&arena);
|
|
TimeStamp start = TimeStamp::now();
|
|
_db.query(request, preply);
|
|
auto elapsed = (TimeStamp::now() - start);
|
|
LOG(MarketData, Diag,
|
|
"Request for: {} took {}us and returned {} bytes before compression",
|
|
request.host(),
|
|
elapsed.microseconds(),
|
|
preply->ByteSize());
|
|
reply(*preply);
|
|
});
|
|
}
|
|
};
|
|
|
|
struct QueryServer {
|
|
QueryServer(std::set<UserSymbol> symbols, utils::Dispatcher& threadpool, const bts::api::ServiceContext& ctx):
|
|
db(symbols),
|
|
relay(db, threadpool, ctx),
|
|
security_timer(),
|
|
health_timer()
|
|
{ }
|
|
QueryServerImpl db;
|
|
MDRelayServer relay;
|
|
std::unique_ptr<utils::DispatcherTimer> security_timer;
|
|
std::unique_ptr<utils::DispatcherTimer> health_timer;
|
|
};
|
|
|
|
QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx) {
|
|
auto& threadpool = utils::Dispatcher::get_named("MDQuery");
|
|
|
|
std::set<UserSymbol> all_symbols;
|
|
get_system_md_symbols(config::get<PlatformConfig>().exchange(), config::get<PlatformConfig>().type(), all_symbols);
|
|
QueryServerPtr container = std::make_shared<QueryServer>(all_symbols, threadpool, ctx);
|
|
auto& db = container->db;
|
|
|
|
// Scan for new securities every half second
|
|
container->security_timer = std::make_unique<utils::DispatcherTimer>(threadpool, config::get<RelayOptions>().security_scan_interval(), [&db]() {
|
|
db.scan_securities();
|
|
});
|
|
container->security_timer->start();
|
|
|
|
// Scan for dead producers every 30 seconds
|
|
container->health_timer = std::make_unique<utils::DispatcherTimer>(threadpool, config::get<RelayOptions>().recovery_interval(), [&db, &threadpool]() {
|
|
db.check_heartbeats(threadpool);
|
|
});
|
|
container->health_timer->start();
|
|
|
|
// Add a few threads to keep md latency low
|
|
for (int x = 1; x < std::min(4, os::num_cores()); ++x) {
|
|
threadpool.add_thread();
|
|
}
|
|
|
|
return container;
|
|
}
|
|
|
|
std::unique_ptr<IQueryServer> create_query_server(const std::set<md2::UserSymbol>& symbols) {
|
|
return std::make_unique<QueryServerImpl>(symbols);
|
|
}
|
|
|
|
}}//namespace
|
|
#endif
|