diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index 10512f708..9fa215c3c 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -78,6 +78,10 @@ message CMEOptions repeated uint32 affinity = 10; optional uint32 udp_socket_buffer_size = 11 [default = 33554432]; //32mb } + enum BookType { + BOOKTYPE_CONSOLIDATED = 0; + BOOKTYPE_DIRECT = 1; + } message CMEChannel { repeated string id = 2; optional string feed_engine = 3 [default = "default"]; @@ -100,6 +104,7 @@ message CMEOptions optional bool join_snapshot_on_gap = 32 [default = false]; optional bool load_security_db = 33 [default = true]; + optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED]; } optional string onixs_license = 1 [default = "{default}"]; diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 388e15c97..d94c4a34a 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -8,6 +8,7 @@ #include "lib/logging/logging.h" #include "lib/os/os.h" #include "lib/quickbus/quickbus.h" +#include "lib/utils/Dispatcher.h" #include "lib/md/SyntheticSecurityService.h" #include "server/sparkd/exchange_sim/ExchangeSim.h" @@ -55,6 +56,11 @@ int Run::execute(std::vector const &) { support::ErrorReportServer error_report_service(sc); admin::AdminServiceServer admin_service(sc); metrics::ShellLatencyServer shell_latency_service(sc, stats, utils::Dispatcher::background()); + + auto bg_health = std::make_shared("spark_background_thread"); + utils::DispatcherTimer timer(utils::Dispatcher::background(), 5000, [=]() { bg_health->increment(); }); + timer.start(); + stats.publish(bg_health); std::thread([]() { os::set_thread_name("ExchangeSim"); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index fd5e34f81..ecb96d617 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -1,8 +1,8 @@ #include "MDP3Listener.h" #include "lib/utils/datetime.h" #include -#include "server/sparkd/tracing/bts_trace.h" #include "lib/utils/helper_macros.h" +#include "lib/config/protos/MarketData.pb.h" using namespace std; using namespace bts::co2; @@ -84,7 +84,8 @@ MDP3Listener::MDP3Listener( MDProducer& producer, SecurityIDRecorder& recorder, co2::SettlementRecorder& , - std::vector channels) : + const std::vector channels, + CMEOptions::BookType bt) : _index(index), _producer(producer), _packet_arrived(), @@ -94,6 +95,7 @@ MDP3Listener::MDP3Listener( _symbol_filter_storage(), _symbol_filter(), _channels(channels), + _booktype(bt), _concurrency_monitor(0), _book_updates() { @@ -185,11 +187,6 @@ double double_of_price(const Price& price, int8_t display_factor) { return (price.mantissa() * pow10(price.exponent())) * pow10(display_factor); } -inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) { - PriceSize result(price.mantissa(), price.exponent() + display_factor, size); - return result; -} - inline datetime::LLDateTime ll_of_timestamp(const onixs::Timestamp& ts) { return datetime::ll_from_parts(ts.year(), (int)ts.month(), ts.day(), ts.hour(), ts.minute(), ts.second(), ts.millisecond()); } @@ -337,49 +334,18 @@ void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security&, const o _index.add_spread_legs(fsec.numeric_id, leg_exchange_ids); } - -// -// Book -void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) { - tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", _latest_sequence, (uint16_t) _latest_sequence, 0); - - auto data = _index.get_security(sec.id()); - if (data == nullptr) return; - bts_assert(data->id == (uint32_t)sec.id()); - auto display_factor = data->handle.security().display_factor; - - // Put the data into the cache. Flush at the end of the packet. - auto key = data->handle.key(); - auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) { - return key == entry.handle.key(); - }); - bool found = find_iter != _book_updates.end(); - if (!found) { - _book_updates.emplace_back(data->handle); - } - BookData& cache = found ? *find_iter : _book_updates.back(); - - // Overwrite all entries every time - size_t bids_size = book.bids().size(); - size_t asks_size = book.offers().size(); - if (!data->handle.has_depth()) { - bids_size = min(bids_size, size_t { 1 }); - asks_size = min(asks_size, size_t { 1 }); - } - for (size_t x = 0; x < bids_size; ++x) { - cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity()); - } - for (size_t x = 0; x < asks_size; ++x) { - cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity()); - } - // NaN out the remaining depth - for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) { - cache.bids[x] = PriceSize::NO_MARKET; - } - for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) { - cache.asks[x] = PriceSize::NO_MARKET; +void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) { + if (_booktype == CMEOptions::BOOKTYPE_DIRECT) { + onBookUpdate(sec, book); } } + +void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) { + if (_booktype == CMEOptions::BOOKTYPE_CONSOLIDATED) { + onBookUpdate(sec, book); + } +} + // // Trades diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index e5caff108..21b966ab7 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -3,6 +3,7 @@ #include "lib/utils/bts_no_copy.h" #include "lib/md/coconut2.h" #include "CMESecurityIndex.h" +#include "server/sparkd/tracing/bts_trace.h" #include "onixs.h" #include @@ -13,13 +14,23 @@ namespace bts { namespace md { +inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) { + PriceSize result(price.mantissa(), price.exponent() + display_factor, size); + return result; +} + class MDP3Listener : public onixs::HandlerListener, public onixs::MarketDataListener, public onixs::SecurityListener { public: - MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector); + MDP3Listener(CMESecurityIndex& index, + MDProducer&, + SecurityIDRecorder&, + co2::SettlementRecorder&, + const std::vector, + CMEOptions::BookType); virtual ~MDP3Listener(); // @@ -50,10 +61,52 @@ public: // // Book // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override; - // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override; + void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override; // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ImpliedBook&) override; void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override; + template + void onBookUpdate(const onixs::Security& sec, const TBook& book) { + tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", 0,_latest_sequence); + + auto data = _index.get_security(sec.id()); + if (data == nullptr) return; + bts_assert(data->id == (uint32_t)sec.id()); + auto display_factor = data->handle.security().display_factor; + + // Put the data into the cache. Flush at the end of the packet. + auto key = data->handle.key(); + auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) { + return key == entry.handle.key(); + }); + bool found = find_iter != _book_updates.end(); + if (!found) { + _book_updates.emplace_back(data->handle); + } + BookData& cache = found ? *find_iter : _book_updates.back(); + + // Overwrite all entries every time + size_t bids_size = book.bids().size(); + size_t asks_size = book.offers().size(); + if (!data->handle.has_depth()) { + bids_size = std::min(bids_size, size_t { 1 }); + asks_size = std::min(asks_size, size_t { 1 }); + } + for (size_t x = 0; x < bids_size; ++x) { + cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity()); + } + for (size_t x = 0; x < asks_size; ++x) { + cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity()); + } + // NaN out the remaining depth + for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) { + cache.bids[x] = PriceSize::NO_MARKET; + } + for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) { + cache.asks[x] = PriceSize::NO_MARKET; + } + } + // // Trades void onTrade(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override; @@ -124,6 +177,7 @@ private: std::vector _symbol_filter_storage; std::set _symbol_filter; std::vector _channels; + CMEOptions::BookType _booktype; std::atomic _concurrency_monitor; struct BookData { diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 598d7ff49..afec4cea9 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -213,7 +213,9 @@ unique_ptr MDP3Main::prepare_handler(const string& channel_id) { settings.feeds().connectivityConfigurationFile(config_path()); // Market data options - settings.bookManagement().consolidatedBooks().maintain(true); + if (options.book_type() == CMEOptions::BOOKTYPE_CONSOLIDATED) { + settings.bookManagement().consolidatedBooks().maintain(true); + } // 10k default is *far* too low settings.recoveryQueueMaxSize(options.recovery_queue_size()); @@ -334,12 +336,13 @@ unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, con producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } - unique_ptr listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels)); + const auto& options = channel_settings_for(channel.id); + + unique_ptr listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels, options.book_type())); for (auto& symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } - const auto& options = channel_settings_for(channel.id); if (options.load_security_db()) { // Load the list of known securities and add them to the index for fast startup db::load_securities(channel.symbols, [&](const api::Security& security) {