merge from prod
This commit is contained in:
commit
802160abf1
5 files changed
+87
-53
No files matched your search
@@ -78,6 +78,10 @@ message CMEOptions
|
||||
repeated uint32 affinity = 10;
|
||||
optional uint32 udp_socket_buffer_size = 11 [default = 33554432]; //32mb
|
||||
}
|
||||
enum BookType {
|
||||
BOOKTYPE_CONSOLIDATED = 0;
|
||||
BOOKTYPE_DIRECT = 1;
|
||||
}
|
||||
message CMEChannel {
|
||||
repeated string id = 2;
|
||||
optional string feed_engine = 3 [default = "default"];
|
||||
@@ -100,6 +104,7 @@ message CMEOptions
|
||||
optional bool join_snapshot_on_gap = 32 [default = false];
|
||||
|
||||
optional bool load_security_db = 33 [default = true];
|
||||
optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED];
|
||||
|
||||
}
|
||||
optional string onixs_license = 1 [default = "{default}"];
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/os/os.h"
|
||||
#include "lib/quickbus/quickbus.h"
|
||||
#include "lib/utils/Dispatcher.h"
|
||||
|
||||
#include "lib/md/SyntheticSecurityService.h"
|
||||
#include "server/sparkd/exchange_sim/ExchangeSim.h"
|
||||
@@ -55,6 +56,11 @@ int Run::execute(std::vector<std::string> const &) {
|
||||
support::ErrorReportServer error_report_service(sc);
|
||||
admin::AdminServiceServer admin_service(sc);
|
||||
metrics::ShellLatencyServer shell_latency_service(sc, stats, utils::Dispatcher::background());
|
||||
|
||||
auto bg_health = std::make_shared<metrics::IntCounterMetric>("spark_background_thread");
|
||||
utils::DispatcherTimer timer(utils::Dispatcher::background(), 5000, [=]() { bg_health->increment(); });
|
||||
timer.start();
|
||||
stats.publish(bg_health);
|
||||
|
||||
std::thread([]() {
|
||||
os::set_thread_name("ExchangeSim");
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
#include "MDP3Listener.h"
|
||||
#include "lib/utils/datetime.h"
|
||||
#include <vector>
|
||||
#include "server/sparkd/tracing/bts_trace.h"
|
||||
#include "lib/utils/helper_macros.h"
|
||||
#include "lib/config/protos/MarketData.pb.h"
|
||||
|
||||
using namespace std;
|
||||
using namespace bts::co2;
|
||||
@@ -84,7 +84,8 @@ MDP3Listener::MDP3Listener(
|
||||
MDProducer& producer,
|
||||
SecurityIDRecorder& recorder,
|
||||
co2::SettlementRecorder& ,
|
||||
std::vector<MDProducerChannel> channels) :
|
||||
const std::vector<MDProducerChannel> channels,
|
||||
CMEOptions::BookType bt) :
|
||||
_index(index),
|
||||
_producer(producer),
|
||||
_packet_arrived(),
|
||||
@@ -94,6 +95,7 @@ MDP3Listener::MDP3Listener(
|
||||
_symbol_filter_storage(),
|
||||
_symbol_filter(),
|
||||
_channels(channels),
|
||||
_booktype(bt),
|
||||
_concurrency_monitor(0),
|
||||
_book_updates()
|
||||
{
|
||||
@@ -185,11 +187,6 @@ double double_of_price(const Price& price, int8_t display_factor) {
|
||||
return (price.mantissa() * pow10(price.exponent())) * pow10(display_factor);
|
||||
}
|
||||
|
||||
inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
|
||||
PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
|
||||
return result;
|
||||
}
|
||||
|
||||
inline datetime::LLDateTime ll_of_timestamp(const onixs::Timestamp& ts) {
|
||||
return datetime::ll_from_parts(ts.year(), (int)ts.month(), ts.day(), ts.hour(), ts.minute(), ts.second(), ts.millisecond());
|
||||
}
|
||||
@@ -337,49 +334,18 @@ void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security&, const o
|
||||
_index.add_spread_legs(fsec.numeric_id, leg_exchange_ids);
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// Book
|
||||
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
|
||||
tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", _latest_sequence, (uint16_t) _latest_sequence, 0);
|
||||
|
||||
auto data = _index.get_security(sec.id());
|
||||
if (data == nullptr) return;
|
||||
bts_assert(data->id == (uint32_t)sec.id());
|
||||
auto display_factor = data->handle.security().display_factor;
|
||||
|
||||
// Put the data into the cache. Flush at the end of the packet.
|
||||
auto key = data->handle.key();
|
||||
auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
|
||||
return key == entry.handle.key();
|
||||
});
|
||||
bool found = find_iter != _book_updates.end();
|
||||
if (!found) {
|
||||
_book_updates.emplace_back(data->handle);
|
||||
}
|
||||
BookData& cache = found ? *find_iter : _book_updates.back();
|
||||
|
||||
// Overwrite all entries every time
|
||||
size_t bids_size = book.bids().size();
|
||||
size_t asks_size = book.offers().size();
|
||||
if (!data->handle.has_depth()) {
|
||||
bids_size = min(bids_size, size_t { 1 });
|
||||
asks_size = min(asks_size, size_t { 1 });
|
||||
}
|
||||
for (size_t x = 0; x < bids_size; ++x) {
|
||||
cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
|
||||
}
|
||||
for (size_t x = 0; x < asks_size; ++x) {
|
||||
cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
|
||||
}
|
||||
// NaN out the remaining depth
|
||||
for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
|
||||
cache.bids[x] = PriceSize::NO_MARKET;
|
||||
}
|
||||
for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
|
||||
cache.asks[x] = PriceSize::NO_MARKET;
|
||||
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) {
|
||||
if (_booktype == CMEOptions::BOOKTYPE_DIRECT) {
|
||||
onBookUpdate(sec, book);
|
||||
}
|
||||
}
|
||||
|
||||
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
|
||||
if (_booktype == CMEOptions::BOOKTYPE_CONSOLIDATED) {
|
||||
onBookUpdate(sec, book);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Trades
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include "lib/utils/bts_no_copy.h"
|
||||
#include "lib/md/coconut2.h"
|
||||
#include "CMESecurityIndex.h"
|
||||
#include "server/sparkd/tracing/bts_trace.h"
|
||||
|
||||
#include "onixs.h"
|
||||
#include <tbb/concurrent_queue.h>
|
||||
@@ -13,13 +14,23 @@
|
||||
namespace bts {
|
||||
namespace md {
|
||||
|
||||
inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
|
||||
PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
|
||||
return result;
|
||||
}
|
||||
|
||||
class MDP3Listener :
|
||||
public onixs::HandlerListener,
|
||||
public onixs::MarketDataListener,
|
||||
public onixs::SecurityListener
|
||||
{
|
||||
public:
|
||||
MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector<MDProducerChannel>);
|
||||
MDP3Listener(CMESecurityIndex& index,
|
||||
MDProducer&,
|
||||
SecurityIDRecorder&,
|
||||
co2::SettlementRecorder&,
|
||||
const std::vector<MDProducerChannel>,
|
||||
CMEOptions::BookType);
|
||||
virtual ~MDP3Listener();
|
||||
|
||||
//
|
||||
@@ -50,10 +61,52 @@ public:
|
||||
//
|
||||
// Book
|
||||
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override;
|
||||
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
|
||||
void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
|
||||
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ImpliedBook&) override;
|
||||
void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override;
|
||||
|
||||
template <class TBook>
|
||||
void onBookUpdate(const onixs::Security& sec, const TBook& book) {
|
||||
tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", 0,_latest_sequence);
|
||||
|
||||
auto data = _index.get_security(sec.id());
|
||||
if (data == nullptr) return;
|
||||
bts_assert(data->id == (uint32_t)sec.id());
|
||||
auto display_factor = data->handle.security().display_factor;
|
||||
|
||||
// Put the data into the cache. Flush at the end of the packet.
|
||||
auto key = data->handle.key();
|
||||
auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
|
||||
return key == entry.handle.key();
|
||||
});
|
||||
bool found = find_iter != _book_updates.end();
|
||||
if (!found) {
|
||||
_book_updates.emplace_back(data->handle);
|
||||
}
|
||||
BookData& cache = found ? *find_iter : _book_updates.back();
|
||||
|
||||
// Overwrite all entries every time
|
||||
size_t bids_size = book.bids().size();
|
||||
size_t asks_size = book.offers().size();
|
||||
if (!data->handle.has_depth()) {
|
||||
bids_size = std::min(bids_size, size_t { 1 });
|
||||
asks_size = std::min(asks_size, size_t { 1 });
|
||||
}
|
||||
for (size_t x = 0; x < bids_size; ++x) {
|
||||
cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
|
||||
}
|
||||
for (size_t x = 0; x < asks_size; ++x) {
|
||||
cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
|
||||
}
|
||||
// NaN out the remaining depth
|
||||
for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
|
||||
cache.bids[x] = PriceSize::NO_MARKET;
|
||||
}
|
||||
for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
|
||||
cache.asks[x] = PriceSize::NO_MARKET;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Trades
|
||||
void onTrade(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
|
||||
@@ -124,6 +177,7 @@ private:
|
||||
std::vector<std::string> _symbol_filter_storage;
|
||||
std::set<onixs::StrRef> _symbol_filter;
|
||||
std::vector<MDProducerChannel> _channels;
|
||||
CMEOptions::BookType _booktype;
|
||||
std::atomic<int> _concurrency_monitor;
|
||||
|
||||
struct BookData {
|
||||
|
||||
@@ -213,7 +213,9 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(const string& channel_id) {
|
||||
settings.feeds().connectivityConfigurationFile(config_path());
|
||||
|
||||
// Market data options
|
||||
settings.bookManagement().consolidatedBooks().maintain(true);
|
||||
if (options.book_type() == CMEOptions::BOOKTYPE_CONSOLIDATED) {
|
||||
settings.bookManagement().consolidatedBooks().maintain(true);
|
||||
}
|
||||
|
||||
// 10k default is *far* too low
|
||||
settings.recoveryQueueMaxSize(options.recovery_queue_size());
|
||||
@@ -334,12 +336,13 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(onixs::Handler& handler, con
|
||||
producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
|
||||
}
|
||||
|
||||
unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels));
|
||||
const auto& options = channel_settings_for(channel.id);
|
||||
|
||||
unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels, options.book_type()));
|
||||
for (auto& symbol : channel.symbols) {
|
||||
listener->add_filter_symbol(symbol.symbol());
|
||||
}
|
||||
|
||||
const auto& options = channel_settings_for(channel.id);
|
||||
if (options.load_security_db()) {
|
||||
// Load the list of known securities and add them to the index for fast startup
|
||||
db::load_securities(channel.symbols, [&](const api::Security& security) {
|
||||
|
||||
Reference in new issue
Block a user