merge from prod

This commit is contained in:
Eric Harding committed 2018-06-28 11:28:08 -04:00
commit 802160abf1
5 files changed
+87 -53

No files matched your search

+5
View File
@@ -78,6 +78,10 @@ message CMEOptions
repeated uint32 affinity = 10;
optional uint32 udp_socket_buffer_size = 11 [default = 33554432]; //32mb
}
enum BookType {
BOOKTYPE_CONSOLIDATED = 0;
BOOKTYPE_DIRECT = 1;
}
message CMEChannel {
repeated string id = 2;
optional string feed_engine = 3 [default = "default"];
@@ -100,6 +104,7 @@ message CMEOptions
optional bool join_snapshot_on_gap = 32 [default = false];
optional bool load_security_db = 33 [default = true];
optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED];
}
optional string onixs_license = 1 [default = "{default}"];
+6
View File
@@ -8,6 +8,7 @@
#include "lib/logging/logging.h"
#include "lib/os/os.h"
#include "lib/quickbus/quickbus.h"
#include "lib/utils/Dispatcher.h"
#include "lib/md/SyntheticSecurityService.h"
#include "server/sparkd/exchange_sim/ExchangeSim.h"
@@ -55,6 +56,11 @@ int Run::execute(std::vector<std::string> const &) {
support::ErrorReportServer error_report_service(sc);
admin::AdminServiceServer admin_service(sc);
metrics::ShellLatencyServer shell_latency_service(sc, stats, utils::Dispatcher::background());
auto bg_health = std::make_shared<metrics::IntCounterMetric>("spark_background_thread");
utils::DispatcherTimer timer(utils::Dispatcher::background(), 5000, [=]() { bg_health->increment(); });
timer.start();
stats.publish(bg_health);
std::thread([]() {
os::set_thread_name("ExchangeSim");
@@ -1,8 +1,8 @@
#include "MDP3Listener.h"
#include "lib/utils/datetime.h"
#include <vector>
#include "server/sparkd/tracing/bts_trace.h"
#include "lib/utils/helper_macros.h"
#include "lib/config/protos/MarketData.pb.h"
using namespace std;
using namespace bts::co2;
@@ -84,7 +84,8 @@ MDP3Listener::MDP3Listener(
MDProducer& producer,
SecurityIDRecorder& recorder,
co2::SettlementRecorder& ,
std::vector<MDProducerChannel> channels) :
const std::vector<MDProducerChannel> channels,
CMEOptions::BookType bt) :
_index(index),
_producer(producer),
_packet_arrived(),
@@ -94,6 +95,7 @@ MDP3Listener::MDP3Listener(
_symbol_filter_storage(),
_symbol_filter(),
_channels(channels),
_booktype(bt),
_concurrency_monitor(0),
_book_updates()
{
@@ -185,11 +187,6 @@ double double_of_price(const Price& price, int8_t display_factor) {
return (price.mantissa() * pow10(price.exponent())) * pow10(display_factor);
}
inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
return result;
}
inline datetime::LLDateTime ll_of_timestamp(const onixs::Timestamp& ts) {
return datetime::ll_from_parts(ts.year(), (int)ts.month(), ts.day(), ts.hour(), ts.minute(), ts.second(), ts.millisecond());
}
@@ -337,49 +334,18 @@ void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security&, const o
_index.add_spread_legs(fsec.numeric_id, leg_exchange_ids);
}
//
// Book
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", _latest_sequence, (uint16_t) _latest_sequence, 0);
auto data = _index.get_security(sec.id());
if (data == nullptr) return;
bts_assert(data->id == (uint32_t)sec.id());
auto display_factor = data->handle.security().display_factor;
// Put the data into the cache. Flush at the end of the packet.
auto key = data->handle.key();
auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
return key == entry.handle.key();
});
bool found = find_iter != _book_updates.end();
if (!found) {
_book_updates.emplace_back(data->handle);
}
BookData& cache = found ? *find_iter : _book_updates.back();
// Overwrite all entries every time
size_t bids_size = book.bids().size();
size_t asks_size = book.offers().size();
if (!data->handle.has_depth()) {
bids_size = min(bids_size, size_t { 1 });
asks_size = min(asks_size, size_t { 1 });
}
for (size_t x = 0; x < bids_size; ++x) {
cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
}
for (size_t x = 0; x < asks_size; ++x) {
cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
}
// NaN out the remaining depth
for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
cache.bids[x] = PriceSize::NO_MARKET;
}
for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
cache.asks[x] = PriceSize::NO_MARKET;
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) {
if (_booktype == CMEOptions::BOOKTYPE_DIRECT) {
onBookUpdate(sec, book);
}
}
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
if (_booktype == CMEOptions::BOOKTYPE_CONSOLIDATED) {
onBookUpdate(sec, book);
}
}
//
// Trades
@@ -3,6 +3,7 @@
#include "lib/utils/bts_no_copy.h"
#include "lib/md/coconut2.h"
#include "CMESecurityIndex.h"
#include "server/sparkd/tracing/bts_trace.h"
#include "onixs.h"
#include <tbb/concurrent_queue.h>
@@ -13,13 +14,23 @@
namespace bts {
namespace md {
inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
return result;
}
class MDP3Listener :
public onixs::HandlerListener,
public onixs::MarketDataListener,
public onixs::SecurityListener
{
public:
MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector<MDProducerChannel>);
MDP3Listener(CMESecurityIndex& index,
MDProducer&,
SecurityIDRecorder&,
co2::SettlementRecorder&,
const std::vector<MDProducerChannel>,
CMEOptions::BookType);
virtual ~MDP3Listener();
//
@@ -50,10 +61,52 @@ public:
//
// Book
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override;
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ImpliedBook&) override;
void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override;
template <class TBook>
void onBookUpdate(const onixs::Security& sec, const TBook& book) {
tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", 0,_latest_sequence);
auto data = _index.get_security(sec.id());
if (data == nullptr) return;
bts_assert(data->id == (uint32_t)sec.id());
auto display_factor = data->handle.security().display_factor;
// Put the data into the cache. Flush at the end of the packet.
auto key = data->handle.key();
auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
return key == entry.handle.key();
});
bool found = find_iter != _book_updates.end();
if (!found) {
_book_updates.emplace_back(data->handle);
}
BookData& cache = found ? *find_iter : _book_updates.back();
// Overwrite all entries every time
size_t bids_size = book.bids().size();
size_t asks_size = book.offers().size();
if (!data->handle.has_depth()) {
bids_size = std::min(bids_size, size_t { 1 });
asks_size = std::min(asks_size, size_t { 1 });
}
for (size_t x = 0; x < bids_size; ++x) {
cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
}
for (size_t x = 0; x < asks_size; ++x) {
cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
}
// NaN out the remaining depth
for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
cache.bids[x] = PriceSize::NO_MARKET;
}
for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
cache.asks[x] = PriceSize::NO_MARKET;
}
}
//
// Trades
void onTrade(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
@@ -124,6 +177,7 @@ private:
std::vector<std::string> _symbol_filter_storage;
std::set<onixs::StrRef> _symbol_filter;
std::vector<MDProducerChannel> _channels;
CMEOptions::BookType _booktype;
std::atomic<int> _concurrency_monitor;
struct BookData {
@@ -213,7 +213,9 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(const string& channel_id) {
settings.feeds().connectivityConfigurationFile(config_path());
// Market data options
settings.bookManagement().consolidatedBooks().maintain(true);
if (options.book_type() == CMEOptions::BOOKTYPE_CONSOLIDATED) {
settings.bookManagement().consolidatedBooks().maintain(true);
}
// 10k default is *far* too low
settings.recoveryQueueMaxSize(options.recovery_queue_size());
@@ -334,12 +336,13 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(onixs::Handler& handler, con
producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
}
unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels));
const auto& options = channel_settings_for(channel.id);
unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels, options.book_type()));
for (auto& symbol : channel.symbols) {
listener->add_filter_symbol(symbol.symbol());
}
const auto& options = channel_settings_for(channel.id);
if (options.load_security_db()) {
// Load the list of known securities and add them to the index for fast startup
db::load_securities(channel.symbols, [&](const api::Security& security) {