SPARK-2430 prioritize draining incremental queue over publishing shapshots

This commit is contained in:
Eric Harding committed 2019-05-14 11:52:51 -04:00
1 parent 3789d73593
commit bee1df45ad
4 files changed
+128 -51

No files matched your search

+120 -43
View File
@@ -1,15 +1,21 @@
#include "MBOProcessor.h"
#include "lib/utils/Dispatcher.h"
#include "lib/utils/ConcurrentPool.h"
#include "lib/config/config.h"
#include "lib/config/protos/toggles.pb.h"
#include "lib/utils/thread_manager.h"
#include <map>
#include <functional>
#include <vector>
#include <sstream>
#include <thread>
#include <atomic>
#ifndef _WIN32
#include <google/btree/btree_set.h>
#endif
#include <tbb/concurrent_hash_map.h>
#include <tbb/concurrent_queue.h>
namespace bts {
@@ -57,34 +63,126 @@ void print_pool_stats(const utils::ConcurrentPool<MBOSnapshot>& pool) {
}
typedef tbb::concurrent_hash_map<SecurityKey, MBOSnapshot*, SecurityKeyHashCompare> MapType;
struct Incremental {
MBOWorkingState* state;
inc::Update update;
};
struct MBOProcessorImpl {
utils::ConcurrentPool<MBOSnapshot> pool;
tbb::concurrent_queue<Incremental> incremental_input;
MapType snapshots;
utils::Dispatcher& dispatcher;
utils::DispatcherTimer timer;
MBOProcessorImpl(utils::Dispatcher& d) :
bool running;
utils::TimeStamp last_pool_print;
std::thread thread;
MBOProcessorImpl() :
pool(2048,512),
snapshots(),
dispatcher(d),
timer(d, 600*1000, [&]() { print_pool_stats(pool); })
{
if (config::get<config::toggles::MBOStats>().enabled()) {
timer.start();
}
running(true),
last_pool_print(utils::TimeStamp::invalid()),
thread(threading::make_thread("MBO", [=]() { process_incremental(); }))
{
if (config::get<config::toggles::MBOStats>().enabled()) {
last_pool_print = utils::TimeStamp::now();
}
}
~MBOProcessorImpl() {
running = false;
thread.join();
}
void publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
bts_assert(*snapshot != nullptr);
MapType::accessor accessor;
snapshots.insert(accessor, key);
std::swap(accessor->second, *snapshot);
if (*snapshot != nullptr) {
// Note: relase() must happen inside accessor lifetime to ensure correctness
pool.release(snapshot);
}
}
void publish_incremental(MBOWorkingState& state) {
// determine if we need to publish
auto key = SecurityKey(state.handle.key());
if (key.value() != invalid_security_key) {
auto snap = pool.allocate();
snap->bids.reserve(state.bid.size());
snap->asks.reserve(state.ask.size());
copy_to_snapshot(state, snap);
publish_snapshot(key, &snap);
}
}
void process_incremental() {
#ifndef _WIN32
// drain queue and update state
// insert changed state pointers into a set
// after draining publish the set in small batches
// re-check input queue regularly
btree::btree_set<MBOWorkingState*> dirty;
std::vector<MBOWorkingState*> clean;
while(running) {
Incremental temp;
// Drain entire input queue quickly
while(incremental_input.try_pop(temp)) {
bts_assert_d(temp.state != nullptr);
if (temp.state != nullptr) {
apply_incremental(*temp.state, temp.update);
dirty.insert(temp.state);
}
}
// publish a small batch of snapshots (then check queue again)
for (auto iter = dirty.begin(); iter != dirty.end(); ++iter) {
publish_incremental(**iter);
clean.push_back(*iter);
if (clean.size() >= 128) break;
}
// Remove 'clean' from set
for (const auto& iter : clean) {
dirty.erase(iter);
}
clean.clear();
// If the queue is still empty sleep briefly
if (incremental_input.empty()) {
if (last_pool_print != TimeStamp::invalid()) {
auto now = TimeStamp::now();
if ((now - last_pool_print).seconds() > 600) {
last_pool_print = now;
print_pool_stats(pool);
}
}
std::this_thread::sleep_for(
std::chrono::milliseconds(1)
);
}
}
#endif
}
};
//
// Processor
MBOProcessor::MBOProcessor(utils::Dispatcher* d) :
_me(std::make_shared<MBOProcessorImpl>(d == nullptr ? utils::Dispatcher::background() : *d)),
_producer(_me),
_consumer(_me)
{}
MBOProcessor::MBOProcessor() :
_me(std::make_shared<MBOProcessorImpl>()),
_writer(_me),
_reader(_me)
{
}
MBOWriter& MBOProcessor::writer() { return _producer;}
MBOReader& MBOProcessor::reader() { return _consumer; }
MBOWriter& MBOProcessor::writer() { return _writer;}
MBOReader& MBOProcessor::reader() { return _reader; }
//
// Writer
@@ -98,18 +196,11 @@ void MBOWriter::release_snapshot(MBOSnapshot** s) {
_me->pool.release(s);
}
void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
bts_assert(*snapshot != nullptr);
MapType::accessor accessor;
_me->snapshots.insert(accessor, key);
std::swap(accessor->second, *snapshot);
if (*snapshot != nullptr) {
// Note: relase() must happen inside accessor lifetime to ensure correctness
_me->pool.release(snapshot);
}
_me->publish_snapshot(key,snapshot);
}
#if 0
bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) {
// bts_assert_msg(false, "TO USE THIS VALIDATION YOU MUST MAKE enqueue_incremental SYNCRONOUS");
#define REQUIRE(a,b,y) \
if ((a) != (b)) { \
LOG(md,Error, "Validation error exch:{} sec:{} {}!={} other:{}", exchid, key.value(), a, b, y); \
@@ -137,25 +228,11 @@ bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot*
#undef REQUIRE
return true;
}
#endif
void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) {
LOG(md,Diag, "sec:{} : {}", state.handle.key(), update);
_me->dispatcher.post([this,&state,update]() {
// fast part
apply_incremental(state,update);
// slow part
publish_incremental(state);
});
}
void MBOWriter::publish_incremental(MBOWorkingState& state) {
// determine if we need to publish
auto key = SecurityKey(state.handle.key());
if (key.value() != invalid_security_key) {
auto snap = allocate_snapshot(state.bid.size(), state.ask.size());
copy_to_snapshot(state, snap);
publish_snapshot(key, &snap);
}
_me->incremental_input.push({&state, update});
}
void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) {
@@ -267,4 +344,4 @@ ostream& operator<<(ostream& os, const inc::Update& update) {
return os;
}
}
}
+4 -6
View File
@@ -2,7 +2,6 @@
#include "lib/domain_model/SecurityKey.h"
#include "lib/domain_model/Security.h"
#include "lib/utils/TimeStamp.h"
#include "lib/utils/Dispatcher.h"
#include "IntPrice.h"
#include "ExchangeOrderKey.h"
#include "MDProducerHandle.h"
@@ -152,8 +151,6 @@ class MBOWriter {
// Enqueue the update to the mbo work queue.
void enqueue_incremental(MBOWorkingState&, const inc::Update&);
// Update working state and publish directly.
void publish_incremental(MBOWorkingState&);
};
class MBOReader {
@@ -169,11 +166,11 @@ class MBOReader {
class MBOProcessor {
std::shared_ptr<MBOProcessorImpl> _me;
MBOWriter _producer;
MBOReader _consumer;
MBOWriter _writer;
MBOReader _reader;
public:
MBOProcessor(utils::Dispatcher* _=nullptr);
MBOProcessor();
MBOWriter& writer();
MBOReader& reader();
};
@@ -184,3 +181,4 @@ public:
namespace std {
std::ostream& operator<<(std::ostream& os, const bts::md::inc::Update&);
}
+1 -1
View File
@@ -60,7 +60,7 @@ int Run::execute(std::vector<std::string> const &) {
persistence::PersistenceServiceServer settings_saver(sc, "settings");
user::AuthServiceServer login_server(sc);
md::MBOProcessor mbo(&utils::Dispatcher::get_named("MBO"));
md::MBOProcessor mbo;
auto* mbowriter = config::get<config::toggles::PlacementInQueue>().enabled() ? &mbo.writer() : nullptr;
md::run_market_datapublishers(mbowriter);
@@ -120,7 +120,8 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con
onBookUpdate<COCONUT_MAX_IMPLIED_DEPTH>(sec, book, book_type_implied);
}
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::MboBook& book) {
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook& ) {
#if 0
if (_mbo) {
auto data = _index.get_security(sec.id());
if (data == nullptr) return;
@@ -136,6 +137,7 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con
}
_mbo->validate_snapshot(sec.id(), SecurityKey(key), snapshot);
}
#endif
}
//