SPARK-2430 prioritize draining incremental queue over publishing shapshots
This commit is contained in:
1 parent
3789d73593
commit
bee1df45ad
4 files changed
+128
-51
No files matched your search
+120
-43
@@ -1,15 +1,21 @@
|
||||
#include "MBOProcessor.h"
|
||||
#include "lib/utils/Dispatcher.h"
|
||||
#include "lib/utils/ConcurrentPool.h"
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/config/protos/toggles.pb.h"
|
||||
#include "lib/utils/thread_manager.h"
|
||||
|
||||
#include <map>
|
||||
#include <functional>
|
||||
#include <vector>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
#include <atomic>
|
||||
|
||||
#ifndef _WIN32
|
||||
#include <google/btree/btree_set.h>
|
||||
#endif
|
||||
#include <tbb/concurrent_hash_map.h>
|
||||
#include <tbb/concurrent_queue.h>
|
||||
|
||||
|
||||
namespace bts {
|
||||
@@ -57,34 +63,126 @@ void print_pool_stats(const utils::ConcurrentPool<MBOSnapshot>& pool) {
|
||||
}
|
||||
|
||||
typedef tbb::concurrent_hash_map<SecurityKey, MBOSnapshot*, SecurityKeyHashCompare> MapType;
|
||||
struct Incremental {
|
||||
MBOWorkingState* state;
|
||||
inc::Update update;
|
||||
};
|
||||
struct MBOProcessorImpl {
|
||||
utils::ConcurrentPool<MBOSnapshot> pool;
|
||||
tbb::concurrent_queue<Incremental> incremental_input;
|
||||
MapType snapshots;
|
||||
utils::Dispatcher& dispatcher;
|
||||
utils::DispatcherTimer timer;
|
||||
MBOProcessorImpl(utils::Dispatcher& d) :
|
||||
bool running;
|
||||
utils::TimeStamp last_pool_print;
|
||||
std::thread thread;
|
||||
|
||||
MBOProcessorImpl() :
|
||||
pool(2048,512),
|
||||
snapshots(),
|
||||
dispatcher(d),
|
||||
timer(d, 600*1000, [&]() { print_pool_stats(pool); })
|
||||
{
|
||||
if (config::get<config::toggles::MBOStats>().enabled()) {
|
||||
timer.start();
|
||||
}
|
||||
running(true),
|
||||
last_pool_print(utils::TimeStamp::invalid()),
|
||||
thread(threading::make_thread("MBO", [=]() { process_incremental(); }))
|
||||
{
|
||||
if (config::get<config::toggles::MBOStats>().enabled()) {
|
||||
last_pool_print = utils::TimeStamp::now();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
~MBOProcessorImpl() {
|
||||
running = false;
|
||||
thread.join();
|
||||
}
|
||||
|
||||
void publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
|
||||
bts_assert(*snapshot != nullptr);
|
||||
MapType::accessor accessor;
|
||||
snapshots.insert(accessor, key);
|
||||
std::swap(accessor->second, *snapshot);
|
||||
if (*snapshot != nullptr) {
|
||||
// Note: relase() must happen inside accessor lifetime to ensure correctness
|
||||
pool.release(snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
void publish_incremental(MBOWorkingState& state) {
|
||||
// determine if we need to publish
|
||||
auto key = SecurityKey(state.handle.key());
|
||||
if (key.value() != invalid_security_key) {
|
||||
auto snap = pool.allocate();
|
||||
snap->bids.reserve(state.bid.size());
|
||||
snap->asks.reserve(state.ask.size());
|
||||
copy_to_snapshot(state, snap);
|
||||
publish_snapshot(key, &snap);
|
||||
}
|
||||
}
|
||||
|
||||
void process_incremental() {
|
||||
#ifndef _WIN32
|
||||
// drain queue and update state
|
||||
// insert changed state pointers into a set
|
||||
// after draining publish the set in small batches
|
||||
// re-check input queue regularly
|
||||
|
||||
btree::btree_set<MBOWorkingState*> dirty;
|
||||
std::vector<MBOWorkingState*> clean;
|
||||
|
||||
while(running) {
|
||||
Incremental temp;
|
||||
|
||||
// Drain entire input queue quickly
|
||||
while(incremental_input.try_pop(temp)) {
|
||||
bts_assert_d(temp.state != nullptr);
|
||||
if (temp.state != nullptr) {
|
||||
apply_incremental(*temp.state, temp.update);
|
||||
dirty.insert(temp.state);
|
||||
}
|
||||
}
|
||||
|
||||
// publish a small batch of snapshots (then check queue again)
|
||||
for (auto iter = dirty.begin(); iter != dirty.end(); ++iter) {
|
||||
publish_incremental(**iter);
|
||||
clean.push_back(*iter);
|
||||
if (clean.size() >= 128) break;
|
||||
}
|
||||
|
||||
// Remove 'clean' from set
|
||||
for (const auto& iter : clean) {
|
||||
dirty.erase(iter);
|
||||
}
|
||||
clean.clear();
|
||||
|
||||
// If the queue is still empty sleep briefly
|
||||
if (incremental_input.empty()) {
|
||||
if (last_pool_print != TimeStamp::invalid()) {
|
||||
auto now = TimeStamp::now();
|
||||
if ((now - last_pool_print).seconds() > 600) {
|
||||
last_pool_print = now;
|
||||
print_pool_stats(pool);
|
||||
}
|
||||
}
|
||||
std::this_thread::sleep_for(
|
||||
std::chrono::milliseconds(1)
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
//
|
||||
// Processor
|
||||
|
||||
MBOProcessor::MBOProcessor(utils::Dispatcher* d) :
|
||||
_me(std::make_shared<MBOProcessorImpl>(d == nullptr ? utils::Dispatcher::background() : *d)),
|
||||
_producer(_me),
|
||||
_consumer(_me)
|
||||
{}
|
||||
MBOProcessor::MBOProcessor() :
|
||||
_me(std::make_shared<MBOProcessorImpl>()),
|
||||
_writer(_me),
|
||||
_reader(_me)
|
||||
{
|
||||
}
|
||||
|
||||
MBOWriter& MBOProcessor::writer() { return _producer;}
|
||||
MBOReader& MBOProcessor::reader() { return _consumer; }
|
||||
MBOWriter& MBOProcessor::writer() { return _writer;}
|
||||
MBOReader& MBOProcessor::reader() { return _reader; }
|
||||
|
||||
//
|
||||
// Writer
|
||||
@@ -98,18 +196,11 @@ void MBOWriter::release_snapshot(MBOSnapshot** s) {
|
||||
_me->pool.release(s);
|
||||
}
|
||||
void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
|
||||
bts_assert(*snapshot != nullptr);
|
||||
MapType::accessor accessor;
|
||||
_me->snapshots.insert(accessor, key);
|
||||
std::swap(accessor->second, *snapshot);
|
||||
if (*snapshot != nullptr) {
|
||||
// Note: relase() must happen inside accessor lifetime to ensure correctness
|
||||
_me->pool.release(snapshot);
|
||||
}
|
||||
_me->publish_snapshot(key,snapshot);
|
||||
}
|
||||
|
||||
#if 0
|
||||
bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) {
|
||||
// bts_assert_msg(false, "TO USE THIS VALIDATION YOU MUST MAKE enqueue_incremental SYNCRONOUS");
|
||||
#define REQUIRE(a,b,y) \
|
||||
if ((a) != (b)) { \
|
||||
LOG(md,Error, "Validation error exch:{} sec:{} {}!={} other:{}", exchid, key.value(), a, b, y); \
|
||||
@@ -137,25 +228,11 @@ bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot*
|
||||
#undef REQUIRE
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) {
|
||||
LOG(md,Diag, "sec:{} : {}", state.handle.key(), update);
|
||||
|
||||
_me->dispatcher.post([this,&state,update]() {
|
||||
// fast part
|
||||
apply_incremental(state,update);
|
||||
// slow part
|
||||
publish_incremental(state);
|
||||
});
|
||||
}
|
||||
void MBOWriter::publish_incremental(MBOWorkingState& state) {
|
||||
// determine if we need to publish
|
||||
auto key = SecurityKey(state.handle.key());
|
||||
if (key.value() != invalid_security_key) {
|
||||
auto snap = allocate_snapshot(state.bid.size(), state.ask.size());
|
||||
copy_to_snapshot(state, snap);
|
||||
publish_snapshot(key, &snap);
|
||||
}
|
||||
_me->incremental_input.push({&state, update});
|
||||
}
|
||||
|
||||
void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) {
|
||||
@@ -267,4 +344,4 @@ ostream& operator<<(ostream& os, const inc::Update& update) {
|
||||
return os;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@
|
||||
#include "lib/domain_model/SecurityKey.h"
|
||||
#include "lib/domain_model/Security.h"
|
||||
#include "lib/utils/TimeStamp.h"
|
||||
#include "lib/utils/Dispatcher.h"
|
||||
#include "IntPrice.h"
|
||||
#include "ExchangeOrderKey.h"
|
||||
#include "MDProducerHandle.h"
|
||||
@@ -152,8 +151,6 @@ class MBOWriter {
|
||||
|
||||
// Enqueue the update to the mbo work queue.
|
||||
void enqueue_incremental(MBOWorkingState&, const inc::Update&);
|
||||
// Update working state and publish directly.
|
||||
void publish_incremental(MBOWorkingState&);
|
||||
};
|
||||
|
||||
class MBOReader {
|
||||
@@ -169,11 +166,11 @@ class MBOReader {
|
||||
|
||||
class MBOProcessor {
|
||||
std::shared_ptr<MBOProcessorImpl> _me;
|
||||
MBOWriter _producer;
|
||||
MBOReader _consumer;
|
||||
MBOWriter _writer;
|
||||
MBOReader _reader;
|
||||
|
||||
public:
|
||||
MBOProcessor(utils::Dispatcher* _=nullptr);
|
||||
MBOProcessor();
|
||||
MBOWriter& writer();
|
||||
MBOReader& reader();
|
||||
};
|
||||
@@ -184,3 +181,4 @@ public:
|
||||
namespace std {
|
||||
std::ostream& operator<<(std::ostream& os, const bts::md::inc::Update&);
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ int Run::execute(std::vector<std::string> const &) {
|
||||
persistence::PersistenceServiceServer settings_saver(sc, "settings");
|
||||
user::AuthServiceServer login_server(sc);
|
||||
|
||||
md::MBOProcessor mbo(&utils::Dispatcher::get_named("MBO"));
|
||||
md::MBOProcessor mbo;
|
||||
auto* mbowriter = config::get<config::toggles::PlacementInQueue>().enabled() ? &mbo.writer() : nullptr;
|
||||
md::run_market_datapublishers(mbowriter);
|
||||
|
||||
|
||||
@@ -120,7 +120,8 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con
|
||||
onBookUpdate<COCONUT_MAX_IMPLIED_DEPTH>(sec, book, book_type_implied);
|
||||
}
|
||||
|
||||
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::MboBook& book) {
|
||||
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook& ) {
|
||||
#if 0
|
||||
if (_mbo) {
|
||||
auto data = _index.get_security(sec.id());
|
||||
if (data == nullptr) return;
|
||||
@@ -136,6 +137,7 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con
|
||||
}
|
||||
_mbo->validate_snapshot(sec.id(), SecurityKey(key), snapshot);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
Reference in new issue
Block a user