diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 88cc1f3f2..926d7903a 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -1,15 +1,21 @@ #include "MBOProcessor.h" -#include "lib/utils/Dispatcher.h" #include "lib/utils/ConcurrentPool.h" #include "lib/config/config.h" #include "lib/config/protos/toggles.pb.h" +#include "lib/utils/thread_manager.h" #include #include #include #include +#include +#include +#ifndef _WIN32 +#include +#endif #include +#include namespace bts { @@ -57,34 +63,126 @@ void print_pool_stats(const utils::ConcurrentPool& pool) { } typedef tbb::concurrent_hash_map MapType; +struct Incremental { + MBOWorkingState* state; + inc::Update update; +}; struct MBOProcessorImpl { utils::ConcurrentPool pool; + tbb::concurrent_queue incremental_input; MapType snapshots; - utils::Dispatcher& dispatcher; - utils::DispatcherTimer timer; - MBOProcessorImpl(utils::Dispatcher& d) : + bool running; + utils::TimeStamp last_pool_print; + std::thread thread; + + MBOProcessorImpl() : pool(2048,512), snapshots(), - dispatcher(d), - timer(d, 600*1000, [&]() { print_pool_stats(pool); }) - { - if (config::get().enabled()) { - timer.start(); - } + running(true), + last_pool_print(utils::TimeStamp::invalid()), + thread(threading::make_thread("MBO", [=]() { process_incremental(); })) + { + if (config::get().enabled()) { + last_pool_print = utils::TimeStamp::now(); } + + } + + ~MBOProcessorImpl() { + running = false; + thread.join(); + } + + void publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { + bts_assert(*snapshot != nullptr); + MapType::accessor accessor; + snapshots.insert(accessor, key); + std::swap(accessor->second, *snapshot); + if (*snapshot != nullptr) { + // Note: relase() must happen inside accessor lifetime to ensure correctness + pool.release(snapshot); + } + } + + void publish_incremental(MBOWorkingState& state) { + // determine if we need to publish + auto key = SecurityKey(state.handle.key()); + if (key.value() != invalid_security_key) { + auto snap = pool.allocate(); + snap->bids.reserve(state.bid.size()); + snap->asks.reserve(state.ask.size()); + copy_to_snapshot(state, snap); + publish_snapshot(key, &snap); + } + } + + void process_incremental() { + #ifndef _WIN32 + // drain queue and update state + // insert changed state pointers into a set + // after draining publish the set in small batches + // re-check input queue regularly + + btree::btree_set dirty; + std::vector clean; + + while(running) { + Incremental temp; + + // Drain entire input queue quickly + while(incremental_input.try_pop(temp)) { + bts_assert_d(temp.state != nullptr); + if (temp.state != nullptr) { + apply_incremental(*temp.state, temp.update); + dirty.insert(temp.state); + } + } + + // publish a small batch of snapshots (then check queue again) + for (auto iter = dirty.begin(); iter != dirty.end(); ++iter) { + publish_incremental(**iter); + clean.push_back(*iter); + if (clean.size() >= 128) break; + } + + // Remove 'clean' from set + for (const auto& iter : clean) { + dirty.erase(iter); + } + clean.clear(); + + // If the queue is still empty sleep briefly + if (incremental_input.empty()) { + if (last_pool_print != TimeStamp::invalid()) { + auto now = TimeStamp::now(); + if ((now - last_pool_print).seconds() > 600) { + last_pool_print = now; + print_pool_stats(pool); + } + } + std::this_thread::sleep_for( + std::chrono::milliseconds(1) + ); + } + + } + #endif + } + }; // // Processor -MBOProcessor::MBOProcessor(utils::Dispatcher* d) : - _me(std::make_shared(d == nullptr ? utils::Dispatcher::background() : *d)), - _producer(_me), - _consumer(_me) -{} +MBOProcessor::MBOProcessor() : + _me(std::make_shared()), + _writer(_me), + _reader(_me) +{ +} -MBOWriter& MBOProcessor::writer() { return _producer;} -MBOReader& MBOProcessor::reader() { return _consumer; } +MBOWriter& MBOProcessor::writer() { return _writer;} +MBOReader& MBOProcessor::reader() { return _reader; } // // Writer @@ -98,18 +196,11 @@ void MBOWriter::release_snapshot(MBOSnapshot** s) { _me->pool.release(s); } void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { - bts_assert(*snapshot != nullptr); - MapType::accessor accessor; - _me->snapshots.insert(accessor, key); - std::swap(accessor->second, *snapshot); - if (*snapshot != nullptr) { - // Note: relase() must happen inside accessor lifetime to ensure correctness - _me->pool.release(snapshot); - } + _me->publish_snapshot(key,snapshot); } +#if 0 bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) { - // bts_assert_msg(false, "TO USE THIS VALIDATION YOU MUST MAKE enqueue_incremental SYNCRONOUS"); #define REQUIRE(a,b,y) \ if ((a) != (b)) { \ LOG(md,Error, "Validation error exch:{} sec:{} {}!={} other:{}", exchid, key.value(), a, b, y); \ @@ -137,25 +228,11 @@ bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* #undef REQUIRE return true; } +#endif void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { LOG(md,Diag, "sec:{} : {}", state.handle.key(), update); - - _me->dispatcher.post([this,&state,update]() { - // fast part - apply_incremental(state,update); - // slow part - publish_incremental(state); - }); -} -void MBOWriter::publish_incremental(MBOWorkingState& state) { - // determine if we need to publish - auto key = SecurityKey(state.handle.key()); - if (key.value() != invalid_security_key) { - auto snap = allocate_snapshot(state.bid.size(), state.ask.size()); - copy_to_snapshot(state, snap); - publish_snapshot(key, &snap); - } + _me->incremental_input.push({&state, update}); } void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { @@ -267,4 +344,4 @@ ostream& operator<<(ostream& os, const inc::Update& update) { return os; } -} \ No newline at end of file +} diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index f3a92b59b..de36b3931 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -2,7 +2,6 @@ #include "lib/domain_model/SecurityKey.h" #include "lib/domain_model/Security.h" #include "lib/utils/TimeStamp.h" -#include "lib/utils/Dispatcher.h" #include "IntPrice.h" #include "ExchangeOrderKey.h" #include "MDProducerHandle.h" @@ -152,8 +151,6 @@ class MBOWriter { // Enqueue the update to the mbo work queue. void enqueue_incremental(MBOWorkingState&, const inc::Update&); - // Update working state and publish directly. - void publish_incremental(MBOWorkingState&); }; class MBOReader { @@ -169,11 +166,11 @@ class MBOReader { class MBOProcessor { std::shared_ptr _me; - MBOWriter _producer; - MBOReader _consumer; + MBOWriter _writer; + MBOReader _reader; public: - MBOProcessor(utils::Dispatcher* _=nullptr); + MBOProcessor(); MBOWriter& writer(); MBOReader& reader(); }; @@ -184,3 +181,4 @@ public: namespace std { std::ostream& operator<<(std::ostream& os, const bts::md::inc::Update&); } + diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 547e04fc0..f1df545c4 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -60,7 +60,7 @@ int Run::execute(std::vector const &) { persistence::PersistenceServiceServer settings_saver(sc, "settings"); user::AuthServiceServer login_server(sc); - md::MBOProcessor mbo(&utils::Dispatcher::get_named("MBO")); + md::MBOProcessor mbo; auto* mbowriter = config::get().enabled() ? &mbo.writer() : nullptr; md::run_market_datapublishers(mbowriter); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index b06b185f7..7276310a3 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -120,7 +120,8 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con onBookUpdate(sec, book, book_type_implied); } -void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::MboBook& book) { +void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook& ) { + #if 0 if (_mbo) { auto data = _index.get_security(sec.id()); if (data == nullptr) return; @@ -136,6 +137,7 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con } _mbo->validate_snapshot(sec.id(), SecurityKey(key), snapshot); } + #endif } //