Files
spark/src/lib/md/MBOProcessor.cpp
T

348 lines
10 KiB
C++

#include "MBOProcessor.h"
#include "lib/utils/ConcurrentPool.h"
#include "lib/config/config.h"
#include "lib/config/protos/toggles.pb.h"
#include "lib/utils/thread_manager.h"
#include <map>
#include <functional>
#include <vector>
#include <sstream>
#include <thread>
#include <atomic>
#ifndef _WIN32
#include <google/btree/btree_set.h>
#endif
#include <tbb/concurrent_hash_map.h>
#include <tbb/concurrent_queue.h>
namespace bts {
namespace md {
struct SecurityKeyHashCompare {
static size_t hash(const SecurityKey& k) {
return k.value();
}
static bool equal(const SecurityKey& a, const SecurityKey& b) {
return a.value() == b.value();
}
};
void print_pool_stats(const utils::ConcurrentPool<MBOSnapshot>& pool) {
size_t count = 0;
std::vector<int> bid_sz,ask_sz;
pool.unsafe_iterate([&](const MBOSnapshot& s) {
if (s.bids.size() > 0 || s.asks.size() > 0) {
count++;
bid_sz.push_back(s.bids.size());
ask_sz.push_back(s.asks.size());
}
});
if (count == 0) return;
std::sort(bid_sz.begin(), bid_sz.end());
std::sort(ask_sz.begin(), ask_sz.end());
size_t bid_tot =0, ask_tot = 0;
for(size_t i : bid_sz) { bid_tot += i; }
for(size_t i : ask_sz) { ask_tot += i; }
std::stringstream bid_percentiles, ask_percentiles;
for (double x : { 0.5, 0.75, 0.9, 0.95, 0.99 }) {
size_t index = count * x;
bid_percentiles << bid_sz[index] << " ";
ask_percentiles << ask_sz[index] << " ";
}
bid_percentiles << bid_sz[count-1] << " ";
ask_percentiles << ask_sz[count-1] << " ";
LOG(md,Info,
"MBO statistics: count: {} - %[50,75,90,95,99,100] bid: [{}] - ask: [{}]",
count, bid_percentiles.str(), ask_percentiles.str());
}
typedef tbb::concurrent_hash_map<SecurityKey, MBOSnapshot*, SecurityKeyHashCompare> MapType;
struct Incremental {
MBOWorkingState* state;
inc::Update update;
};
struct MBOProcessorImpl {
utils::ConcurrentPool<MBOSnapshot> pool;
tbb::concurrent_queue<Incremental> incremental_input;
MapType snapshots;
bool running;
utils::TimeStamp last_pool_print;
std::thread thread;
MBOProcessorImpl() :
pool(2048,512),
snapshots(),
running(true),
last_pool_print(utils::TimeStamp::invalid()),
thread(threading::make_thread("MBO", [=]() { process_incremental(); }))
{
if (config::get<config::toggles::MBOStats>().enabled()) {
last_pool_print = utils::TimeStamp::now();
}
}
~MBOProcessorImpl() {
running = false;
thread.join();
}
void publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
bts_assert(*snapshot != nullptr);
MapType::accessor accessor;
snapshots.insert(accessor, key);
std::swap(accessor->second, *snapshot);
if (*snapshot != nullptr) {
// Note: relase() must happen inside accessor lifetime to ensure correctness
pool.release(snapshot);
}
}
void publish_incremental(MBOWorkingState& state) {
// determine if we need to publish
auto key = SecurityKey(state.handle.key());
if (key.value() != invalid_security_key) {
auto snap = pool.allocate();
snap->bids.reserve(state.bid.size());
snap->asks.reserve(state.ask.size());
copy_to_snapshot(state, snap);
publish_snapshot(key, &snap);
}
}
void process_incremental() {
#ifndef _WIN32
// drain queue and update state
// insert changed state pointers into a set
// after draining publish the set in small batches
// re-check input queue regularly
btree::btree_set<MBOWorkingState*> dirty;
std::vector<MBOWorkingState*> clean;
while(running) {
Incremental temp;
// Drain entire input queue quickly
while(incremental_input.try_pop(temp)) {
bts_assert_d(temp.state != nullptr);
if (temp.state != nullptr) {
apply_incremental(*temp.state, temp.update);
dirty.insert(temp.state);
}
}
// publish a small batch of snapshots (then check queue again)
for (auto iter = dirty.begin(); iter != dirty.end(); ++iter) {
publish_incremental(**iter);
clean.push_back(*iter);
if (clean.size() >= 128) break;
}
// Remove 'clean' from set
for (const auto& iter : clean) {
dirty.erase(iter);
}
clean.clear();
// If the queue is still empty sleep briefly
if (incremental_input.empty()) {
if (last_pool_print != TimeStamp::invalid()) {
auto now = TimeStamp::now();
if ((now - last_pool_print).seconds() > 600) {
last_pool_print = now;
print_pool_stats(pool);
}
}
std::this_thread::sleep_for(
std::chrono::milliseconds(1)
);
}
}
#endif
}
};
//
// Processor
MBOProcessor::MBOProcessor() :
_me(std::make_shared<MBOProcessorImpl>()),
_writer(_me),
_reader(_me)
{
}
MBOWriter& MBOProcessor::writer() { return _writer;}
MBOReader& MBOProcessor::reader() { return _reader; }
//
// Writer
MBOWriter::MBOWriter(std::shared_ptr<MBOProcessorImpl>& me): _me(me) { }
MBOWriter::~MBOWriter() { }
MBOSnapshot* MBOWriter::allocate_snapshot(size_t, size_t) {
return _me->pool.allocate();
}
void MBOWriter::release_snapshot(MBOSnapshot** s) {
_me->pool.release(s);
}
void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
_me->publish_snapshot(key,snapshot);
}
#if 0
bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) {
#define REQUIRE(a,b,y) \
if ((a) != (b)) { \
LOG(md,Error, "Validation error exch:{} sec:{} {}!={} other:{}", exchid, key.value(), a, b, y); \
return false; \
}
// retrieve current snapshot and check they are equal
_me->dispatcher.post([this,exchid,key,control]() {
MBOReader reader(_me);
auto snap = reader.acquire_snapshot(key);
if (snap) {
REQUIRE (snap->bids.size(), control->bids.size(), '!');
REQUIRE (snap->asks.size(), control->asks.size(), '!');
for (size_t x = 0; x < snap->bids.size(); ++x) {
REQUIRE (snap->bids[x].id, control->bids[x].id, x);
REQUIRE (snap->bids[x].size, control->bids[x].size, x);
}
for (size_t x = 0; x < snap->asks.size(); ++x) {
REQUIRE (snap->asks[x].id, control->asks[x].id, x);
REQUIRE (snap->asks[x].size, control->asks[x].size, x);
}
}
return true;
});
#undef REQUIRE
return true;
}
#endif
void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) {
LOG(md,Diag, "sec:{} : {}", state.handle.key(), update);
_me->incremental_input.push({&state, update});
}
void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) {
if (dest == nullptr) return;
// copy working state to snapshot
dest->updated = state.updated;
auto type_map = [](auto pair) {
return SnapshotOrder { pair.second.id, std::get<IntPrice>(pair.first), pair.second.size };
};
std::transform(state.bid.begin(), state.bid.end(), std::back_inserter(dest->bids), type_map);
std::transform(state.ask.begin(), state.ask.end(), std::back_inserter(dest->asks), type_map);
}
template <class C>
void apply_incremental(uint32_t security_key, C& collection, std::unordered_map<ExchangeOrderKey, MBOWorkingState::key_t>& index, const inc::Update& update) {
auto key = std::make_tuple(update.price, update.priority);
switch(update.action) {
case inc::UpdateAction::New:
collection.insert(std::make_pair(key, MBOWorkingState::Data { update.order_id, update.size }));
index[update.order_id] = key;
break;
case inc::UpdateAction::Change: {
// look up old key
auto r = index.find(update.order_id);
if (r != index.end()) {
auto oldkey = r->second;
collection.erase(oldkey);
index.erase(r);
}
else {
LOG(md,Warn, "MBO Update w/out Add for: sec:{} {}", security_key, update);
}
collection.insert(std::make_pair(key, MBOWorkingState::Data { update.order_id, update.size }));
index[update.order_id] = key;
}
break;
case inc::UpdateAction::Delete:
collection.erase(key);
index.erase(update.order_id);
break;
}
}
void apply_incremental(MBOWorkingState& state, const inc::Update& update) {
// apply the update to the working state
if (update.type == inc::UpdateType::Bid) {
apply_incremental(state.handle.key(), state.bid, state.orderid_index, update);
}
else {
apply_incremental(state.handle.key(),state.ask, state.orderid_index, update);
}
}
//
// Reader
MBOReader::MBOReader(std::shared_ptr<MBOProcessorImpl>& me): _me(me) { }
MBOReader::~MBOReader() { }
MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const {
// get a result eagerly so we don't have to do it while accessor has the entry locked
auto result = _me->pool.allocate();
MapType::const_accessor accessor;
if (_me->snapshots.find(accessor, key) && accessor->second != nullptr) {
result->copy_from(*accessor->second);
}
else {
release_snapshot(&result);
}
return result;
}
void MBOReader::release_snapshot(MBOSnapshot**s) const {
_me->pool.release(s);
}
}}
namespace std {
using namespace bts;
using namespace bts::md;
ostream& operator<<(ostream& os, const inc::Update& update) {
os << "Update: { "
<< "id:" << update.order_id.value() << " "
<< "pri:" << update.priority << " "
<< update.price.price() << " "
<< update.size << " ";
switch(update.type) {
case inc::UpdateType::Bid:
os << "B ";
break;
case inc::UpdateType::Ask:
os << "A ";
break;
}
switch(update.action) {
case inc::UpdateAction::New:
os << "+";
break;
case inc::UpdateAction::Change:
os << "C";
break;
case inc::UpdateAction::Delete:
os << "-";
break;
}
os << "}";
return os;
}
}