diff --git a/.vscode/launch.json b/.vscode/launch.json index fd68a6626..f3c362de0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -215,7 +215,7 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/server/sparkd", - "args": ["test", "[md]"], + "args": ["test", "[mbo]"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/server/", "environment": [], diff --git a/src/lib/md/IntPrice.h b/src/lib/md/IntPrice.h index 1f563548c..dfca4e73f 100644 --- a/src/lib/md/IntPrice.h +++ b/src/lib/md/IntPrice.h @@ -60,6 +60,12 @@ class IntPrice { return delta.mantissa() < 0; } } + bool operator <=(const IntPrice& other) { + return *this == other || *this < other; + } + bool operator >=(const IntPrice& other) { + return !(*this < other); + } IntPrice operator - (const IntPrice& o) const { return op(o, [](int64_t m1, int64_t m2) { return m1 - m2; }); } diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 4fec20880..1079df5bc 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -1,4 +1,4 @@ -#include "MBOProcessor.h" +#include "MBOProcessor.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/ConcurrentPool.h" #include "lib/config/config.h" @@ -12,8 +12,8 @@ #include -namespace bts { -namespace md { +namespace bts { +namespace md { struct SecurityKeyHashCompare { static size_t hash(const SecurityKey& k) { @@ -51,8 +51,8 @@ void print_pool_stats(const utils::ConcurrentPool& pool) { } bid_percentiles << bid_sz[count-1] << " "; ask_percentiles << ask_sz[count-1] << " "; - LOG(md,Info, - "MBO statistics: count: {} - %[50,75,90,95,99,100] bid: [{}] - ask: [{}]", + LOG(md,Info, + "MBO statistics: count: {} - %[50,75,90,95,99,100] bid: [{}] - ask: [{}]", count, bid_percentiles.str(), ask_percentiles.str()); } @@ -60,11 +60,13 @@ typedef tbb::concurrent_hash_map pool; MapType snapshots; + utils::Dispatcher& dispatcher; utils::DispatcherTimer timer; - MBOProcessorImpl() : - pool(2048,512), - snapshots(), - timer(utils::Dispatcher::background(), 600*1000, [&]() { print_pool_stats(pool); }) + MBOProcessorImpl(utils::Dispatcher& d) : + pool(2048,512), + snapshots(), + dispatcher(d), + timer(d, 600*1000, [&]() { print_pool_stats(pool); }) { if (config::get().enabled()) { timer.start(); @@ -75,8 +77,8 @@ struct MBOProcessorImpl { // // Processor -MBOProcessor::MBOProcessor() : - _me(std::make_shared()), +MBOProcessor::MBOProcessor(utils::Dispatcher* d) : + _me(std::make_shared(d == nullptr ? utils::Dispatcher::background() : *d)), _producer(_me), _consumer(_me) {} @@ -86,9 +88,10 @@ MBOReader& MBOProcessor::reader() { return _consumer; } // // Writer + MBOWriter::MBOWriter(std::shared_ptr& me): _me(me) { } MBOWriter::~MBOWriter() { } -MBOSnapshot* MBOWriter::allocate_snapshot() { +MBOSnapshot* MBOWriter::allocate_snapshot(size_t, size_t) { return _me->pool.allocate(); } void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot* snapshot) { @@ -102,6 +105,68 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot* snapshot) { } } +bool MBOWriter::validate_snapshot(SecurityKey key, MBOSnapshot* control) { + #define REQUIRE(x) if (!(x)) { LOG(md,Error, "{}", std::string(#x)); return false; } + // retrieve current snapshot and check they are equal + MBOReader reader(_me); + auto snap = reader.acquire_snapshot(key); + REQUIRE (snap->bids.size() == control->bids.size()) + REQUIRE (snap->asks.size() == control->asks.size()) + for (size_t x = 0; x < snap->bids.size(); ++x) { + REQUIRE (snap->bids[x].id == control->bids[x].id); + REQUIRE (snap->bids[x].price == control->bids[x].price); + REQUIRE (snap->bids[x].size == control->bids[x].size); + } + for (size_t x = 0; x < snap->asks.size(); ++x) { + REQUIRE (snap->asks[x].id == control->asks[x].id); + REQUIRE (snap->asks[x].price == control->asks[x].price); + REQUIRE (snap->asks[x].size == control->asks[x].size); + } + #undef REQUIRE + return true; +} +void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { + _me->dispatcher.post([this,&state,update]() { + this->apply_incremental(state,update); + }); +} +void MBOWriter::apply_incremental(MBOWorkingState& state, const inc::Update& update) { + // fast part + apply_incremental(state, update); + // determine if we need to publish + // slow part + auto snap = allocate_snapshot(state.bid.size(), state.ask.size()); + copy_to_snapshot(state, snap); + publish_snapshot(state.security, snap); +} + +void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { + if (dest == nullptr) return; + // copy working state to snapshot + dest->updated = state.updated; + auto type_map = [](auto pair) { + return SnapshotOrder { pair.second.id, std::get(pair.first), pair.second.size }; + }; + std::transform(state.bid.rbegin(), state.bid.rend(), std::back_inserter(dest->bids), type_map); + std::transform(state.ask.begin(), state.ask.end(), std::back_inserter(dest->asks), type_map); +} + +void apply_incremental(MBOWorkingState& state, const inc::Update& update) { + // apply the update to the working state + MBOWorkingState::Side& collection = (update.type == inc::UpdateType::Bid) ? state.bid : state.ask; + auto key = std::make_tuple(update.price, update.priority); + switch(update.action) { + case inc::UpdateAction::New: + case inc::UpdateAction::Change: + collection[key] = { update.order_id, update.size }; + break; + case inc::UpdateAction::Delete: + collection.erase(key); + break; + } +} + + // // Reader @@ -123,4 +188,4 @@ void MBOReader::release_snapshot(MBOSnapshot**s) const { _me->pool.release(s); } -}} +}} diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 17246fd21..04c3747d7 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -2,16 +2,20 @@ #include "lib/domain_model/SecurityKey.h" #include "lib/domain_model/Security.h" #include "lib/utils/TimeStamp.h" -#include "lib/utils/ValueWrapper.h" +#include "lib/utils/Dispatcher.h" #include "IntPrice.h" #include "ExchangeOrderKey.h" +#include "google/btree/btree_map.h" + #include #include +#include +#include namespace bts { namespace md { - + struct SnapshotOrder { ExchangeOrderKey id; IntPrice price; @@ -39,6 +43,59 @@ struct MBOSnapshot { } }; +namespace inc { + + // according to this link MBO book only has these updates types + // https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP+3.0+-+Market+Data+Snapshot+Full+Refresh#MDP3.0-MarketDataSnapshotFullRefresh-MarketbyOrder(MBO)DataBlock + enum class UpdateType { + Bid, // 0 + Ask, // 1 + // ImpliedBid, // 'E' + // ImpliedAsk, // 'F' + // BookReset // 'J' + }; + + // OrderUpdateAction=37708 + enum class UpdateAction { + New, //0 + Change, //1 + Delete //2 + }; + + typedef int64_t Priority; + struct Update { + ExchangeOrderKey order_id; + Priority priority; + IntPrice price; + int32_t size; + UpdateType type; + UpdateAction action; + }; +} + +struct MBOWorkingState { + typedef std::tuple key_t; + struct Data { + ExchangeOrderKey id; + int32_t size; + }; + //todo: b-tree this + typedef std::map Side; + + SecurityKey security; + utils::TimeStamp updated; + Side bid,ask; +}; +void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* destination); + +void apply_incremental(MBOWorkingState&, const inc::Update&); +template +void apply_incremental(MBOWorkingState& state, const V& vec) { + for (const auto& update : vec) { + apply_incremental(state, update); + } +} + struct MBOProcessorImpl; class MBOProcessor; @@ -47,16 +104,27 @@ class MBOWriter { friend class MBOProcessor; MBOWriter(std::shared_ptr&); ~MBOWriter(); + public: + // + // Snapshot api + MBOSnapshot* allocate_snapshot(size_t bid_count=0, size_t ask_count=0); + void publish_snapshot(SecurityKey, MBOSnapshot*); + bool validate_snapshot(SecurityKey, MBOSnapshot*); - MBOSnapshot* allocate_snapshot(); + // + // Incremental api - void publish_snapshot(SecurityKey key, MBOSnapshot* snapshot); + // Enqueue the update to the mbo work queue. + void enqueue_incremental(MBOWorkingState&, const inc::Update&); + // Update working state and publish directly. + void apply_incremental(MBOWorkingState&, const inc::Update&); }; class MBOReader { std::shared_ptr _me; friend class MBOProcessor; + friend class MBOWriter; MBOReader(std::shared_ptr&); ~MBOReader(); public: @@ -70,7 +138,7 @@ class MBOProcessor { MBOReader _consumer; public: - MBOProcessor(); + MBOProcessor(utils::Dispatcher* _=nullptr); MBOWriter& writer(); MBOReader& reader(); }; diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp index 70c4492e9..95517feb5 100644 --- a/src/lib/md/test/mbo_tests.cpp +++ b/src/lib/md/test/mbo_tests.cpp @@ -1,12 +1,17 @@ #ifndef _WIN32 #include "catch/catch.hpp" +#include "../IntPrice.h" #include "../MBOProcessor.h" + +#include +#include + namespace bts { namespace md { namespace test { int mbo_tests = 0; -TEST_CASE("mbo basic", "[md][mbo]") { +TEST_CASE("MBO publish snapshot", "[md][mbo]") { MBOProcessor p; // publish a snapshot auto sec = SecurityKey(99); @@ -23,5 +28,130 @@ TEST_CASE("mbo basic", "[md][mbo]") { REQUIRE(s->asks.size() == 0); } +std::vector gen_test_orders(uint64_t id=100, inc::UpdateType type=inc::UpdateType::Bid) { + std::vector result; + inc::Priority priority = 10000; + for (int p = 100; p < 501; p+=100) { + // 6 sizes + for (int32_t s = 1; s <= 5; ++s) { + IntPrice price { p + 5 , -2}; + auto order_id = ExchangeOrderKey(++id); + auto size = s; + priority++; + result.push_back( inc::Update { + order_id, + priority, + price, + size, + type, + inc::UpdateAction::New + }); + // std::cout << std::endl << order_id.value() << ": " << price.price() << "-" << size; + } + } + std::random_shuffle(result.begin(), result.end()); + return result; +} + +TEST_CASE("MBO state to snapshot", "[md][mbo]") { + // convert state to snapshot + MBOWorkingState state; + + state.security = SecurityKey(42); + state.updated = utils::TimeStamp::now(); + + auto bids = gen_test_orders(1000, inc::UpdateType::Bid); + apply_incremental(state, bids); + auto asks = gen_test_orders(2000, inc::UpdateType::Ask); + apply_incremental(state, asks); + + MBOSnapshot result; + copy_to_snapshot(state, &result); + + REQUIRE(result.updated == state.updated); + REQUIRE(result.bids.size() == state.bid.size()); + REQUIRE(result.asks.size() == state.ask.size()); + REQUIRE(bids.size() == result.bids.size()); + REQUIRE(asks.size() == result.asks.size()); + for (size_t x = 0; x < result.bids.size() - 1; ++x) { + // std::cout << std::endl << result.bids[x].id.value() << ": " << result.bids[x].price.price() << "-" << result.bids[x].size; + REQUIRE(result.bids[x].price >= result.bids[x+1].price); + } + for (size_t x = 0; x < result.asks.size() - 1; ++x) { + // std::cout << std::endl << result.asks[x].id.value() << ": " << result.asks[x].price.price() << "-" << result.asks[x].size; + REQUIRE(result.asks[x].price <= result.asks[x+1].price); + } + +} + +template +std::vector concat(const std::vector& a, const std::vector& b) { + std::vector res; + std::copy(a.begin(), a.end(), std::back_inserter(res)); + std::copy(b.begin(), b.end(), std::back_inserter(res)); + return res; +} + +TEST_CASE("MBO incremental", "[md][mbo]") { + // basic incremental test + MBOWorkingState state; + + auto orders = + concat( gen_test_orders(128, inc::UpdateType::Bid), + gen_test_orders(256, inc::UpdateType::Ask)); + + std::random_shuffle(orders.begin(), orders.end()); + apply_incremental(state, orders); + + REQUIRE(state.bid.size() + state.ask.size() == orders.size()); + + // generate some updates/deletes + std::vector updates; + // - change all size=2 to size=6 + // - delete all size=3 + size_t removed = 0; + for (const auto& e : orders) { + if (e.size == 2) { + updates.push_back( inc::Update { + e.order_id, + e.priority, + e.price, + 6, + e.type, + inc::UpdateAction::Change + }); + } + else if (e.size == 3) { + ++removed; + updates.push_back( inc::Update { + e.order_id, + e.priority, + e.price, + e.size, + e.type, + inc::UpdateAction::Delete + }); + } + } + apply_incremental(state,updates); + + REQUIRE(state.bid.size() + state.ask.size() == (orders.size() - removed)); + + MBOSnapshot result; + copy_to_snapshot(state, &result); + + auto all = concat (result.bids, result.asks); + std::map counts; + for (const auto& order : all) { + REQUIRE(order.size != 3); + REQUIRE(order.size != 2); + counts[order.size] ++; + } + REQUIRE(counts[6] > 0); + REQUIRE(counts[6] == counts[1]); + REQUIRE(counts[5] == counts[4]); + REQUIRE(counts[6] == counts[5]); +} + }}} #endif \ No newline at end of file diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 5f255fc56..d225abfde 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -60,7 +60,7 @@ int Run::execute(std::vector const &) { persistence::PersistenceServiceServer settings_saver(sc, "settings"); user::AuthServiceServer login_server(sc); - md::MBOProcessor mbo; + md::MBOProcessor mbo(&utils::Dispatcher::get_named("MBO")); md::run_market_datapublishers(&mbo.writer()); api::synthetic::SyntheticSecurityService synthetic_service(sc); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index c18a823d7..cd757f5c6 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -62,16 +62,42 @@ public: // int qty; // a.displayQty(qty); // uint8_t ref; - // a.referenceId(ref); - // const auto& ref_entry = args.message().entries()[(size_t)ref]; - // std::cout - // << a.orderId() - // << qty - // << ref_entry.securityId() - // ; + // if (a.referenceId(ref)) { + // const auto& ref_entry = args.message().entries()[(size_t)ref]; + // LOG(md,Info, "INCREMENTAL32: {} {} {}", + // a.orderId(), + // qty, + // ref_entry.securityId()); + // } // } // }; - // void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args&) override {}; + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args& args) override { + for(const auto& a : args.message().orderIdEntries()) { + int qty; + a.displayQty(qty); + uint8_t ref; + if (a.referenceId(ref)) { + const auto& ref_entry = args.message().entries()[(size_t)ref]; + LOG(md,Info, "INCREMENTAL46: {} {} {}", + a.orderId(), + qty, + ref_entry.securityId()); + } + } + }; + + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook44Args&) override { + if (_mbo) { + LOG(md,Info, "GOT SNAPSHOT44"); + } + } + + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook53Args&) override { + if (_mbo) { + LOG(md,Info, "GOT SNAPSHOT53"); + } + } + // diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index ac8da7ff3..025de1078 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -129,7 +129,11 @@ onixs::SessionSettings session_settings_for_channel(const string& channel_id) { } if (!options.join_snapshot_on_gap()) { + #ifdef DEBUG + settings.gapAndErrorHandling(onixs::GapAndErrorHandlingOptions::RecoverMarketState); + #else settings.gapAndErrorHandling(onixs::GapAndErrorHandlingOptions::ContinueProcessing); + #endif } if (options.use_natural_refresh()) { settings.joinRecovery(onixs::JoinRecoveryOptions::Disabled);