From be6c9166911f8998e216179a0452b81832b4f61c Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 28 Mar 2019 16:49:59 -0400 Subject: [PATCH 001/100] Allow std::function to move into Dispatcher --- src/lib/utils/Dispatcher.cpp | 2 +- src/lib/utils/Dispatcher.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/utils/Dispatcher.cpp b/src/lib/utils/Dispatcher.cpp index 9c504ee8c..1d7ab64d3 100755 --- a/src/lib/utils/Dispatcher.cpp +++ b/src/lib/utils/Dispatcher.cpp @@ -200,7 +200,7 @@ void Dispatcher::invoke(Action action) } } -void Dispatcher::post(Action action, int64_t delay_ms) +void Dispatcher::post(Action&& action, int64_t delay_ms) { if (delay_ms > 0) { diff --git a/src/lib/utils/Dispatcher.h b/src/lib/utils/Dispatcher.h index 4a4470fb8..2adc656af 100755 --- a/src/lib/utils/Dispatcher.h +++ b/src/lib/utils/Dispatcher.h @@ -87,7 +87,7 @@ public: // Perform an action on the dispatcher thread. Block until the action is complete. void invoke(Action action); // Queue an action on the dispatcher thread and return immediately. - void post(Action action, int64_t delay_ms=0); + void post(Action&& action, int64_t delay_ms=0); boost::asio::io_service* io_service(); From e9da3384024e6ad00fa90f3beb315ab72e117cac Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 28 Mar 2019 17:08:01 -0400 Subject: [PATCH 002/100] optimize IntPrice lessthan --- src/lib/md/IntPrice.h | 10 +++++++--- src/server/sparkd/cmds/Benchmark.cpp | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/lib/md/IntPrice.h b/src/lib/md/IntPrice.h index a66806eb3..191988fe3 100644 --- a/src/lib/md/IntPrice.h +++ b/src/lib/md/IntPrice.h @@ -43,7 +43,9 @@ class IntPrice { return _mantissa == other._mantissa; } else { - return price()==other.price(); + // return price()==other.price(); + auto delta = *this - other; + return delta.mantissa() == 0; } } bool operator!=(const IntPrice& other) const { @@ -54,7 +56,10 @@ class IntPrice { return _mantissa < other._mantissa; } else { - return price() < other.price(); + // avoiding conversion to double is 4-5% faster + // return price() < other.price(); + auto delta = *this - other; + return delta.mantissa() < 0; } } IntPrice operator - (const IntPrice& o) const { @@ -169,7 +174,6 @@ inline md::IntPrice round_to_quote_price(const IntPrice& tick, const IntPrice& p return price; } - inline std::ostream& operator<<(std::ostream& os, const IntPrice& p) { os << p.price(); return os; diff --git a/src/server/sparkd/cmds/Benchmark.cpp b/src/server/sparkd/cmds/Benchmark.cpp index 360f29f89..d62e01f9e 100644 --- a/src/server/sparkd/cmds/Benchmark.cpp +++ b/src/server/sparkd/cmds/Benchmark.cpp @@ -640,6 +640,21 @@ static void perf_log(benchmark::State &state) { } BENCHMARK(perf_log); +void int_price_less(benchmark::State& state) { + for (auto _ : state) { + state.PauseTiming(); + int64_t count = state.range(0); + std::vector prices; + prices.reserve((int)count); + for (int64_t x = 0; x < count; ++x) { + prices.push_back({x % 100, (int8_t)-(x%5)}); + } + state.ResumeTiming(); + std::sort(prices.begin(), prices.end()); + } +} +BENCHMARK(int_price_less)->Range(8, 8 << 11); + std::string Benchmark::help() { return "Run micro-benchmarks. Use --benchmark_filter=regex to run only " "specific benchmarks."; From bad5bc30975a05b042a00e35f1c40f3c044bbd17 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 11:08:37 -0400 Subject: [PATCH 003/100] SPARK-2202 MBOProcessor producer side --- src/lib/config/protos/MarketData.proto | 3 +- src/lib/md/MBOProcessor.cpp | 81 ++++++++++++++++++ src/lib/md/MBOProcessor.h | 82 +++++++++++++++++++ src/lib/utils/ConcurrentPool.h | 55 +++++++++++++ src/server/sparkd/cmds/Run.cpp | 7 +- src/server/sparkd/cmds/Web.cpp | 2 +- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 25 +++++- .../md/platforms/mdp3_onixs/MDP3Listener.h | 27 +++++- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 16 ++-- .../sparkd/md/platforms/mdp3_onixs/MDP3Main.h | 6 +- src/server/sparkd/md/run_md.cpp | 25 +++--- src/server/sparkd/md/run_md.h | 4 +- 12 files changed, 302 insertions(+), 31 deletions(-) create mode 100644 src/lib/md/MBOProcessor.cpp create mode 100644 src/lib/md/MBOProcessor.h create mode 100644 src/lib/utils/ConcurrentPool.h diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index c52377235..3efb46c77 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -113,7 +113,8 @@ message CMEOptions optional bool load_security_db = 33 [default = true]; reserved 34; - // optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED]; + // optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED]; + optional bool maintain_mbo = 35 [default = true]; } optional string onixs_license = 1 [default = "{default}"]; diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp new file mode 100644 index 000000000..4366fb528 --- /dev/null +++ b/src/lib/md/MBOProcessor.cpp @@ -0,0 +1,81 @@ +#include "MBOProcessor.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/ConcurrentPool.h" + +#include +#include + +#include + + +namespace bts { +namespace md { + +struct SecurityKeyHashCompare { + static size_t hash(const SecurityKey& k) { + return k.value(); + } + static bool equal(const SecurityKey& a, const SecurityKey& b) { + return a.value() == b.value(); + } +}; + +typedef tbb::concurrent_hash_map MapType; +struct MBOProcessorImpl { + utils::ConcurrentPool pool; + MapType snapshots; + MBOProcessorImpl() : pool(2048,512), snapshots() {} +}; + +// +// Processor + +MBOProcessor::MBOProcessor() : + _me(std::make_shared()), + _producer(_me), + _consumer(_me) +{} + +MBOWriter& MBOProcessor::writer() { return _producer;} +MBOReader& MBOProcessor::reader() { return _consumer; } + +// +// Writer +MBOWriter::MBOWriter(std::shared_ptr& me): _me(me) { } +MBOWriter::~MBOWriter() { } +snapshot::MBOSnapshot* MBOWriter::allocate_snapshot() { + return _me->pool.allocate(); +} +void MBOWriter::publish_snapshot(SecurityKey key, snapshot::MBOSnapshot* snapshot) { + bts_assert(snapshot != nullptr); + MapType::accessor accessor; + _me->snapshots.insert(accessor, key); + std::swap(accessor->second, snapshot); + if (snapshot != nullptr) { + // Note: relase() should happen inside accessor lifetime + _me->pool.release(&snapshot); + } +} + +// +// Reader + +MBOReader::MBOReader(std::shared_ptr& me): _me(me) { } +MBOReader::~MBOReader() { } +snapshot::MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) { + // get a result eagerly so we don't have to do it while accessor has the entry locked + auto result = _me->pool.allocate(); + MapType::const_accessor accessor; + if (_me->snapshots.find(accessor, key) && accessor->second != nullptr) { + result->copy_from(*accessor->second); + } + else { + release_snapshot(&result); + } + return result; +} +void MBOReader::release_snapshot(snapshot::MBOSnapshot** s) { + _me->pool.release(s); +} + +}} diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h new file mode 100644 index 000000000..854c47a6d --- /dev/null +++ b/src/lib/md/MBOProcessor.h @@ -0,0 +1,82 @@ +#pragma once +#include "lib/domain_model/SecurityKey.h" +#include "lib/domain_model/Security.h" +#include "lib/utils/TimeStamp.h" +#include "lib/utils/ValueWrapper.h" +#include "IntPrice.h" + +#include +#include + +namespace bts { +namespace md { + +using OrderKey = utils::ValueWrapper; + +namespace snapshot { + struct SnapshotOrder { + OrderKey id; + int32_t size; + }; + struct MBOSnapshot { + MBOSnapshot() = default; + MBOSnapshot(const MBOSnapshot&) = delete; + MBOSnapshot(MBOSnapshot&& other) = delete; + + utils::TimeStamp updated; + std::vector bids; + std::vector asks; + + void copy_from(const MBOSnapshot& src) { + bids = src.bids; + asks = src.asks; + updated = src.updated; + } + + void reset() { + updated = utils::TimeStamp(); + bids.clear(); + asks.clear(); + } + }; + +} + +struct MBOProcessorImpl; +class MBOProcessor; + +class MBOWriter { + std::shared_ptr _me; + friend class MBOProcessor; + MBOWriter(std::shared_ptr&); + ~MBOWriter(); + public: + + snapshot::MBOSnapshot* allocate_snapshot(); + + void publish_snapshot(SecurityKey key, snapshot::MBOSnapshot* snapshot); +}; + +class MBOReader { + std::shared_ptr _me; + friend class MBOProcessor; + MBOReader(std::shared_ptr&); + ~MBOReader(); + public: + snapshot::MBOSnapshot* acquire_snapshot(SecurityKey key); + void release_snapshot(snapshot::MBOSnapshot**); +}; + +class MBOProcessor { + std::shared_ptr _me; + MBOWriter _producer; + MBOReader _consumer; + +public: + MBOProcessor(); + MBOWriter& writer(); + MBOReader& reader(); +}; + + +}} diff --git a/src/lib/utils/ConcurrentPool.h b/src/lib/utils/ConcurrentPool.h new file mode 100644 index 000000000..91ad99095 --- /dev/null +++ b/src/lib/utils/ConcurrentPool.h @@ -0,0 +1,55 @@ +#pragma once +#include +#include +#include + +namespace bts { +namespace utils { + +template +class ConcurrentPool { + tbb::concurrent_vector> _all; + tbb::concurrent_queue _free; + size_t _grow_size; + std::atomic _outstanding; + public: + ConcurrentPool(size_t initial_size=256, size_t grow_size=32): + _all(), _free(), _grow_size(grow_size), _outstanding(0) { + reserve(initial_size); + } + + ~ConcurrentPool() { } + + T* allocate() { + T* result = nullptr; + ++_outstanding; + if (_free.try_pop(result)) { + return result; + } + else { + reserve(_grow_size); + return allocate(); + } + } + + void reserve(size_t count) { + std::unique_ptr buffer {new T[count]}; + for (size_t x = 0; x < count; ++x) { + T* temp = &buffer[x]; + release(&temp); + } + _all.push_back(std::move(buffer)); + } + + void release(T** item) { + if (*item) { + (*item)->reset(); + _free.push(*item); + *item = nullptr; + --_outstanding; + } + } + +}; + +}} //namespace diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 2dc515483..3b788c4a2 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -14,6 +14,7 @@ #include "lib/md/MDClient.h" #include "lib/md/SyntheticSecurityService.h" #include "lib/md/relay/QueryServer.h" +#include "lib/md/MBOProcessor.h" #include "lib/metrics/metrics.h" #include "lib/metrics/system_metrics.h" #include "lib/orderman/order_factory/DynamicOrderRegistry.h" @@ -57,10 +58,12 @@ int Run::execute(std::vector const &) { persistence::PersistenceServiceServer settings_saver(sc, "settings"); user::AuthServiceServer login_server(sc); - md::run_market_datapublishers(); + + md::MBOProcessor mbo; + md::run_market_datapublishers(&mbo.writer()); api::synthetic::SyntheticSecurityService synthetic_service(sc); - md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service); + md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service /*, mbo*/); md::QueryServerPtr md_query_server( diff --git a/src/server/sparkd/cmds/Web.cpp b/src/server/sparkd/cmds/Web.cpp index 029ae34bc..abbe80ed6 100644 --- a/src/server/sparkd/cmds/Web.cpp +++ b/src/server/sparkd/cmds/Web.cpp @@ -37,7 +37,7 @@ int Web::execute(const std::vector& args) { user::AuthServiceServer login_server(sc); if (std::find(args.begin(), args.end(), "-md") != args.end()) { - md::run_market_datapublishers(); + md::run_market_datapublishers(nullptr); } api::synthetic::SyntheticSecurityService synthetic_service(sc); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index 6049e982b..f25450a6d 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -15,7 +15,7 @@ namespace md { MDP3Listener::MDP3Listener( CMESecurityIndex& index, MDProducer& producer, - // co2::SettlementRecorder& , + MBOWriter* mbo, const std::vector channels) : _index(index), _producer(producer), @@ -27,7 +27,8 @@ MDP3Listener::MDP3Listener( _symbol_filter(), _channels(channels), _concurrency_monitor(0), - _book_updates() + _book_updates(), + _mbo(mbo) { _book_updates.reserve(32); } @@ -109,7 +110,7 @@ void MDP3Listener::onEndOfPacket(onixs::Handler&, const onixs::PacketArgs&) { } // -// Security discovery +// Book updates void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) { onBookUpdate(sec, book, book_type_direct); @@ -119,6 +120,24 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con onBookUpdate(sec, book, book_type_implied); } +void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::MboBook& book) { + if (_mbo) { + auto data = _index.get_security(sec.id()); + if (data == nullptr) return; + auto key = data->handle.key(); + if (key == invalid_security_key) return; + auto* snapshot = _mbo->allocate_snapshot(); + // copy from onixs book + for (const auto& bid : book.bids()) { + snapshot->bids.push_back({ OrderKey(bid.id()), bid.quantity() }); + } + for (const auto& ask : book.offers()) { + snapshot->asks.push_back({ OrderKey(ask.id()), ask.quantity() }); + } + _mbo->publish_snapshot(SecurityKey(key), snapshot); + } +} + // // Trades diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index fc20add4c..c18a823d7 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -2,6 +2,7 @@ #include "lib/utils/bts_no_copy.h" #include "lib/md/coconut2.h" +#include "lib/md/MBOProcessor.h" #include "CMESecurityIndex.h" #include "lib/tracing/bts_trace.h" @@ -13,11 +14,11 @@ #include #include #include +#include namespace bts { namespace md { - class MDP3Listener : public onixs::HandlerListener, public onixs::MarketDataListener, @@ -31,7 +32,7 @@ class MDP3Listener : public: MDP3Listener(CMESecurityIndex& index, MDProducer&, - // co2::SettlementRecorder&, + MBOWriter* mbo, const std::vector); virtual ~MDP3Listener(); @@ -54,6 +55,25 @@ public: void onPacket(onixs::Handler&, const onixs::PacketArgs&) override; void onEndOfPacket(onixs::Handler&, const onixs::PacketArgs&) override; + // + // Handle MBO incremental updates directly + // void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args& args) override { + // for(const auto& a : args.message().orderIdEntries()) { + // int qty; + // a.displayQty(qty); + // uint8_t ref; + // a.referenceId(ref); + // const auto& ref_entry = args.message().entries()[(size_t)ref]; + // std::cout + // << a.orderId() + // << qty + // << ref_entry.securityId() + // ; + // } + // }; + // void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args&) override {}; + + // // Security discovery template @@ -74,9 +94,9 @@ public: // // Book - // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override; void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override; void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ImpliedBook&) override; + void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override; // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override; template @@ -199,6 +219,7 @@ private: std::pair, std::array> asks; }; std::vector _book_updates; + MBOWriter* _mbo; }; ////////////////////////////////////////// diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 7c0db2832..c711993ac 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -1,6 +1,7 @@ #include "MDP3Main.h" #include "lib/md/cme/MDP3Utils.h" +#include "lib/md/MBOProcessor.h" #include "lib/config/config.h" #include "lib/md/coconut2.h" #include "lib/os/os.h" @@ -130,14 +131,14 @@ onixs::SessionSettings session_settings_for_channel(const string& channel_id) { return settings; } -void MDP3Main::run(std::vector& symbols) { +void MDP3Main::run(std::vector& symbols, MBOWriter* mbo) { await_cme_config(); initialize(); const auto& channels = parse_channels(symbols); _engines = create_feed_engines(channels); for (const auto& channel : channels) { - unique_ptr phandler = prepare_handler(channel.id); - _listeners.push_back(prepare_listener(*phandler, channel)); + unique_ptr phandler = prepare_handler(channel.id, mbo); + _listeners.push_back(prepare_listener(*phandler, channel, mbo)); _handlers.insert(std::make_pair(channel.id, move(phandler))); std::stringstream symstr; for (const auto& sym : channel.symbols) { symstr << sym.symbol() << "-" << SecurityType_Name(sym.type()) << " "; } @@ -196,7 +197,7 @@ void MDP3Main::initialize() { // // // Initialize HandlerSettings and allocate the channel Handler -unique_ptr MDP3Main::prepare_handler(const string& channel_id) { +unique_ptr MDP3Main::prepare_handler(const string& channel_id, MBOWriter* mbo) { const auto& pub_options = config::get(); const auto& options = channel_settings_for(channel_id); LOG(md,Info,"Loaded channel options for channel {}:\n{}",channel_id ,options.DebugString()); @@ -209,6 +210,9 @@ unique_ptr MDP3Main::prepare_handler(const string& channel_id) { settings.bookManagement().directBooks().maintain(true); settings.bookManagement().impliedBooks().maintain(true); settings.bookManagement().consolidatedBooks().maintain(false); + if (options.maintain_mbo() && mbo != nullptr) { + settings.bookManagement().mboBooks().maintain(true); + } // 10k default is *far* too low settings.recoveryQueueMaxSize(options.recovery_queue_size()); @@ -331,14 +335,14 @@ unique_ptr MDP3Main::prepare_handler(const string& channel_id) { // // Create a listener for this channel and register callbacks -unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, const CMEChannel& channel) { +unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, const CMEChannel& channel, MBOWriter* mbo) { // We need to get a handler to the producer channels so we can produce heartbeats vector producer_channels; for (auto& symbol : channel.symbols) { producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } - unique_ptr listener(new MDP3Listener(_index, _producer, /*_settlement_recorder,*/ producer_channels)); + unique_ptr listener(new MDP3Listener(_index, _producer, mbo, producer_channels)); for (auto& symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index 4eb1fbf62..eb321087f 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -15,6 +15,8 @@ namespace md { // void run_onixs_mdp3_server(UserSymbol* sym, size_t symcount); +class MBOWriter; + struct CMEChannel; /* @@ -29,10 +31,10 @@ public: MDP3Main(); ~MDP3Main(); - void run(std::vector&); + void run(std::vector&, MBOWriter*); void initialize(); std::unique_ptr prepare_handler(const std::string& channel); - std::unique_ptr prepare_listener(onixs::Handler& handler, const CMEChannel& channel); + std::unique_ptr prepare_listener(onixs::Handler& handler, const CMEChannel& channel, MBOWriter*); private: BTS_NO_COPY(MDP3Main); diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index 56e2c2762..bf216d7ec 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -20,6 +20,7 @@ #include #include #include +#include using namespace bts; using namespace std; @@ -65,12 +66,12 @@ void run_replay() { replay_recording(file, naggle_ms, fallback_path); } -void run_onixs() { +void run_onixs(MBOWriter* mbo) { // get all glbx symbols std::vector symbols; get_system_md_symbols(api::VENUE_GLBX, api::SECURITYTYPE_UNSET, &symbols); MDP3Main main; - main.run(symbols); + main.run(symbols, mbo); } void run_cleanup() { @@ -78,15 +79,14 @@ void run_cleanup() { } -std::thread* _md_thread; -void run_market_data_publishers_forever() { - os::set_thread_name("md publisher"); +std::optional _md_thread; +void run_market_data_publishers_forever(MBOWriter* mbo) { const auto& conf = config::get(); if (conf.use_replay()) { run_replay(); } else if (conf.use_onixs()) { - run_onixs(); + run_onixs(mbo); } } @@ -110,14 +110,15 @@ void boot_sleep() { } } -void run_market_datapublishers() { - bts_assert(_md_thread == nullptr); +void run_market_datapublishers(MBOWriter* mbo) { + bts_assert(_md_thread == std::nullopt); run_cleanup(); if (!config::get().use_md()) return; // start a thread for the md - _md_thread = new std::thread([]() { - run_market_data_publishers_forever(); - }); + _md_thread = std::optional( + threading::make_thread("md publisher", [mbo]() { + run_market_data_publishers_forever(mbo); + })); if (config::get().use_sim()) { threading::make_thread("ExchangeSim", []() { sim::start_exchange_sim(); }) @@ -327,7 +328,7 @@ void md_tool(std::vector const &args) { else if (cmd == "cleanup") run_cleanup(); else if (cmd == "dump") run_dump(vector(args.begin()+1, args.end())); else if (cmd == "replay") run_replay(); - else if (cmd == "run") run_market_data_publishers_forever(); + else if (cmd == "run") run_market_data_publishers_forever(nullptr); else if (cmd == "relay") relay_everything(); else if (cmd == "test_relay") relay_everything(); else if (cmd == "dumpsecurity" || cmd == "dumpsecurities") dump_db(vector(args.begin()+1, args.end())); diff --git a/src/server/sparkd/md/run_md.h b/src/server/sparkd/md/run_md.h index 587a6e0c1..dc771b0bf 100644 --- a/src/server/sparkd/md/run_md.h +++ b/src/server/sparkd/md/run_md.h @@ -5,7 +5,9 @@ namespace bts { namespace md { -void run_market_datapublishers(); +class MBOWriter; + +void run_market_datapublishers(MBOWriter*); void md_tool(std::vector const& args); }} From ce438e93a63fdef4ff23287156449e442007a101 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 14:00:26 -0400 Subject: [PATCH 004/100] Revert IntPrice regression (unit test failure) --- src/lib/md/IntPrice.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib/md/IntPrice.h b/src/lib/md/IntPrice.h index 191988fe3..1f563548c 100644 --- a/src/lib/md/IntPrice.h +++ b/src/lib/md/IntPrice.h @@ -43,9 +43,7 @@ class IntPrice { return _mantissa == other._mantissa; } else { - // return price()==other.price(); - auto delta = *this - other; - return delta.mantissa() == 0; + return price()==other.price(); } } bool operator!=(const IntPrice& other) const { From 2020221e652ad6ee3552a0e40ce969f895caf35f Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 14:02:29 -0400 Subject: [PATCH 005/100] MBOProducer cleanup and const correctness --- src/lib/md/MBOProcessor.cpp | 14 ++++----- src/lib/md/MBOProcessor.h | 55 ++++++++++++++++------------------ src/lib/utils/ConcurrentPool.h | 2 +- 3 files changed, 34 insertions(+), 37 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 4366fb528..de7fd5363 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -20,9 +20,9 @@ struct SecurityKeyHashCompare { } }; -typedef tbb::concurrent_hash_map MapType; +typedef tbb::concurrent_hash_map MapType; struct MBOProcessorImpl { - utils::ConcurrentPool pool; + utils::ConcurrentPool pool; MapType snapshots; MBOProcessorImpl() : pool(2048,512), snapshots() {} }; @@ -43,16 +43,16 @@ MBOReader& MBOProcessor::reader() { return _consumer; } // Writer MBOWriter::MBOWriter(std::shared_ptr& me): _me(me) { } MBOWriter::~MBOWriter() { } -snapshot::MBOSnapshot* MBOWriter::allocate_snapshot() { +MBOSnapshot* MBOWriter::allocate_snapshot() { return _me->pool.allocate(); } -void MBOWriter::publish_snapshot(SecurityKey key, snapshot::MBOSnapshot* snapshot) { +void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot* snapshot) { bts_assert(snapshot != nullptr); MapType::accessor accessor; _me->snapshots.insert(accessor, key); std::swap(accessor->second, snapshot); if (snapshot != nullptr) { - // Note: relase() should happen inside accessor lifetime + // Note: relase() must happen inside accessor lifetime to ensure correctness _me->pool.release(&snapshot); } } @@ -62,7 +62,7 @@ void MBOWriter::publish_snapshot(SecurityKey key, snapshot::MBOSnapshot* snapsho MBOReader::MBOReader(std::shared_ptr& me): _me(me) { } MBOReader::~MBOReader() { } -snapshot::MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) { +const MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { // get a result eagerly so we don't have to do it while accessor has the entry locked auto result = _me->pool.allocate(); MapType::const_accessor accessor; @@ -74,7 +74,7 @@ snapshot::MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) { } return result; } -void MBOReader::release_snapshot(snapshot::MBOSnapshot** s) { +void MBOReader::release_snapshot(MBOSnapshot**const s) const { _me->pool.release(s); } diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 854c47a6d..0c02f10dc 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -13,34 +13,31 @@ namespace md { using OrderKey = utils::ValueWrapper; -namespace snapshot { - struct SnapshotOrder { - OrderKey id; - int32_t size; - }; - struct MBOSnapshot { - MBOSnapshot() = default; - MBOSnapshot(const MBOSnapshot&) = delete; - MBOSnapshot(MBOSnapshot&& other) = delete; +struct SnapshotOrder { + OrderKey id; + int32_t size; +}; +struct MBOSnapshot { + MBOSnapshot() = default; + MBOSnapshot(const MBOSnapshot&) = delete; + MBOSnapshot(MBOSnapshot&& other) = delete; - utils::TimeStamp updated; - std::vector bids; - std::vector asks; + utils::TimeStamp updated; + std::vector bids; + std::vector asks; - void copy_from(const MBOSnapshot& src) { - bids = src.bids; - asks = src.asks; - updated = src.updated; - } + void copy_from(const MBOSnapshot& src) { + bids = src.bids; + asks = src.asks; + updated = src.updated; + } - void reset() { - updated = utils::TimeStamp(); - bids.clear(); - asks.clear(); - } - }; - -} + void reset() { + updated = utils::TimeStamp(); + bids.clear(); + asks.clear(); + } +}; struct MBOProcessorImpl; class MBOProcessor; @@ -52,9 +49,9 @@ class MBOWriter { ~MBOWriter(); public: - snapshot::MBOSnapshot* allocate_snapshot(); + MBOSnapshot* allocate_snapshot(); - void publish_snapshot(SecurityKey key, snapshot::MBOSnapshot* snapshot); + void publish_snapshot(SecurityKey key, MBOSnapshot* snapshot); }; class MBOReader { @@ -63,8 +60,8 @@ class MBOReader { MBOReader(std::shared_ptr&); ~MBOReader(); public: - snapshot::MBOSnapshot* acquire_snapshot(SecurityKey key); - void release_snapshot(snapshot::MBOSnapshot**); + const MBOSnapshot* acquire_snapshot(SecurityKey key) const; + void release_snapshot(MBOSnapshot**const) const; }; class MBOProcessor { diff --git a/src/lib/utils/ConcurrentPool.h b/src/lib/utils/ConcurrentPool.h index 91ad99095..96802c5dc 100644 --- a/src/lib/utils/ConcurrentPool.h +++ b/src/lib/utils/ConcurrentPool.h @@ -41,7 +41,7 @@ class ConcurrentPool { _all.push_back(std::move(buffer)); } - void release(T** item) { + void release(T**const item) { if (*item) { (*item)->reset(); _free.push(*item); From e1cfa7bf0bbe09172781fbf3f38c1ae4ba64c772 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 14:06:25 -0400 Subject: [PATCH 006/100] Fix build break --- src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index eb321087f..6cb777afe 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -33,7 +33,7 @@ public: void run(std::vector&, MBOWriter*); void initialize(); - std::unique_ptr prepare_handler(const std::string& channel); + std::unique_ptr prepare_handler(const std::string& channel, MBOWriter*); std::unique_ptr prepare_listener(onixs::Handler& handler, const CMEChannel& channel, MBOWriter*); private: From dc1a80afd215686da7833ed02320a141adbadf63 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 14:07:48 -0400 Subject: [PATCH 007/100] Make MBOReader available to MDConsumerHandle --- src/lib/md/Diag/MDPeriodicDumper.cpp | 2 +- src/lib/md/MDClient.cpp | 15 +++++++++----- src/lib/md/MDClient.h | 3 +++ src/lib/md/MDConsumer.cpp | 19 +++++++++++++---- src/lib/md/MDConsumer.h | 2 ++ src/lib/md/MDConsumerHandle.cpp | 5 +++-- src/lib/md/MDConsumerHandle.h | 23 +++++++++++++++++++- src/lib/md/test/Co2TestInstrument.cpp | 2 +- src/lib/md/test/md2_tests.cpp | 30 +++++++++++++-------------- src/server/sparkd/cmds/Run.cpp | 2 +- 10 files changed, 73 insertions(+), 30 deletions(-) diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index 46e5bfbcd..d25ab3d5f 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -160,7 +160,7 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, threading::configure_current_thread(thread_name); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols); + MDConsumer consumer(cache, nullptr, nullptr, &symbols); // read all securities into a vector std::vector entries; diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index 1cb454971..3695640ba 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -3,6 +3,7 @@ #include "SecurityCache.h" #include "SecurityOracle.h" #include "lib/utils/TimeStamp.h" +#include "MBOProcessor.h" #include namespace bts { @@ -15,19 +16,22 @@ struct MDClientImpl{ std::unique_ptr _security_timer; std::unique_ptr _recovery_timer; std::set _symbols; + MBOReader* _mbo; MDConsumer _consumer; utils::Event _event; MDClientImpl( utils::Dispatcher* d, api::ISyntheticSecurityService* s, + MBOReader* mbo, const std::set* symbols): _dispatcher(d), _ss(s), _security_timer(), _recovery_timer(), _symbols(symbols ? *symbols : get_system_md_symbols()), - _consumer(_cache, _ss, &_symbols), + _mbo(mbo), + _consumer(_cache, _ss, mbo, &_symbols), _event() { if (d) { @@ -42,7 +46,7 @@ struct MDClientImpl{ void check_heartbeats() { if (!_consumer.all_heartbeats_ok()) { - MDConsumer temp(_cache,_ss); + MDConsumer temp(_cache,_ss,_mbo,&_symbols); if (temp.all_heartbeats_ok()) { _consumer.swap(temp); LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); @@ -71,11 +75,12 @@ struct MDClientImpl{ MDClient::MDClient( utils::Dispatcher* d, api::ISyntheticSecurityService* s, + MBOReader* mbo, const std::set* symbols) : - _self(new MDClientImpl(d,s, symbols)) + _self(new MDClientImpl(d,s,mbo,symbols)) { } -MDClient::~MDClient() { -} + +MDClient::~MDClient() { } size_t MDClient::scan_securities() { return _self->scan_securities(); } void MDClient::check_heartbeats() { return _self->check_heartbeats(); } diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h index 47925bc88..51b03a777 100644 --- a/src/lib/md/MDClient.h +++ b/src/lib/md/MDClient.h @@ -11,7 +11,9 @@ namespace bts { namespace md { class SecurityOracle; +class MBOReader; struct MDClientImpl; + class MDClient { std::unique_ptr _self; @@ -20,6 +22,7 @@ public: MDClient( utils::Dispatcher* d=nullptr, api::ISyntheticSecurityService* ss=nullptr, + MBOReader* mbo=nullptr, const std::set* s=nullptr); ~MDClient(); diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 20a4f73da..a303c34ad 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -3,6 +3,7 @@ #include "MemorySegmentUtils.h" #include "PlatformLoader.h" #include "SecurityCache.h" +#include "MBOProcessor.h" #include "lib/config/config.h" #include "lib/utils/SharedList.h" #include "lib/utils/WaitFreeQueue.h" @@ -535,6 +536,7 @@ class MDConsumerImpl { Mutex _workmutex; vector> _instances; std::shared_ptr _sci; + MBOReader* _mbo; map> _deferred_handles; vector _callbacks; mutable SecurityCache _cache; @@ -545,10 +547,12 @@ public: const set& symbols, SecurityCache& cache, api::ISyntheticSecurityService* ss, + MBOReader* mbo, bool shared, co2::Platform platform) : _workmutex(), _instances(), _sci(), + _mbo(mbo), _deferred_handles(), _callbacks(), _cache(cache) @@ -625,7 +629,7 @@ public: for (auto& inst : _instances) { if (inst->find_handle(key, &cold, &hot)) { MDConsumerHandle handle; - handle.init(inst->cold_header(), *cold, hot); + handle.init(inst->cold_header(), *cold, hot, _mbo); return handle; } } @@ -657,9 +661,15 @@ public: } // Erase anything that wasn't found from the resulting map + // Set the mbo pointer on anything else for (auto it = result.begin(); it != result.end(); ) { - if (!it->second.is_initialized()) it = result.erase(it); - else ++it; + if (!it->second.is_initialized()) { + it = result.erase(it); + } + else { + it->second.set_mbo(_mbo); + ++it; + } } return result; @@ -910,6 +920,7 @@ public: MDConsumer::MDConsumer( SecurityCache& cache, api::ISyntheticSecurityService* ss, + MBOReader* mbo, const std::set* symbols, bool shared, co2::Platform platform) : _pimpl() { @@ -920,7 +931,7 @@ MDConsumer::MDConsumer( symbols = &all_symbols; } - _pimpl.reset(new MDConsumerImpl(*symbols, cache, ss, shared, platform)); + _pimpl.reset(new MDConsumerImpl(*symbols, cache, ss, mbo, shared, platform)); } MDConsumer::~MDConsumer() {} diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 02c762a8f..103e8bb19 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -26,6 +26,7 @@ class MDConsumerImpl; class MDConsumerInst; class MDConsumerChannel; class SecurityCache; +class MBOReader; class MDConsumer { @@ -33,6 +34,7 @@ public: MDConsumer( SecurityCache&, api::ISyntheticSecurityService* ss, + MBOReader* mbo=nullptr, const std::set* sym = nullptr, bool shared=true, co2::Platform platform=co2::PLATFORM_UNSET); diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index 0218be0eb..616480af4 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -14,7 +14,7 @@ namespace md { MDConsumerHandle::MDConsumerHandle() { memset(this, 0, sizeof(MDConsumerHandle)); _type = HANDLETYPE_VOID; - static_assert(sizeof(MDConsumerHandle) <= 32, ""); + static_assert(sizeof(MDConsumerHandle) <= 40, ""); } MDConsumerHandle::MDConsumerHandle(const std::shared_ptr& def) { memset(this, 0, sizeof(MDConsumerHandle)); @@ -33,9 +33,10 @@ MDConsumerHandle::MDConsumerHandle(const MDConsumerHandle& other) { MDConsumerHandle::~MDConsumerHandle() {} -void MDConsumerHandle::init(const ColdHeader& cheader, const ColdEntry& centry, const void* pvhot) { +void MDConsumerHandle::init(const ColdHeader& cheader, const ColdEntry& centry, const void* pvhot, const MBOReader* mbo) { set_cold(&cheader, ¢ry); set_hot(pvhot); + set_mbo(mbo); } TimeStamp copy_bbo(const HotEntry& hot, PriceSize& out_bid, PriceSize& out_ask) { diff --git a/src/lib/md/MDConsumerHandle.h b/src/lib/md/MDConsumerHandle.h index 38a7772eb..e85b6b93c 100644 --- a/src/lib/md/MDConsumerHandle.h +++ b/src/lib/md/MDConsumerHandle.h @@ -13,6 +13,8 @@ namespace bts { namespace md { class MDConsumerHandle; +class MBOReader; + struct SyntheticStrategy { std::unique_ptr strategy; api::Security security; @@ -141,6 +143,9 @@ TimeStamp optimistic_copy( return t; } +// +// MDConsumerHandle +// // Can be an invalid handle if you request a security which has not been discovered yet. class BTS_EXPORT MDConsumerHandle @@ -151,7 +156,7 @@ public: ~MDConsumerHandle(); MDConsumerHandle(const MDConsumerHandle& other); - void init(const ColdHeader&, const ColdEntry&, const void* hot); + void init(const ColdHeader&, const ColdEntry&, const void* hot, const MBOReader*); bool is_initialized() const { return _type != HANDLETYPE_VOID; } bool is_synthetic() const { return _type == HANDLETYPE_SYNTHETIC; } @@ -269,6 +274,7 @@ public: private: friend class MDConsumerInst; + friend class MDConsumerImpl; template HandleReadResult get_book(Read read, TimeStamp& time, uint32_t copydepth, PriceSize* bids, PriceSize* asks) const { @@ -326,6 +332,19 @@ private: _type = HANDLETYPE_BBO; } } + void set_mbo(const MBOReader* mbo) { + switch(_type) { + case HANDLETYPE_DEPTH: + book.mbo = mbo; + break; + case HANDLETYPE_BBO: + bbo.mbo = mbo; + break; + default: + //ok + break; + } + } enum HandleType : uint8_t { HANDLETYPE_VOID, @@ -337,11 +356,13 @@ private: const HotDepthEntry* hot; const ColdEntry* cdata; const ColdHeader* cheader; + const MBOReader* mbo; }; struct BBOHandle { const HotEntry* hot; const ColdEntry* cdata; const ColdHeader* cheader; + const MBOReader* mbo; }; struct SyntheticHandle { std::shared_ptr def; diff --git a/src/lib/md/test/Co2TestInstrument.cpp b/src/lib/md/test/Co2TestInstrument.cpp index 06d037b1c..0c2c2274a 100644 --- a/src/lib/md/test/Co2TestInstrument.cpp +++ b/src/lib/md/test/Co2TestInstrument.cpp @@ -43,7 +43,7 @@ MDProducerHandle Co2TestInstrument::producer() { MDConsumerHandle Co2TestInstrument::consumer() { MDConsumerHandle handle; - handle.init(*_cheader, *_cold, _hot.get()); + handle.init(*_cheader, *_cold, _hot.get(), nullptr/*mbo*/); return handle; } diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 3a9a5dcdd..d945fea8a 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -370,7 +370,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); vector consumer_securities; // Poll for security list - none visible until they have keys @@ -441,7 +441,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); // Add the on_security callback vector securities; @@ -481,7 +481,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -539,7 +539,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -600,7 +600,7 @@ public: // Symbol list for the consumer auto symbols = get_test_symbols(); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); // Add the on_security callback - callback fires for existing securities vector securities; @@ -628,7 +628,7 @@ public: // Symbol list for the consumer SecurityCache cache; auto symbols = get_test_symbols(); - MDClient client(nullptr, nullptr, &symbols); + MDClient client(nullptr, nullptr, nullptr, &symbols); MDConsumer consumer = client.current_consumer(); map products; consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { @@ -761,7 +761,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); set keys; consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) { keys.insert(security->id()); }); @@ -806,7 +806,7 @@ public: csymbols.insert(UserSymbol("S3", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S4", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &csymbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &csymbols, true, TEST_PLATFORM); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -853,7 +853,7 @@ public: csymbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); csymbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &csymbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &csymbols, true, TEST_PLATFORM); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -905,7 +905,7 @@ public: consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); consumer_symbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &consumer_symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &consumer_symbols, true, TEST_PLATFORM); size_t security_count = 0; // Indirection for check function @@ -974,7 +974,7 @@ public: set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &consumer_symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &consumer_symbols, true, TEST_PLATFORM); auto h1 = consumer.get_handle(k1); REQUIRE(h1.is_initialized()); auto h2 = consumer.get_handle(k2); @@ -1260,7 +1260,7 @@ struct Co2RelayTests { UserSymbol option_symbol("FOOX", VENUE_BTS, SECURITYTYPE_OPTION); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols); + MDConsumer consumer(cache, nullptr, nullptr, &symbols); return consumer; } @@ -1327,7 +1327,7 @@ struct Co2RelayTests { // todo: setup_query_server should use client.symbols() instead of it's own list // const auto symbols = setup_symbols(); - MDClient client(nullptr, nullptr, &symbols); + MDClient client(nullptr, nullptr, nullptr, &symbols); auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices @@ -1460,7 +1460,7 @@ struct Co2RelayTests { vector producers; Entitlements entitlements; const auto symbols = setup_symbols(); - MDClient client(nullptr, nullptr, &symbols); + MDClient client(nullptr, nullptr, nullptr, &symbols); auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices to all levels @@ -1638,7 +1638,7 @@ namespace ImpliedTests { UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols); + MDConsumer consumer(cache, nullptr, nullptr, &symbols); vector consumer_securities; // Poll for security list - none visible until they have keys size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 3b788c4a2..3dd41a86c 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -63,7 +63,7 @@ int Run::execute(std::vector const &) { md::run_market_datapublishers(&mbo.writer()); api::synthetic::SyntheticSecurityService synthetic_service(sc); - md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service /*, mbo*/); + md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service, &mbo.reader()); md::QueryServerPtr md_query_server( From a4575e3fa315503176916e39510e63603c6bb5b9 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 15:38:26 -0400 Subject: [PATCH 008/100] Fix windows build --- src/lib/md/MDConsumerInterop.cpp | 2 +- src/lib/md/test/Co2ManagedTestHelper.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/md/MDConsumerInterop.cpp b/src/lib/md/MDConsumerInterop.cpp index 5027f3d75..17a36b072 100644 --- a/src/lib/md/MDConsumerInterop.cpp +++ b/src/lib/md/MDConsumerInterop.cpp @@ -32,7 +32,7 @@ CAPI MDConsumer* create_md_consumer(BlitUserSymbol* usersymbols, uint32_t count, for (size_t x = 0; x < count; ++x) { symbols.insert(UserSymbol(usersymbols[x].symbol, usersymbols[x].exchange, usersymbols[x].type)); } - return new MDConsumer(cache, nullptr/*synthetics*/, &symbols, shared); + return new MDConsumer(cache, nullptr/*synthetics*/, nullptr /*mbo*/, &symbols, shared); } CAPI void destroy_md_consumer(MDConsumer* consumer) { diff --git a/src/lib/md/test/Co2ManagedTestHelper.cpp b/src/lib/md/test/Co2ManagedTestHelper.cpp index 4d69543ac..30caf3bde 100644 --- a/src/lib/md/test/Co2ManagedTestHelper.cpp +++ b/src/lib/md/test/Co2ManagedTestHelper.cpp @@ -26,7 +26,7 @@ extern "C" BTS_EXPORT void co2_get_1234_handle(size_t& size, MDConsumerHandle& auto pheader = reinterpret_cast(&buffer[0]); pheader->has_depth = false; size = sizeof(MDConsumerHandle); - handle.init(*pheader, *pcold, phot.get()); + handle.init(*pheader, *pcold, phot.get(), nullptr); // Copy the addresses for comparison *hotaddr = (uintptr_t)phot.get(); *coldaddr = (uintptr_t)pcold.get(); From ad6fc6d23b66da40ed5db9f89be2e455e41dbf26 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 15:38:54 -0400 Subject: [PATCH 009/100] Cannot have a const pointer returned to the pool --- src/lib/md/MBOProcessor.cpp | 4 ++-- src/lib/md/MBOProcessor.h | 4 ++-- src/lib/utils/ConcurrentPool.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index de7fd5363..4f89023a6 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -62,7 +62,7 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot* snapshot) { MBOReader::MBOReader(std::shared_ptr& me): _me(me) { } MBOReader::~MBOReader() { } -const MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { +MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { // get a result eagerly so we don't have to do it while accessor has the entry locked auto result = _me->pool.allocate(); MapType::const_accessor accessor; @@ -74,7 +74,7 @@ const MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { } return result; } -void MBOReader::release_snapshot(MBOSnapshot**const s) const { +void MBOReader::release_snapshot(MBOSnapshot**s) const { _me->pool.release(s); } diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 0c02f10dc..04270e8bd 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -60,8 +60,8 @@ class MBOReader { MBOReader(std::shared_ptr&); ~MBOReader(); public: - const MBOSnapshot* acquire_snapshot(SecurityKey key) const; - void release_snapshot(MBOSnapshot**const) const; + MBOSnapshot* acquire_snapshot(SecurityKey key) const; + void release_snapshot(MBOSnapshot**) const; }; class MBOProcessor { diff --git a/src/lib/utils/ConcurrentPool.h b/src/lib/utils/ConcurrentPool.h index 96802c5dc..91ad99095 100644 --- a/src/lib/utils/ConcurrentPool.h +++ b/src/lib/utils/ConcurrentPool.h @@ -41,7 +41,7 @@ class ConcurrentPool { _all.push_back(std::move(buffer)); } - void release(T**const item) { + void release(T** item) { if (*item) { (*item)->reset(); _free.push(*item); From 69f9f2791e0479577eae13bf7605f9a717c74501 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 15:39:22 -0400 Subject: [PATCH 010/100] SPARK-2203 --- src/lib/md/MDConsumerHandle.cpp | 11 +++++++++++ src/lib/md/MDConsumerHandle.h | 3 +++ 2 files changed, 14 insertions(+) diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index 616480af4..af2729ac2 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -1,5 +1,6 @@ #ifndef NO_CPP0X #include "MDConsumerHandle.h" +#include "MBOProcessor.h" #include "lib/domain_model/Security.h" #include @@ -372,6 +373,16 @@ CoconutSecurityStatus MDConsumerHandle::status() const { never; } +std::unique_ptr> MDConsumerHandle::get_mbo() { + if (_type == HANDLETYPE_DEPTH) { + MBOSnapshot* snapshot = book.mbo->acquire_snapshot(SecurityKey(key())); + if (snapshot != nullptr) { + return {snapshot,[mbo=book.mbo](auto* s) { mbo->release_snapshot(&s); }}; + } + } + return nullptr; + } + }}//namespace diff --git a/src/lib/md/MDConsumerHandle.h b/src/lib/md/MDConsumerHandle.h index e85b6b93c..72c387fe8 100644 --- a/src/lib/md/MDConsumerHandle.h +++ b/src/lib/md/MDConsumerHandle.h @@ -14,6 +14,7 @@ namespace md { class MDConsumerHandle; class MBOReader; +struct MBOSnapshot; struct SyntheticStrategy { std::unique_ptr strategy; @@ -272,6 +273,8 @@ public: return *this; } + std::unique_ptr> get_mbo(); + private: friend class MDConsumerInst; friend class MDConsumerImpl; From f513f7d9f2561825f51dd44657a10829b331a4f3 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 1 Apr 2019 12:11:03 -0400 Subject: [PATCH 011/100] SPARK-2204 mbo api in QueryServer --- src/lib/api/services/MarketDataRelay.proto | 16 ++++- src/lib/md/MDConsumerHandle.cpp | 2 +- src/lib/md/MDConsumerHandle.h | 2 +- src/lib/md/relay/QueryServer.cpp | 73 ++++++++++++++++++++++ 4 files changed, 89 insertions(+), 4 deletions(-) diff --git a/src/lib/api/services/MarketDataRelay.proto b/src/lib/api/services/MarketDataRelay.proto index d83daf33b..d5b949673 100644 --- a/src/lib/api/services/MarketDataRelay.proto +++ b/src/lib/api/services/MarketDataRelay.proto @@ -17,17 +17,29 @@ message ExpirationNameList { message ExpirationNameListReply { repeated ExpirationNameList securities = 2; } - message SecurityKeyList { repeated uint32 keys = 1; } +message OrderQuery { + uint32 security_id = 1; + uint32 order_id = 2; +} +message PIQRequest { + repeated OrderQuery orders = 1; +} +message PIQResponse { + // order id => position + map position = 1; +} + service MarketDataRelay { option (bts.api.service_description)="Poll the market data service for updates"; rpc requestmd(bts.co2.MDRequest) returns (bts.co2.MDReply); rpc request_products(bts.api.VoidMessage) returns (ProductListReply); rpc request_names(bts.api.VoidMessage) returns (ExpirationNameListReply); - rpc get_securities(SecurityKeyList) returns (bts.api.SecurityList); + + rpc get_piq(PIQRequest) returns (PIQResponse); } diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index af2729ac2..40efeded5 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -373,7 +373,7 @@ CoconutSecurityStatus MDConsumerHandle::status() const { never; } -std::unique_ptr> MDConsumerHandle::get_mbo() { +std::unique_ptr> MDConsumerHandle::get_mbo() const { if (_type == HANDLETYPE_DEPTH) { MBOSnapshot* snapshot = book.mbo->acquire_snapshot(SecurityKey(key())); if (snapshot != nullptr) { diff --git a/src/lib/md/MDConsumerHandle.h b/src/lib/md/MDConsumerHandle.h index 72c387fe8..506e64f7b 100644 --- a/src/lib/md/MDConsumerHandle.h +++ b/src/lib/md/MDConsumerHandle.h @@ -273,7 +273,7 @@ public: return *this; } - std::unique_ptr> get_mbo(); + std::unique_ptr> get_mbo() const; private: friend class MDConsumerInst; diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index e8331308b..76771c20c 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -9,6 +9,7 @@ #include "../MDConsumer.h" #include "../MDClient.h" #include "../PlatformLoader.h" +#include "../MBOProcessor.h" #include "lib/logging/logging.h" @@ -129,6 +130,27 @@ struct MDConsumerGroup { }; +class HandleIndex { + mutable std::mutex _mutex; + std::unordered_map _index; +public: + template + auto lock(CB cb) { + std::lock_guard guard(_mutex); + return cb(_index); + } + template + auto lock(CB cb) const { + std::lock_guard guard(_mutex); + return cb(_index); + } + void clear() { + lock([](std::unordered_map& index) { + index.clear(); + }); + } +}; + void convert(PriceSize ps, PriceSizeMessage* pps) { pps->set_mantissa(ps.mantissa()); @@ -168,6 +190,7 @@ class QueryServerImpl : public IQueryServer { MDConsumer _consumer; int _sub; boost::container::flat_map> _groups; + HandleIndex _handle_index; public: QueryServerImpl(MDClient& client, metrics::MetricsContainer& stats) : @@ -198,6 +221,10 @@ public: const auto& pgroup = iter->second; bts_assert(pgroup); pgroup->add_security(handle); + // Also insert into handle index + _handle_index.lock([handle](auto& index) { + index[SecurityKey(handle.key())] = handle; + }); } }); } @@ -211,6 +238,7 @@ public: for (auto& pair : _groups) { pair.second->set_consumer(_consumer); } + _handle_index.clear(); subscribe_securities(); utils::Dispatcher::background().post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000); } @@ -392,6 +420,42 @@ public: } } + void get_piq(const PIQRequest& req, PIQResponse* result) const { + // for each pair in request + // look up the mbo + // search for the order in the mbo and add it to the result if found + for(const auto& entry : req.orders()) { + auto id = SecurityKey(entry.security_id()); + auto handle = _handle_index.lock( + [id](auto& index){ + auto iter = index.find(id); + if (iter != index.end()) { + return iter->second; + } + return MDConsumerHandle(); + }); + if (handle.is_initialized()) { + // get mbo + auto mbo = handle.get_mbo(); + // search for order + IntPrice price = {}; + size_t position = 0; + // guess at side + for (size_t x = 0; x < mbo->bids.size(); ++x) { + // append to result + auto& oo = (*mbo)[x]; + // note: needs price + + // if (oo. + } + for (size_t x = 0; x < mbo->asks.size(); ++x) { + // append to result + } + (void)result; + } + } + } + }; struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { @@ -487,6 +551,15 @@ struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { reply(*preply); }); } + + virtual void get_piq(const api::AuthToken&, const PIQRequest& request, const std::function& reply) override { + _threadpool.post([=]() { + google::protobuf::Arena arena; + PIQResponse* presponse = google::protobuf::Arena::CreateMessage(&arena); + _db.get_piq(request, presponse); + reply(*presponse); + }); + } }; struct QueryServer { From 086e90acbe8226a58afab9b068448952cd263d7d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 2 Apr 2019 11:13:28 -0400 Subject: [PATCH 012/100] Move queryserver to linux only --- src/lib/md/test/md2_tests.cpp | 34 ++++--------------------- src/lib/md/test/md_test_utils.cpp | 41 +++++++++++++++++++++++++++++++ src/lib/md/test/md_test_utils.h | 2 ++ 3 files changed, 48 insertions(+), 29 deletions(-) create mode 100644 src/lib/md/test/md_test_utils.cpp diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index d945fea8a..d9da74ddd 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -1,3 +1,4 @@ +#ifndef _WIN32 #include "catch/catch.hpp" #include "../SettlementRecorder.h" #include "../coconut2.h" @@ -15,6 +16,7 @@ #include "lib/config/protos/MarketData.pb.h" #include "lib/utils/Sqlite_utils.h" +#include "md_test_utils.h" #include #include @@ -55,29 +57,8 @@ Platform TEST_PLATFORM = PLATFORM_SIM; // // Helper functions for Producer/Consumer tests // -vector create_test_securities(MDProducer& producer, const char* postfix ="") { - FixedFutureSecurity ffs; - FixedOptionSecurity fos; - ffs.symbol("FOO"+string(postfix)); - fos.symbol("FOOX"+string(postfix)); - ffs.exchange = VENUE_BTS; - fos.exchange = VENUE_BTS; - ffs.key = invalid_security_key; - fos.key = invalid_security_key; - ffs.expiration_year = fos.expiration_year = 2015; - ffs.expiration_day = fos.expiration_day = 27; - fos.strike = 1000; - fos.call_put = PUTORCALL_CALL; - // Add 12 expirations to 2 different shared memory regions - vector result; - for (uint32_t x = 1; x <= 12; ++x) { - ffs.expiration_month = fos.expiration_month = x; - result.push_back(producer.add_security(ffs)); - result.push_back(producer.add_security(fos)); - // In a real app these securities would now be passed to the SecurityIDRecorder - } - return result; -} + +std::vector create_test_securities(MDProducer& producer, const char* postfix = ""); set get_test_symbols(const char* postfix="") { set symbols; @@ -146,12 +127,6 @@ Security mkspread(string symbol, security_key key, Security leg1, Security leg2, return mkspread(symbol, key, vlegs); } -void assign_ids(vector::iterator begin, vector::iterator end, security_key start_key = 1000) { - for (; begin != end; ++begin) { - (*begin).set_key(++start_key); - } -} - class Co2Tests { @@ -2417,3 +2392,4 @@ TEST_CASE("IntPrice", "[md]") { }}} +#endif \ No newline at end of file diff --git a/src/lib/md/test/md_test_utils.cpp b/src/lib/md/test/md_test_utils.cpp new file mode 100644 index 000000000..bc2848c16 --- /dev/null +++ b/src/lib/md/test/md_test_utils.cpp @@ -0,0 +1,41 @@ +#include "md_test_utils.h" +#include "../MDProducer.h" + +#include +#include + +namespace bts { +namespace md { +namespace test { + +std::vector create_test_securities(MDProducer& producer, const char* postfix ="") { + FixedFutureSecurity ffs; + FixedOptionSecurity fos; + ffs.symbol("FOO"+std::string(postfix)); + fos.symbol("FOOX"+std::string(postfix)); + ffs.exchange = api::VENUE_BTS; + fos.exchange = api::VENUE_BTS; + ffs.key = invalid_security_key; + fos.key = invalid_security_key; + ffs.expiration_year = fos.expiration_year = 2015; + ffs.expiration_day = fos.expiration_day = 27; + fos.strike = 1000; + fos.call_put = api::PUTORCALL_CALL; + // Add 12 expirations to 2 different shared memory regions + std::vector result; + for (uint32_t x = 1; x <= 12; ++x) { + ffs.expiration_month = fos.expiration_month = x; + result.push_back(producer.add_security(ffs)); + result.push_back(producer.add_security(fos)); + // In a real app these securities would now be passed to the SecurityIDRecorder + } + return result; +} + +void assign_ids(std::vector::iterator begin, std::vector::iterator end, security_key start_key) { + for (; begin != end; ++begin) { + (*begin).set_key(++start_key); + } +} + +}}}//namespace \ No newline at end of file diff --git a/src/lib/md/test/md_test_utils.h b/src/lib/md/test/md_test_utils.h index 23a82070f..7eccddc3c 100644 --- a/src/lib/md/test/md_test_utils.h +++ b/src/lib/md/test/md_test_utils.h @@ -2,6 +2,7 @@ #include "lib/md/IntPrice.h" #include "lib/md/PriceSize.h" +#include "../MDProducerHandle.h" #include namespace bts { @@ -28,5 +29,6 @@ Book compute_synthetic_book(const api::Security& security, const std::vector::iterator begin, std::vector::iterator end, security_key start_key = 1000); }}}//namespace \ No newline at end of file From df14522433c66bdbed5cde15b0fd33b9ea9f4c84 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 2 Apr 2019 13:07:42 -0400 Subject: [PATCH 013/100] SPARK-2204 QueryServer api --- src/lib/CMakeDeps_win.cmake | 2 +- src/lib/md/MBOProcessor.h | 1 + src/lib/md/relay/QueryServer.cpp | 47 +++++++++++++------ .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 4 +- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/lib/CMakeDeps_win.cmake b/src/lib/CMakeDeps_win.cmake index 879f19a67..18c633341 100644 --- a/src/lib/CMakeDeps_win.cmake +++ b/src/lib/CMakeDeps_win.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 04270e8bd..bbb8c7e21 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -15,6 +15,7 @@ using OrderKey = utils::ValueWrapper; struct SnapshotOrder { OrderKey id; + IntPrice price; int32_t size; }; struct MBOSnapshot { diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 76771c20c..816f55e78 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -1,4 +1,5 @@ #ifndef NO_CPP0X +#ifndef _WIN32 #include "QueryServer.h" #include "lib/config/config.h" #include "lib/utils/Dispatcher.h" @@ -31,6 +32,7 @@ #include #include #include +#include #include #include @@ -420,15 +422,36 @@ public: } } + static std::optional search_orders(const OrderKey oid, const std::vector& orders) { + IntPrice price = {}; + size_t position = 0; + for (size_t x = 0; x < orders.size(); ++x) { + const auto& oo = orders[x]; + if (oo.id == oid) { + return { position }; + } + else if (price == oo.price) { + position += 1; + } + else { + price = oo.price; + position = 0; + } + } + return std::nullopt; + } + void get_piq(const PIQRequest& req, PIQResponse* result) const { // for each pair in request // look up the mbo // search for the order in the mbo and add it to the result if found + auto& rmap = *result->mutable_position(); for(const auto& entry : req.orders()) { - auto id = SecurityKey(entry.security_id()); + auto order_key = OrderKey(entry.order_id()); + auto security = SecurityKey(entry.security_id()); auto handle = _handle_index.lock( - [id](auto& index){ - auto iter = index.find(id); + [security](auto& index){ + auto iter = index.find(security); if (iter != index.end()) { return iter->second; } @@ -438,20 +461,13 @@ public: // get mbo auto mbo = handle.get_mbo(); // search for order - IntPrice price = {}; - size_t position = 0; - // guess at side - for (size_t x = 0; x < mbo->bids.size(); ++x) { - // append to result - auto& oo = (*mbo)[x]; - // note: needs price - - // if (oo. + auto search_result = search_orders(order_key, mbo->bids); + if (!search_result) { + search_result = search_orders(order_key, mbo->asks); } - for (size_t x = 0; x < mbo->asks.size(); ++x) { - // append to result + if (search_result) { + rmap[order_key.value()] = search_result.value(); } - (void)result; } } } @@ -628,3 +644,4 @@ std::unique_ptr create_query_server(MDClient& mdc, metrics::Metric }}//namespace #endif +#endif diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index f25450a6d..bd943250f 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -129,10 +129,10 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con auto* snapshot = _mbo->allocate_snapshot(); // copy from onixs book for (const auto& bid : book.bids()) { - snapshot->bids.push_back({ OrderKey(bid.id()), bid.quantity() }); + snapshot->bids.push_back({ OrderKey(bid.id()), convert_price(bid.price(), 0), bid.quantity() }); } for (const auto& ask : book.offers()) { - snapshot->asks.push_back({ OrderKey(ask.id()), ask.quantity() }); + snapshot->asks.push_back({ OrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); } _mbo->publish_snapshot(SecurityKey(key), snapshot); } From 8793c6c527744f4aa4e4ea94a2c47b5357743ece Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 2 Apr 2019 15:00:02 -0400 Subject: [PATCH 014/100] Replace ManyConsole with CommandLineParser --- src/client/sparkcmd/Cmd/ConfigCheckCommand.cs | 21 +++--- src/client/sparkcmd/Cmd/ConsoleCommandEx.cs | 20 ------ .../sparkcmd/Cmd/ScriptClientCommand.cs | 33 +++++---- src/client/sparkcmd/Cmd/ServiceTail.cs | 34 ++++----- src/client/sparkcmd/Cmd/ServiceTestCommand.cs | 29 ++++---- .../sparkcmd/Cmd/SettingsManagerCommand.cs | 11 ++- src/client/sparkcmd/Cmd/TestCommand.cs | 71 +++++++++++-------- src/client/sparkcmd/Program.cs | 29 ++++++-- src/client/sparkcmd/sparkcmd.csproj | 7 +- 9 files changed, 139 insertions(+), 116 deletions(-) delete mode 100644 src/client/sparkcmd/Cmd/ConsoleCommandEx.cs diff --git a/src/client/sparkcmd/Cmd/ConfigCheckCommand.cs b/src/client/sparkcmd/Cmd/ConfigCheckCommand.cs index 432fce36b..7284e48db 100644 --- a/src/client/sparkcmd/Cmd/ConfigCheckCommand.cs +++ b/src/client/sparkcmd/Cmd/ConfigCheckCommand.cs @@ -1,14 +1,19 @@ -using ManyConsole; using System.Linq; +using System; +using System.Collections.Generic; +using CommandLine; namespace bts { - public class Configcheck : ConsoleCommand { - public Configcheck() { - IsCommand("configcheck", "Ye olde Configcheck"); - HasAdditionalArguments(null, " [Keys to check]"); - } - public override int Run(string[] args) { - mbts.config.Configuration.configcheck(args.Length, args.ToArray()); + + [Verb("configcheck", HelpText="Ye olde Configcheck")] + public class ConfigcheckOptions { + [Value(0, MetaName="keys", HelpText="Keys to print")] + public IEnumerable keys { get; set; } + } + public class Configcheck { + public static int Run(ConfigcheckOptions options) { + var keys = options.keys.ToArray(); + mbts.config.Configuration.configcheck(keys.Length, keys); return 0; } } diff --git a/src/client/sparkcmd/Cmd/ConsoleCommandEx.cs b/src/client/sparkcmd/Cmd/ConsoleCommandEx.cs deleted file mode 100644 index 18c73075d..000000000 --- a/src/client/sparkcmd/Cmd/ConsoleCommandEx.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ManyConsole { - public static class ConsoleCommandEx { - public static string get_argument(this ConsoleCommand _, string arg_name) { - var arg = Environment.GetCommandLineArgs().FirstOrDefault(str => str.StartsWith("--"+arg_name)); - if (arg != null) { - var parts = arg.Split(new[] { '=' }, 2); - if (parts.Length > 1) { - return parts[1]; - } - } - return ""; - } - } -} diff --git a/src/client/sparkcmd/Cmd/ScriptClientCommand.cs b/src/client/sparkcmd/Cmd/ScriptClientCommand.cs index 18f0c18c9..5f00cf392 100644 --- a/src/client/sparkcmd/Cmd/ScriptClientCommand.cs +++ b/src/client/sparkcmd/Cmd/ScriptClientCommand.cs @@ -6,32 +6,37 @@ using System.Reflection; using System.Threading.Tasks; using System.Reactive.Linq; using System.Reactive; -using ManyConsole; using System.Linq; +using CommandLine; namespace sparkcmd { - public class ScriptClientCommand : ConsoleCommand { + [Verb("script", HelpText="Remote control for shell diagnostic host")] + public class ScriptClientOptions { + [Option('s', "server", HelpText="Run a script server for testing.")] + public bool server { get; set; } + + [Option('h', "host", HelpText="Connect to {host}")] + public string host { get; set; } + + [Option('f', "fromfile", HelpText="Read commands from file instead of stdin")] + public string file { get; set; } + + } + public class ScriptClientCommand { bool _server = false; string _host; string _logfile; - public ScriptClientCommand() { - IsCommand("script", "Remote control for shell diagnostic host"); - HasOption("s|server", "Run a script server for testing.", x => _server = true); - HasOption("h|host", "Connect to {host}", h => _host = h); - HasOption("f|fromfile", "Read commands from file instead of stdin", f => _logfile = f); - } - public override int Run(string[] args) { - if (_server) { + public static int Run(ScriptClientOptions o) { + if (o.server) { ScriptClient.runserver(); } else { - var host = string.IsNullOrEmpty(_host) ? args.First() : _host; - if (string.IsNullOrEmpty(_logfile) && File.Exists(_logfile)) { - ScriptClient.start(host, File.OpenRead(_logfile)); + if (string.IsNullOrEmpty(o.file) && File.Exists(o.file)) { + ScriptClient.start(o.host, File.OpenRead(o.file)); } else { - ScriptClient.start(host, Console.OpenStandardInput()); + ScriptClient.start(o.host, Console.OpenStandardInput()); } } return 0; diff --git a/src/client/sparkcmd/Cmd/ServiceTail.cs b/src/client/sparkcmd/Cmd/ServiceTail.cs index 7ebbb0c92..926bc5a6c 100644 --- a/src/client/sparkcmd/Cmd/ServiceTail.cs +++ b/src/client/sparkcmd/Cmd/ServiceTail.cs @@ -1,6 +1,6 @@ using bts.api.core; using bts.test; -using ManyConsole; +using CommandLine; using System; using System.Collections.Generic; using System.Linq; @@ -8,24 +8,26 @@ using System.Text; using System.Threading.Tasks; namespace bts { - class TailService : ConsoleCommand { + class TailServiceOptions { + [Option('m', "method", Required=true, HelpText = "'[ServiceName].[Method]' to call")] + public string method { get;set; } + + [Option('t', "time", HelpText="Include timestamp")] + public bool time { get; set; } + + [Option('p', "path", HelpText="Path inside the row to use for display. Ex: 'Id' or 'Name' ")] + public string path { get; set; } + + [Option('a', "all", HelpText="Include 'all' rows on every print")] + public bool all { get; set; } + } + class TailService { static bool _all = false; static bool _timestamp = false; - public TailService() { - void ignore(string _) { } - IsCommand("tailservice", "Tail updates to named service api. Ex: tail --method=SyntheticSecurityService.metadata"); - HasRequiredOption("method", "Service.method to tail", ignore); - HasOption("time", "Include timestamp", _ => _timestamp = true); - // The following are not implemented. If you need them get to work. - HasOption("path", "Path inside the row to use for display. Ex: 'Id' or 'Name' ", ignore); - HasOption("all", "Invlude 'all' rows on every print", _ => _all = true); - HasAdditionalArguments(null, " [nothing]"); - } - public override int Run(string[] remainingArguments) { - var arg = this.get_argument("method"); - var split = arg.Split('.'); + public static int Run(TailServiceOptions o) { + var split = o.method.Split('.'); if (split.Length != 2) { - Console.WriteLine("Try --method=[ServiceName].[Method]"); + Console.WriteLine("Try --method [ServiceName].[Method]"); return -1; } tail(split[0], split[1]); diff --git a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs index 2b2d1acd0..331469b30 100644 --- a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs +++ b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs @@ -2,30 +2,29 @@ using bts.api; using bts.api.core; using bts.api.synthetic; using bts.test; -using ManyConsole; +using CommandLine; using System; using System.Threading; namespace bts { - public class ServiceTest : ConsoleCommand { - bool _synthetic; - bool _user; - bool _admin; - public ServiceTest() { - IsCommand("service", "Put code here for testing during service development"); - HasOption("synthetic", "Run synthetic test", _ => _synthetic = true); - HasOption("user", "user test", _ => _user = true); - HasOption("admin", "admin test", _ => _admin = true); - } - public override int Run(string[] remainingArguments) { - if (_synthetic) { + public class ServiceTestOptions { + [Option(nameof(synthetic))] + public bool synthetic { get; set; } + [Option(nameof(user))] + public bool user { get; set; } + [Option(nameof(admin))] + public bool admin { get; set; } + } + public class ServiceTest { + public static int Run(ServiceTestOptions o) { + if (o.synthetic) { synthetic_test(); } - if (_user) { + if (o.user) { user_test(); } - if (_admin) { + if (o.admin) { admin_test(); } return 0; diff --git a/src/client/sparkcmd/Cmd/SettingsManagerCommand.cs b/src/client/sparkcmd/Cmd/SettingsManagerCommand.cs index 3c74b9c87..7d01ad08d 100644 --- a/src/client/sparkcmd/Cmd/SettingsManagerCommand.cs +++ b/src/client/sparkcmd/Cmd/SettingsManagerCommand.cs @@ -1,14 +1,13 @@ -using ManyConsole; using System.Reflection; +using CommandLine; namespace bts { - public class SettinsManagerCommand : ConsoleCommand { - public SettinsManagerCommand() { - IsCommand("SettingsManager", "Copy workspaces around"); - } + [Verb("settingsmanager", HelpText="Copy workspaces around")] + public class SettingsManagerOptions { } - public override int Run(string[] remainingArguments) { + public class SettingsManagerCommand { + public static int Run(SettingsManagerOptions _) { var asm = Assembly.Load("spark"); var type = asm.GetType("bts.settings.SettingsManagerTool"); var run = type.GetMethod("run", BindingFlags.Static | BindingFlags.Public); diff --git a/src/client/sparkcmd/Cmd/TestCommand.cs b/src/client/sparkcmd/Cmd/TestCommand.cs index b8a0f790c..89dbf7498 100644 --- a/src/client/sparkcmd/Cmd/TestCommand.cs +++ b/src/client/sparkcmd/Cmd/TestCommand.cs @@ -1,6 +1,5 @@ using bts.test; -using ManyConsole; using System; using mbts; using System.Collections.Generic; @@ -8,47 +7,59 @@ using bts.api.test; using System.Threading.Tasks; using mbts.config; using System.Linq; +using CommandLine; namespace sparkcmd { - public class TestCommand : ConsoleCommand { - uint _timeout = 0; - uint _parallel = 1; - bool _list; - TestType test_type = TestType.Unit; + [Verb("test", HelpText="Run Tests")] + public class TestCommandOptions { - bool self_contained = false; + [Option('t', "timeout", HelpText="Timeout unit tets", Default=0)] + public uint timeout { get; set; } + [Option('l', "list", HelpText="List tests")] + public bool list { get; set; } + [Option('i', "insular", HelpText="Tests run sparkd and use their own config (if specified)")] + public bool self_contained { get; set; } - public TestCommand() { - IsCommand("test", "Run managed unit tests"); - // i comes thorugh as "t" which seems daft. - HasOption("t|timeout", "Timeout unit tests", i => _timeout = 10); - HasOption("n|parallel", "How many tests to run in parallel", i => _parallel = i); - HasOption("l|list", "List all tests", _ => _list = true); - HasOption("unit","Run unit tests", _ => test_type = TestType.Unit); - HasOption("system","Run system tests", _ => test_type = TestType.System); - HasOption("shell","Run shell tests", _ => test_type = TestType.Shell); - HasOption("i|insular|selfcontained", "Tests run sparkd and use their own config (if specified)", _ => self_contained = true); - HasAdditionalArguments(null, " [Test List]"); + // [Option("unit", HelpText="Run unit tests")] + // public bool unit { get; set; } + // [Option("system", HelpText="Run system tests")] + // public bool system { get; set; } + // [Option("shell", HelpText="Run shell tests")] + // public bool shell { get; set; } + + [Value(0, MetaName="TestType")] + public string type { get; set; } + + [Value(1, MetaName="Tests", HelpText="Test list")] + public IEnumerable tests { get; set; } + + } + public static class TestCommand { + static TestType test_type(string t) { + switch(t.ToLower()) { + case "unit": return TestType.Unit; + case "system": return TestType.System; + case "shell": return TestType.Shell; + } + return TestType.Unit; } - public override int Run(string[] tests) { - if (_list) { - list_tests(); + public static int Run(TestCommandOptions o) { + if (o.list) { + list_tests(test_type(o.type)); return 0; } - //Configuration.initialize_for_test(); - - if (_timeout == 0 && tests.Length == 1 && !self_contained) { + if (o.timeout == 0 && o.tests.Count() == 1 && !o.self_contained) { //System.Threading.Thread.Sleep(5000); using (var trr = new TestResultReceiver(true)) { - InProcessTestRunner.run_test_synchronously(tests[0], Enumerator, trr.get_reporter()); + InProcessTestRunner.run_test_synchronously(o.tests.First(), Enumerator, trr.get_reporter()); trr.print_summary(); return trr.ErrorCount; } } - else if (tests.Length > 0) { + else if (o.tests.Count() > 0) { return run_tests(Enumerator.match(tests)); } else { @@ -56,16 +67,16 @@ namespace sparkcmd { } } - Lazy _enumerator = new Lazy(() => new TestEnumerator("spark_test")); - TestEnumerator Enumerator => _enumerator.Value; + static Lazy _enumerator = new Lazy(() => new TestEnumerator("spark_test")); + static TestEnumerator Enumerator => _enumerator.Value; - int run_tests(IEnumerable tests) { + static int run_tests(IEnumerable tests) { var runner = new TestRunner(); // waits for all child processes to exit or time out return runner.run_tests(tests, _timeout, self_contained); } - void list_tests() { + static void list_tests(TestType test_type) { Console.WriteLine("All tests: "); foreach (var t in Enumerator.get_groups_of_type(test_type)) { ConsoleEx.WriteLine(t.name, ConsoleColor.Green); diff --git a/src/client/sparkcmd/Program.cs b/src/client/sparkcmd/Program.cs index 790308e27..14b3df9bb 100644 --- a/src/client/sparkcmd/Program.cs +++ b/src/client/sparkcmd/Program.cs @@ -2,16 +2,37 @@ using bts.api.core; using bts.api.synthetic; using bts.api.test; -using ManyConsole; using System; using System.Linq; +using CommandLine; +using bts; +using System.Collections.Generic; +using System.Linq.Expressions; +using System.Reflection; namespace sparkcmd { class Program { + + // static object option(Func run) {return run;} + // static Type argType(object arg) { + // return arg.GetType().GetGenericArguments().First(); + // } + // static int Run(Func,int> on_error, params object[] args) { + // var types = args.Select(argType).ToArray(); + // var parse = CommandLine.Parser.Default.GetType().GetMethod("ParseArguments", BindingFlags.Public).MakeGenericMethod(types); + // return 3; + // } static int Main(string[] args) { - mbts.config.Configuration.initialize(); - var commands = ConsoleCommandDispatcher.FindCommandsInSameAssemblyAs(typeof(Program)); - return ConsoleCommandDispatcher.DispatchCommand(commands, args, Console.Out); + // note: this can be made to infer the options type list from the list of delegates + return CommandLine.Parser.Default + .ParseArguments(args) + .MapResult( + Configcheck.Run, + ScriptClientCommand.Run, + TailService.Run, + ServiceTest.Run, + SettingsManagerCommand.Run, + errors => 1); } } } \ No newline at end of file diff --git a/src/client/sparkcmd/sparkcmd.csproj b/src/client/sparkcmd/sparkcmd.csproj index 9046c4610..1a8c7f607 100644 --- a/src/client/sparkcmd/sparkcmd.csproj +++ b/src/client/sparkcmd/sparkcmd.csproj @@ -21,7 +21,8 @@ - - - + + + + From c923be6ced4a9ba98c4c4255c9ac434b7c53781f Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 3 Apr 2019 16:11:07 -0400 Subject: [PATCH 015/100] SPARK 2206 add test command to print out PIQ data for all orders --- src/client/spark_core/utils/OrderUtilities.cs | 19 ++++++++++ src/client/sparkcmd/Cmd/ServiceTestCommand.cs | 38 +++++++++++++++++++ src/lib/api/services/MarketDataRelay.proto | 2 +- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 9 ++++- 4 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 src/client/spark_core/utils/OrderUtilities.cs diff --git a/src/client/spark_core/utils/OrderUtilities.cs b/src/client/spark_core/utils/OrderUtilities.cs new file mode 100644 index 000000000..b2042ad14 --- /dev/null +++ b/src/client/spark_core/utils/OrderUtilities.cs @@ -0,0 +1,19 @@ +namespace bts.api.orderman { + + public static class OrderUtilities { + public static NativeOrderStateImmutable? native_order_state(this OrderStateImmutable order) { + switch(order.CustomCase) { + case OrderState.CustomOneofCase.Market: + return order.Market.NativeState; + case OrderState.CustomOneofCase.Limit: + return order.Limit.NativeState; + case OrderState.CustomOneofCase.Stop: + return order.Stop.NativeState; + case OrderState.CustomOneofCase.StopLimit: + return order.StopLimit.NativeState; + default: + return null; + } + } + } +} \ No newline at end of file diff --git a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs index 2b2d1acd0..5450df0d7 100644 --- a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs +++ b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs @@ -1,10 +1,15 @@ using bts.api; using bts.api.core; +using bts.api.orderman; using bts.api.synthetic; +using bts.co2; using bts.test; using ManyConsole; using System; +using System.Reactive.Linq; using System.Threading; +using System.Threading.Tasks; +using System.Linq; namespace bts { public class ServiceTest : ConsoleCommand { @@ -28,10 +33,43 @@ namespace bts { if (_admin) { admin_test(); } + else { + mbo_test(); + } return 0; } + private static void mbo_test() { + using (var ctx = new ServiceContext()) { + var relay_svc = ctx.locate(); + var order_sv = ctx.locate(); + + var upd = order_sv.subscribei_orders().FirstAsync().Wait(); /* don't use Wait() */ + var orders = + upd.all + .Where(o => + o.State.Status == OrderStatus.Working && + o.State.native_order_state().HasValue) + .ToList(); + + var request = new PIQRequest(); + request.Orders.AddRange(orders.Select(o => + new OrderQuery { + SecurityId = o.Def.SecurityKey, + OrderId = UInt64.Parse(o.State.native_order_state().Value.ExchangeOrderId) })); + + Console.WriteLine($"Requesting positions for {request.Orders.Count} orders..."); + var result = relay_svc.get_piq(request); + + Console.WriteLine($"got response - {result.Position.Count} entries"); + foreach (var kvp in result.Position) { + Console.Write($"> {kvp.Key} => "); + ConsoleEx.WriteLine($"{kvp.Value}", ConsoleColor.Yellow); + } + } + } + private static void synthetic_test() { using (var ctx = new ServiceContext()) { var security = new Security { diff --git a/src/lib/api/services/MarketDataRelay.proto b/src/lib/api/services/MarketDataRelay.proto index d5b949673..b2284a2e3 100644 --- a/src/lib/api/services/MarketDataRelay.proto +++ b/src/lib/api/services/MarketDataRelay.proto @@ -23,7 +23,7 @@ message SecurityKeyList { message OrderQuery { uint32 security_id = 1; - uint32 order_id = 2; + uint64 order_id = 2; } message PIQRequest { repeated OrderQuery orders = 1; diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index c711993ac..cd4f49518 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -120,7 +120,14 @@ map> create_feed_engines(const vector Date: Thu, 4 Apr 2019 10:30:12 -0400 Subject: [PATCH 016/100] SPARK-2197 Add some MBO stats logged every 5 minutes --- src/lib/config/protos/toggles.proto | 4 +++ src/lib/md/MBOProcessor.cpp | 45 ++++++++++++++++++++++++++++- src/lib/utils/ConcurrentPool.h | 18 ++++++++++-- 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/lib/config/protos/toggles.proto b/src/lib/config/protos/toggles.proto index 5e1fbd736..3aeedd91b 100644 --- a/src/lib/config/protos/toggles.proto +++ b/src/lib/config/protos/toggles.proto @@ -34,4 +34,8 @@ message SoundNotifications { message VolumeAtPrice{ optional bool enabled = 1 [default = true]; +} + +message MBOStats { + optional bool enabled = 1 [default = true]; } \ No newline at end of file diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 4f89023a6..4d5f2a33d 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -1,9 +1,13 @@ #include "MBOProcessor.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/ConcurrentPool.h" +#include "lib/config/config.h" +#include "lib/config/protos/toggles.pb.h" #include #include +#include +#include #include @@ -20,11 +24,50 @@ struct SecurityKeyHashCompare { } }; +void print_pool_stats(const utils::ConcurrentPool& pool) { + size_t count = 0; + std::vector bid_sz,ask_sz; + pool.unsafe_iterate([&](const MBOSnapshot& s) { + if (s.bids.size() > 0 || s.asks.size() > 0) { + count++; + bid_sz.push_back(s.bids.size()); + ask_sz.push_back(s.asks.size()); + } + }); + + std::sort(bid_sz.begin(), bid_sz.end()); + std::sort(ask_sz.begin(), ask_sz.end()); + size_t bid_tot =0, ask_tot = 0; + for(size_t i : bid_sz) { bid_tot += i; } + for(size_t i : ask_sz) { ask_tot += i; } + + std::stringstream bid_percentiles, ask_percentiles; + for (double x : { 0.5, 0.75, 0.9, 0.95, 0.99 }) { + size_t index = count * x; + bid_percentiles << bid_sz[index] << " "; + ask_percentiles << ask_sz[index] << " "; + } + bid_percentiles << bid_sz[count-1] << " "; + ask_percentiles << ask_sz[count-1] << " "; + LOG(md,Info, + "MBO statistics: count: {} - %[50,75,90,95,99,100] bid: [{}] - ask: [{}]", + count, bid_percentiles.str(), ask_percentiles.str()); +} + typedef tbb::concurrent_hash_map MapType; struct MBOProcessorImpl { utils::ConcurrentPool pool; MapType snapshots; - MBOProcessorImpl() : pool(2048,512), snapshots() {} + utils::DispatcherTimer timer; + MBOProcessorImpl() : + pool(2048,512), + snapshots(), + timer(utils::Dispatcher::background(), 600*1000, [&]() { print_pool_stats(pool); }) + { + if (config::get().enabled()) { + timer.start(); + } + } }; // diff --git a/src/lib/utils/ConcurrentPool.h b/src/lib/utils/ConcurrentPool.h index 91ad99095..92d86e7ea 100644 --- a/src/lib/utils/ConcurrentPool.h +++ b/src/lib/utils/ConcurrentPool.h @@ -10,11 +10,12 @@ template class ConcurrentPool { tbb::concurrent_vector> _all; tbb::concurrent_queue _free; + size_t _initial_size; size_t _grow_size; - std::atomic _outstanding; + size_t _outstanding; public: ConcurrentPool(size_t initial_size=256, size_t grow_size=32): - _all(), _free(), _grow_size(grow_size), _outstanding(0) { + _all(), _free(), _initial_size(initial_size), _grow_size(grow_size), _outstanding(0) { reserve(initial_size); } @@ -34,6 +35,7 @@ class ConcurrentPool { void reserve(size_t count) { std::unique_ptr buffer {new T[count]}; + _outstanding += count; for (size_t x = 0; x < count; ++x) { T* temp = &buffer[x]; release(&temp); @@ -50,6 +52,18 @@ class ConcurrentPool { } } + template + void unsafe_iterate(CB cb) const { + for (auto iter = _all.begin(); iter != _all.end(); ++iter) { + size_t size = (iter == _all.begin()) ? _initial_size : _grow_size; + for (size_t x = 0; x < size; ++x) { + const std::unique_ptr& bucket = *iter; + const T* start = bucket.get(); + cb(start[x]); + } + } + } + }; }} //namespace From b664c42caee7cde642219ae7d507be7d5ce6a3ad Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 8 Apr 2019 09:52:45 -0400 Subject: [PATCH 017/100] SPARK-2179 streaming piq 1/2 --- src/lib/api/services/MBOService.proto | 15 +++ src/lib/api/services/MarketDataRelay.proto | 14 --- src/lib/config/protos/toggles.proto | 1 - src/lib/md/MBOServiceServer.cpp | 102 +++++++++++++++++++++ src/lib/md/MBOServiceServer.h | 25 +++++ src/lib/md/relay/QueryServer.cpp | 73 +-------------- src/lib/utils/LockBox.h | 24 +++++ 7 files changed, 168 insertions(+), 86 deletions(-) create mode 100644 src/lib/api/services/MBOService.proto create mode 100644 src/lib/md/MBOServiceServer.cpp create mode 100644 src/lib/md/MBOServiceServer.h create mode 100644 src/lib/utils/LockBox.h diff --git a/src/lib/api/services/MBOService.proto b/src/lib/api/services/MBOService.proto new file mode 100644 index 000000000..0a00fc22e --- /dev/null +++ b/src/lib/api/services/MBOService.proto @@ -0,0 +1,15 @@ +syntax="proto3"; +package bts.md; +option csharp_namespace="bts.md"; +option cc_enable_arenas=true; +import "lib/api/apioptions.proto"; +import "lib/api/apicore.proto"; + +message OrderPiq { + uint32 order_id = 1 [(bts.api.key)=true]; + uint32 position = 2; +} + +service MBOService { + rpc piq (bts.api.VoidMessage) returns (stream OrderPiq); +} \ No newline at end of file diff --git a/src/lib/api/services/MarketDataRelay.proto b/src/lib/api/services/MarketDataRelay.proto index b2284a2e3..197eecae0 100644 --- a/src/lib/api/services/MarketDataRelay.proto +++ b/src/lib/api/services/MarketDataRelay.proto @@ -21,18 +21,6 @@ message SecurityKeyList { repeated uint32 keys = 1; } -message OrderQuery { - uint32 security_id = 1; - uint64 order_id = 2; -} -message PIQRequest { - repeated OrderQuery orders = 1; -} -message PIQResponse { - // order id => position - map position = 1; -} - service MarketDataRelay { option (bts.api.service_description)="Poll the market data service for updates"; rpc requestmd(bts.co2.MDRequest) returns (bts.co2.MDReply); @@ -40,6 +28,4 @@ service MarketDataRelay { rpc request_products(bts.api.VoidMessage) returns (ProductListReply); rpc request_names(bts.api.VoidMessage) returns (ExpirationNameListReply); rpc get_securities(SecurityKeyList) returns (bts.api.SecurityList); - - rpc get_piq(PIQRequest) returns (PIQResponse); } diff --git a/src/lib/config/protos/toggles.proto b/src/lib/config/protos/toggles.proto index 3aeedd91b..6923cfbe9 100644 --- a/src/lib/config/protos/toggles.proto +++ b/src/lib/config/protos/toggles.proto @@ -35,7 +35,6 @@ message SoundNotifications { message VolumeAtPrice{ optional bool enabled = 1 [default = true]; } - message MBOStats { optional bool enabled = 1 [default = true]; } \ No newline at end of file diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp new file mode 100644 index 000000000..0603d3951 --- /dev/null +++ b/src/lib/md/MBOServiceServer.cpp @@ -0,0 +1,102 @@ +#ifndef _WIN32 +#include "MBOServiceServer.h" + +#include "lib/api/services/OrderService.client.pb.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/TimeStamp.h" +#include "MDClient.h" + +#include "lib/utils/LockBox.h" + +namespace bts { +namespace md { + +// Simple class to handle MD recovery and maintain an index of SecurityKey=>MDConsumerHandle +class HandleIndex { + MDClient& _client; + int _sub; + std::optional _consumer; + utils::LockBox> _index; + +public: + HandleIndex(md::MDClient& client): + _client(client), + _sub(0), + _consumer(std::nullopt), + _index() { + _sub = _client.on_consumer().subscribe([=](const MDConsumer& c) { on_consumer(c); }); + on_consumer(_client.current_consumer()); + } + + ~HandleIndex() { _client.on_consumer().unsubscribe(_sub); } + +private: + void on_consumer(const MDConsumer& c) { + MDConsumer temp(c); + if (temp.all_heartbeats_ok()) { + auto existing = _consumer ? _consumer->id() : 0; + LOG(md,Info, "Updateing MBOService MDConsumer. Old {} new {}", existing, temp.id()); + // clear the index + _index.lock([](auto& index) { index.clear(); }); + // Swap to new consumer + if (_consumer) { + _consumer->swap(temp); + } + else { _consumer = temp; } + // Subscribe to on_security + auto id = _consumer->id(); + _consumer->on_security([this,id](const SecurityPtr&, MDConsumerHandle handle) { + if (_consumer->id() == id) { + _index.lock([handle](auto& index) { + index[SecurityKey(handle.key())] = handle; + }); + } + }); + } + } +}; + + + +struct MBOServiceImpl { + MBOServiceImpl( + api::ServiceContext& ctx, + utils::Dispatcher& d, + md::MDClient& c): + dispatcher(d), + client(c), + orders(ctx.locate()), + handle_index(c), + timer(d, 100, [&]() {}) + {} + + utils::Dispatcher& dispatcher; + md::MDClient& client; + api::orderman::OrderService orders; + + HandleIndex handle_index; + utils::DispatcherTimer timer; +}; + +/////////////////////// + +MBOServiceServer::MBOServiceServer( + api::ServiceContext& ctx, + utils::Dispatcher& d, + md::MDClient& c + ) : + MBOServiceBase(ctx), + _me(new MBOServiceImpl(ctx,d,c)) + { + // subscribe to mdclient + // make publishers + // subscribe to orders + + // _me = utils::make_unique(); + } +MBOServiceServer::~MBOServiceServer() { } + + + +}} //namespace +#endif \ No newline at end of file diff --git a/src/lib/md/MBOServiceServer.h b/src/lib/md/MBOServiceServer.h new file mode 100644 index 000000000..9e47aadfa --- /dev/null +++ b/src/lib/md/MBOServiceServer.h @@ -0,0 +1,25 @@ +#pragma once +#include "lib/api/public/ServiceContext.h" +#include "lib/api/services/MBOService.server.pb.h" +#include + +namespace bts { +namespace utils { class Dispatcher; } +namespace api { namespace orderman { class OrderService; } } +namespace md { + +class MDClient; + +struct MBOServiceImpl; +class MBOServiceServer : public MBOServiceBase { + std::unique_ptr _me; + public: + MBOServiceServer( + api::ServiceContext& ctx, + utils::Dispatcher&, + md::MDClient&); + ~MBOServiceServer(); + +}; + +}} //namespace diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 816f55e78..f7f8c9b84 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -1,4 +1,3 @@ -#ifndef NO_CPP0X #ifndef _WIN32 #include "QueryServer.h" #include "lib/config/config.h" @@ -132,28 +131,6 @@ struct MDConsumerGroup { }; -class HandleIndex { - mutable std::mutex _mutex; - std::unordered_map _index; -public: - template - auto lock(CB cb) { - std::lock_guard guard(_mutex); - return cb(_index); - } - template - auto lock(CB cb) const { - std::lock_guard guard(_mutex); - return cb(_index); - } - void clear() { - lock([](std::unordered_map& index) { - index.clear(); - }); - } -}; - - void convert(PriceSize ps, PriceSizeMessage* pps) { pps->set_mantissa(ps.mantissa()); pps->set_exponent(ps.exponent()); @@ -192,7 +169,6 @@ class QueryServerImpl : public IQueryServer { MDConsumer _consumer; int _sub; boost::container::flat_map> _groups; - HandleIndex _handle_index; public: QueryServerImpl(MDClient& client, metrics::MetricsContainer& stats) : @@ -223,10 +199,6 @@ public: const auto& pgroup = iter->second; bts_assert(pgroup); pgroup->add_security(handle); - // Also insert into handle index - _handle_index.lock([handle](auto& index) { - index[SecurityKey(handle.key())] = handle; - }); } }); } @@ -240,7 +212,7 @@ public: for (auto& pair : _groups) { pair.second->set_consumer(_consumer); } - _handle_index.clear(); + // _handle_index.clear(); subscribe_securities(); utils::Dispatcher::background().post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000); } @@ -440,38 +412,6 @@ public: } return std::nullopt; } - - void get_piq(const PIQRequest& req, PIQResponse* result) const { - // for each pair in request - // look up the mbo - // search for the order in the mbo and add it to the result if found - auto& rmap = *result->mutable_position(); - for(const auto& entry : req.orders()) { - auto order_key = OrderKey(entry.order_id()); - auto security = SecurityKey(entry.security_id()); - auto handle = _handle_index.lock( - [security](auto& index){ - auto iter = index.find(security); - if (iter != index.end()) { - return iter->second; - } - return MDConsumerHandle(); - }); - if (handle.is_initialized()) { - // get mbo - auto mbo = handle.get_mbo(); - // search for order - auto search_result = search_orders(order_key, mbo->bids); - if (!search_result) { - search_result = search_orders(order_key, mbo->asks); - } - if (search_result) { - rmap[order_key.value()] = search_result.value(); - } - } - } - } - }; struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { @@ -567,17 +507,9 @@ struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { reply(*preply); }); } - - virtual void get_piq(const api::AuthToken&, const PIQRequest& request, const std::function& reply) override { - _threadpool.post([=]() { - google::protobuf::Arena arena; - PIQResponse* presponse = google::protobuf::Arena::CreateMessage(&arena); - _db.get_piq(request, presponse); - reply(*presponse); - }); - } }; + struct QueryServer { QueryServer(utils::Dispatcher& threadpool, const bts::api::ServiceContext& ctx, @@ -644,4 +576,3 @@ std::unique_ptr create_query_server(MDClient& mdc, metrics::Metric }}//namespace #endif -#endif diff --git a/src/lib/utils/LockBox.h b/src/lib/utils/LockBox.h new file mode 100644 index 000000000..4e37b8024 --- /dev/null +++ b/src/lib/utils/LockBox.h @@ -0,0 +1,24 @@ +#pragma once +#include + +namespace bts { +namespace utils { + +template +class LockBox { + mutable std::mutex _mutex; + T _data; +public: + template + auto lock(CB cb) { + std::lock_guard guard(_mutex); + return cb(_data); + } + template + auto lock(CB cb) const { + std::lock_guard guard(_mutex); + return cb(_data); + } +}; + +}} //namespace From 9fd7db55ca7a0d6f66fbf142cd9a6fa29b47b8e9 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 9 Apr 2019 09:54:42 -0400 Subject: [PATCH 018/100] SPARK-2248 Update.h should have add/remove/change instead of just add/remove --- src/lib/api/public/ClientChannel.h | 47 +++++++++++++++++++----------- src/lib/api/public/RawUpdate.h | 14 +++++++-- src/lib/api/public/Update.h | 32 ++++++++++++++++++-- src/lib/md/MBOServiceServer.cpp | 21 +++++++++---- 4 files changed, 86 insertions(+), 28 deletions(-) diff --git a/src/lib/api/public/ClientChannel.h b/src/lib/api/public/ClientChannel.h index 3918f3f05..25cbd019f 100644 --- a/src/lib/api/public/ClientChannel.h +++ b/src/lib/api/public/ClientChannel.h @@ -5,6 +5,8 @@ #include "ApiException.h" #include "RawUpdate.h" #include "ServiceContext.h" +#include "lib/logging/logging.h" + #include #include #include @@ -12,6 +14,17 @@ namespace bts { namespace api { +template +std::shared_ptr deserialize(const char* buffer, size_t length) { + google::protobuf::io::ArrayInputStream stream(buffer, length); + auto item = std::make_shared(); + if (item->ParseFromZeroCopyStream(&stream)) { + return item; + } + LOG(Api,Error,"Failed to deserialize proto from api stream. This is likely fatal."); + return nullptr; +} + class ClientChannel { int _id; @@ -80,29 +93,29 @@ public: } else { bool first_update = true; - auto cb = [observer, first_update](EventType event_type, const StreamUpdate* changes, int change_count) mutable { + auto cb = [observer, first_update, method, qualifier](EventType event_type, const StreamUpdate* changes, int change_count) mutable { if (event_type == EventType::TABLE_UPDATED) { - std::vector> added_items; + std::vector> added_items, changed_items; std::vector added_keys, - removed_keys; + changed_keys, + removed_keys; for (size_t x = 0; x < change_count; ++x) { const StreamUpdate& change = changes[x]; - if (change.type != StreamUpdate::REMOVE) { - added_keys.reserve(change_count); - added_items.reserve(change_count); - added_keys.push_back(change.key); - google::protobuf::io::ArrayInputStream stream(change.buffer, change.buffer_len); - auto item = std::make_shared(); - if (item->ParseFromZeroCopyStream(&stream)) { - added_items.push_back(move(item)); - } - } - if (change.type != StreamUpdate::CREATE) { - removed_keys.reserve(change_count); - removed_keys.push_back(change.key); + switch(change.type) { + case StreamUpdate::REMOVE: + removed_keys.push_back(change.key); + break; + case StreamUpdate::REPLACE: + changed_items.push_back(deserialize(change.buffer, change.buffer_len)); + changed_keys.push_back(change.key); + break; + case StreamUpdate::CREATE: + added_items.push_back(deserialize(change.buffer, change.buffer_len)); + added_keys.push_back(change.key); + break; } } - observer.on_next(RawUpdate>(added_items, added_keys, removed_keys)); + observer.on_next(RawUpdate>(added_items, added_keys, changed_items, changed_keys, removed_keys)); } else if (event_type == EventType::TABLE_CREATED) { if (!first_update) { diff --git a/src/lib/api/public/RawUpdate.h b/src/lib/api/public/RawUpdate.h index be79fa432..93e7cabe3 100644 --- a/src/lib/api/public/RawUpdate.h +++ b/src/lib/api/public/RawUpdate.h @@ -6,10 +6,16 @@ namespace api { template struct RawUpdate { - RawUpdate(std::vector& added_items, std::vector& added_keys, std::vector& removed_keys) : + RawUpdate(std::vector& added_items, + std::vector& added_keys, + std::vector& changed_items, + std::vector& changed_keys, + std::vector& removed_keys) : is_reset(false), added_items(std::move(added_items)), added_keys(std::move(added_keys)), + changed_items(std::move(changed_items)), + changed_keys(std::move(changed_keys)), removed_keys(std::move(removed_keys)) {} @@ -20,12 +26,16 @@ namespace api { bool is_reset; std::vector added_items; std::vector added_keys; + + std::vector changed_items; + std::vector changed_keys; + std::vector removed_keys; private: RawUpdate() : is_reset(true), - added_items(), added_keys(), removed_keys() + added_items(), added_keys(), changed_items(), changed_keys(), removed_keys() {} }; diff --git a/src/lib/api/public/Update.h b/src/lib/api/public/Update.h index d37345679..4901bded1 100644 --- a/src/lib/api/public/Update.h +++ b/src/lib/api/public/Update.h @@ -9,12 +9,21 @@ namespace bts { namespace api { + template + class Change { + T _old, _new; + public: + Change(const T& old_value, const T& new_value) : _old(old_value), _new(new_value) { } + const T& old() { return _old; } + const T& current() { return _new; } + }; template struct Update { - Update(std::vector& all, std::vector& added, std::vector& removed) : + Update(std::vector& all, std::vector& added, std::vector> changed, std::vector& removed) : _all_items(std::move(all)), _added_items(std::move(added)), + _changed_items(std::move(changed)), _removed_items(std::move(removed)) {} @@ -22,17 +31,20 @@ namespace api { Update(Update&& other) : _all_items(std::move(other._all_items)), _added_items(std::move(other._added_items)), + _changed_items(std::move(other._changed_items)), _removed_items(std::move(other._removed_items)) {} TKey key(const T& t) const { return ServiceType::key(t); } const std::vector& all() const { return _all_items; } const std::vector& added() const { return _added_items; } + const std::vector>& changed() const { return _changed_items; } const std::vector& removed() const { return _removed_items; } private: std::vector _all_items; std::vector _added_items; + std::vector> _changed_items; std::vector _removed_items; }; @@ -49,9 +61,12 @@ namespace api { } state->clear(); std::vector all, added; - return Update(all, added, removed); + std::vector> changed; + return Update(all, added, changed, removed); } else { + // + // Remove std::vector removed; removed.reserve(raw.removed_keys.size()); for (int key : raw.removed_keys) { @@ -61,6 +76,8 @@ namespace api { state->erase(iter); } } + // + // Add for (size_t x = 0; x < raw.added_keys.size(); ++x) { (*state)[raw.added_keys[x]] = raw.added_items[x]; } @@ -69,7 +86,16 @@ namespace api { for (const auto& item : *state) { all.push_back(item.second); } - return Update(all, raw.added_items, removed); + // + // Change + std::vector> changed; + changed.reserve(raw.changed_keys.size()); + for (size_t x = 0; x < raw.changed_keys.size(); ++x) { + changed.push_back(Change{(*state)[raw.changed_keys[x]], raw.changed_items[x]}); + (*state)[raw.changed_keys[x]] = raw.changed_items[x]; + } + + return Update(all, raw.added_items, changed, removed); } }); }; diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index 0603d3951..0c4c2ce49 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -86,14 +86,23 @@ MBOServiceServer::MBOServiceServer( md::MDClient& c ) : MBOServiceBase(ctx), - _me(new MBOServiceImpl(ctx,d,c)) + _me(new MBOServiceImpl(ctx,d,c,make_piq_publisher())) { - // subscribe to mdclient - // make publishers - // subscribe to orders - - // _me = utils::make_unique(); + // subscribe to mdclient ✔ + // make publishers ✔ + // subscribe to orders => stash active orders dict + _me->orders_sub = + _me->orders.subscribe_orders().subscribe([=](const auto& update) { + const auto& added = update.added(); + const auto& removed = update.removed(); + // api thread + _me->dispatcher.post([this,added,removed]() { + // dispatcher thread + // handle add/remove/change + }); + }); } + MBOServiceServer::~MBOServiceServer() { } From 06aa540110044355bae1aa8866db62570f978810 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 9 Apr 2019 09:57:44 -0400 Subject: [PATCH 019/100] SPARK-2248 Update api subscribe clients for this api change --- src/lib/api/public/Update.h | 4 ++-- src/server/sparkd/position/PositionServiceServer.cpp | 4 ++++ src/server/sparkd/user/AdminServiceServer.cpp | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/lib/api/public/Update.h b/src/lib/api/public/Update.h index 4901bded1..7de659ded 100644 --- a/src/lib/api/public/Update.h +++ b/src/lib/api/public/Update.h @@ -14,8 +14,8 @@ namespace api { T _old, _new; public: Change(const T& old_value, const T& new_value) : _old(old_value), _new(new_value) { } - const T& old() { return _old; } - const T& current() { return _new; } + const T& old() const { return _old; } + const T& current() const { return _new; } }; template diff --git a/src/server/sparkd/position/PositionServiceServer.cpp b/src/server/sparkd/position/PositionServiceServer.cpp index 274ae22b9..560710a10 100644 --- a/src/server/sparkd/position/PositionServiceServer.cpp +++ b/src/server/sparkd/position/PositionServiceServer.cpp @@ -48,6 +48,10 @@ me(new PImpl(context, me->publisher.add_or_update(_on_fill(*f,false)); for (const auto& f : update.added()) me->publisher.add_or_update(_on_fill(*f,true)); + for (const auto& f : update.changed()) { + me->publisher.add_or_update(_on_fill(*f.old(),false)); + me->publisher.add_or_update(_on_fill(*f.current(),true)); + } me->publisher.flush(); }); } diff --git a/src/server/sparkd/user/AdminServiceServer.cpp b/src/server/sparkd/user/AdminServiceServer.cpp index 09b33068d..9e317e7fc 100644 --- a/src/server/sparkd/user/AdminServiceServer.cpp +++ b/src/server/sparkd/user/AdminServiceServer.cpp @@ -66,6 +66,10 @@ AdminServiceServer::AdminServiceServer(bts::api::ServiceContext& context) if(it != _order_map.end()) _order_map.erase(it); } + for (const auto& ch : update.changed()) { + const auto& o = ch.current(); + _order_map[o->key()] = o; + } for (const auto& o : update.added()){ _order_map[o->key()]=o; } From d76b098c5171f8a64e0de4c773e2511a274cab02 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 8 Apr 2019 16:42:47 -0400 Subject: [PATCH 020/100] SPARK-2179 streaming piq 2/3 --- src/client/sparkcmd/Cmd/ServiceTestCommand.cs | 52 +++++++++---------- src/lib/md/MBOServiceServer.cpp | 32 +++++++++++- 2 files changed, 56 insertions(+), 28 deletions(-) diff --git a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs index 5450df0d7..1f449cd4d 100644 --- a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs +++ b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs @@ -34,41 +34,41 @@ namespace bts { admin_test(); } else { - mbo_test(); + // mbo_test(); } return 0; } - private static void mbo_test() { - using (var ctx = new ServiceContext()) { - var relay_svc = ctx.locate(); - var order_sv = ctx.locate(); + // private static void mbo_test() { + // using (var ctx = new ServiceContext()) { + // var relay_svc = ctx.locate(); + // var order_sv = ctx.locate(); - var upd = order_sv.subscribei_orders().FirstAsync().Wait(); /* don't use Wait() */ - var orders = - upd.all - .Where(o => - o.State.Status == OrderStatus.Working && - o.State.native_order_state().HasValue) - .ToList(); + // var upd = order_sv.subscribei_orders().FirstAsync().Wait(); /* don't use Wait() */ + // var orders = + // upd.all + // .Where(o => + // o.State.Status == OrderStatus.Working && + // o.State.native_order_state().HasValue) + // .ToList(); - var request = new PIQRequest(); - request.Orders.AddRange(orders.Select(o => - new OrderQuery { - SecurityId = o.Def.SecurityKey, - OrderId = UInt64.Parse(o.State.native_order_state().Value.ExchangeOrderId) })); + // var request = new PIQRequest(); + // request.Orders.AddRange(orders.Select(o => + // new OrderQuery { + // SecurityId = o.Def.SecurityKey, + // OrderId = UInt64.Parse(o.State.native_order_state().Value.ExchangeOrderId) })); - Console.WriteLine($"Requesting positions for {request.Orders.Count} orders..."); - var result = relay_svc.get_piq(request); + // Console.WriteLine($"Requesting positions for {request.Orders.Count} orders..."); + // var result = relay_svc.get_piq(request); - Console.WriteLine($"got response - {result.Position.Count} entries"); - foreach (var kvp in result.Position) { - Console.Write($"> {kvp.Key} => "); - ConsoleEx.WriteLine($"{kvp.Value}", ConsoleColor.Yellow); - } - } - } + // Console.WriteLine($"got response - {result.Position.Count} entries"); + // foreach (var kvp in result.Position) { + // Console.Write($"> {kvp.Key} => "); + // ConsoleEx.WriteLine($"{kvp.Value}", ConsoleColor.Yellow); + // } + // } + // } private static void synthetic_test() { using (var ctx = new ServiceContext()) { diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index 0c4c2ce49..2841633d3 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -8,10 +8,15 @@ #include "lib/utils/LockBox.h" +#include +#include + namespace bts { namespace md { +// // Simple class to handle MD recovery and maintain an index of SecurityKey=>MDConsumerHandle + class HandleIndex { MDClient& _client; int _sub; @@ -56,18 +61,29 @@ private: } }; +struct ActiveOrder { + uint32_t id; + int64_t ex_id; + int32_t position; +}; +// +// MBOService struct MBOServiceImpl { MBOServiceImpl( api::ServiceContext& ctx, utils::Dispatcher& d, - md::MDClient& c): + md::MDClient& c, + MBOServiceBase::PiqPublisher&& pub + ): dispatcher(d), client(c), orders(ctx.locate()), handle_index(c), - timer(d, 100, [&]() {}) + publisher(pub), + timer(d, 200, [&]() {timer_tick();}), + orders_sub() {} utils::Dispatcher& dispatcher; @@ -75,9 +91,21 @@ struct MBOServiceImpl { api::orderman::OrderService orders; HandleIndex handle_index; + MBOServiceBase::PiqPublisher publisher; utils::DispatcherTimer timer; + + rxcpp::composite_subscription orders_sub; + + void timer_tick(); + void flush(); }; +void MBOServiceImpl::timer_tick() { + // for each active + + // publisher.flush(); +} + /////////////////////// MBOServiceServer::MBOServiceServer( From a101a267ed859cbd55a005cbaa051ee27108199c Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 9 Apr 2019 14:58:15 -0400 Subject: [PATCH 021/100] More consistent to call old previous for Change --- src/lib/api/public/Update.h | 2 +- src/server/sparkd/position/PositionServiceServer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/api/public/Update.h b/src/lib/api/public/Update.h index 7de659ded..c4b615950 100644 --- a/src/lib/api/public/Update.h +++ b/src/lib/api/public/Update.h @@ -14,7 +14,7 @@ namespace api { T _old, _new; public: Change(const T& old_value, const T& new_value) : _old(old_value), _new(new_value) { } - const T& old() const { return _old; } + const T& previous() const { return _old; } const T& current() const { return _new; } }; diff --git a/src/server/sparkd/position/PositionServiceServer.cpp b/src/server/sparkd/position/PositionServiceServer.cpp index 560710a10..49706a39c 100644 --- a/src/server/sparkd/position/PositionServiceServer.cpp +++ b/src/server/sparkd/position/PositionServiceServer.cpp @@ -49,7 +49,7 @@ me(new PImpl(context, for (const auto& f : update.added()) me->publisher.add_or_update(_on_fill(*f,true)); for (const auto& f : update.changed()) { - me->publisher.add_or_update(_on_fill(*f.old(),false)); + me->publisher.add_or_update(_on_fill(*f.previous(),false)); me->publisher.add_or_update(_on_fill(*f.current(),true)); } me->publisher.flush(); From 0cf4b5c2c8597f7bac1e9ea970c8cf20535fa592 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 9 Apr 2019 15:07:10 -0400 Subject: [PATCH 022/100] SPARK-2224 3/3 basic piq stream (unblocks jake) --- src/lib/api/services/MBOService.proto | 4 +- src/lib/md/ExchangeOrderKey.h | 7 + src/lib/md/MBOProcessor.h | 5 +- src/lib/md/MBOServiceServer.cpp | 156 ++++++++++++++++-- src/lib/md/relay/QueryServer.cpp | 19 --- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 4 +- 6 files changed, 156 insertions(+), 39 deletions(-) create mode 100644 src/lib/md/ExchangeOrderKey.h diff --git a/src/lib/api/services/MBOService.proto b/src/lib/api/services/MBOService.proto index 0a00fc22e..4a8383f8b 100644 --- a/src/lib/api/services/MBOService.proto +++ b/src/lib/api/services/MBOService.proto @@ -5,11 +5,11 @@ option cc_enable_arenas=true; import "lib/api/apioptions.proto"; import "lib/api/apicore.proto"; -message OrderPiq { +message OrderPIQ { uint32 order_id = 1 [(bts.api.key)=true]; uint32 position = 2; } service MBOService { - rpc piq (bts.api.VoidMessage) returns (stream OrderPiq); + rpc piq (bts.api.VoidMessage) returns (stream OrderPIQ); } \ No newline at end of file diff --git a/src/lib/md/ExchangeOrderKey.h b/src/lib/md/ExchangeOrderKey.h new file mode 100644 index 000000000..b3d59e459 --- /dev/null +++ b/src/lib/md/ExchangeOrderKey.h @@ -0,0 +1,7 @@ +#pragma once +#include "lib/utils/ValueWrapper.h" +namespace bts { +namespace md { +using ExchangeOrderKey = utils::ValueWrapper; +using SparkOrderKey = utils::ValueWrapper; +} } \ No newline at end of file diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index bbb8c7e21..17246fd21 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -4,17 +4,16 @@ #include "lib/utils/TimeStamp.h" #include "lib/utils/ValueWrapper.h" #include "IntPrice.h" +#include "ExchangeOrderKey.h" #include #include namespace bts { namespace md { - -using OrderKey = utils::ValueWrapper; struct SnapshotOrder { - OrderKey id; + ExchangeOrderKey id; IntPrice price; int32_t size; }; diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index 2841633d3..6a9218a80 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -5,11 +5,17 @@ #include "lib/utils/Dispatcher.h" #include "lib/utils/TimeStamp.h" #include "MDClient.h" +#include "ExchangeOrderKey.h" +#include "MBOProcessor.h" #include "lib/utils/LockBox.h" #include #include +#include + + +using namespace bts::api::orderman; namespace bts { namespace md { @@ -61,10 +67,36 @@ private: } }; -struct ActiveOrder { - uint32_t id; - int64_t ex_id; - int32_t position; +class ActiveOrder { + SecurityKey _security; + SparkOrderKey _id; + ExchangeOrderKey _ex_id; + int32_t _position; + +public: + ActiveOrder(): _security(), _id(), _ex_id(), _position(-1) { bts_assert(false); } + ActiveOrder(SecurityKey sec, SparkOrderKey s, ExchangeOrderKey ex): _security(sec), _id(s), _ex_id(ex), _position(-1) {} + ActiveOrder(SecurityKey sec, SparkOrderKey s, const std::string& ex): _security(sec), _id(s), _ex_id(), _position(-1) { + _ex_id = ExchangeOrderKey(std::stoul(ex)); + } + + SparkOrderKey id() const { return _id; } + ExchangeOrderKey exchange_id() const { return _ex_id; } + int32_t& position() { return _position; }; + int32_t position() const { return _position; }; + bool set_position(int32_t pos) { + if (pos != _position) { + _position = pos; + return true; + } + return false; + } + OrderPIQ to_piq() const { + OrderPIQ result; + result.set_order_id(_id.value()); + result.set_position(_position); + return result; + } }; // @@ -81,33 +113,102 @@ struct MBOServiceImpl { client(c), orders(ctx.locate()), handle_index(c), - publisher(pub), + publish(pub), timer(d, 200, [&]() {timer_tick();}), orders_sub() {} utils::Dispatcher& dispatcher; md::MDClient& client; - api::orderman::OrderService orders; + OrderService orders; HandleIndex handle_index; - MBOServiceBase::PiqPublisher publisher; + MBOServiceBase::PiqPublisher publish; utils::DispatcherTimer timer; rxcpp::composite_subscription orders_sub; + std::unordered_map active; void timer_tick(); void flush(); }; void MBOServiceImpl::timer_tick() { - // for each active + //Dispatcher thread + std::vector rows; - // publisher.flush(); + publish(nullptr, &rows, nullptr); } + /////////////////////// +std::optional search_orders(const ExchangeOrderKey oid, const std::vector& orders) { + IntPrice price = {}; + size_t position = 0; + for (size_t x = 0; x < orders.size(); ++x) { + const auto& oo = orders[x]; + if (oo.id == oid) { + return { position }; + } + else if (price == oo.price) { + position += 1; + } + else { + price = oo.price; + position = 0; + } + } + return std::nullopt; +} + +bool update_position(const HandleIndex& , ActiveOrder& order) { + order.set_position(-2); + return true; +} + +const NativeOrderState* native_order_state(const OrderRecord& record) { + switch(record.state().custom_case()) { + case OrderState::kMarket: + return &record.state().market().native_state(); + case OrderState::kLimit: + return &record.state().limit().native_state(); + case OrderState::kStop: + return &record.state().stop().native_state(); + case OrderState::kStopLimit: + return &record.state().stop_limit().native_state(); + default: + return nullptr; + } +} + +bool is_active_exchange_order(const OrderRecord& record) { + return record.state().status() == ORDER_STATUS_WORKING && + native_order_state(record) != nullptr; +} + +void try_remove(MBOServiceImpl& me, std::set& to_remove, const OrderRecord& record) { + auto key = SparkOrderKey(record.key()); + auto iter = me.active.find(key); + if (iter != me.active.end()) { + me.active.erase(key); + to_remove.insert(key.value()); + } +} +void try_add(MBOServiceImpl& me, std::vector& to_add, const OrderRecord& record) { + if (is_active_exchange_order(record)) { + auto native = native_order_state(record); + bts_assert(native); + ActiveOrder order( + SecurityKey(record.def().security_key()), + SparkOrderKey(record.key()), + native->exchange_order_id()); + update_position(me.handle_index, order); + to_add.push_back(order.to_piq()); + me.active[order.id()] = order; + } +} + MBOServiceServer::MBOServiceServer( api::ServiceContext& ctx, utils::Dispatcher& d, @@ -121,12 +222,41 @@ MBOServiceServer::MBOServiceServer( // subscribe to orders => stash active orders dict _me->orders_sub = _me->orders.subscribe_orders().subscribe([=](const auto& update) { - const auto& added = update.added(); - const auto& removed = update.removed(); // api thread - _me->dispatcher.post([this,added,removed]() { + const auto& added = update.added(); + const auto& changed = update.changed(); + const auto& removed = update.removed(); + _me->dispatcher.post([this,added,changed,removed]() { // dispatcher thread - // handle add/remove/change + std::vector to_add, to_change; + std::set to_remove; + + // Remove + for(const auto& r : removed) { + // If it was active remove it and add to to_remove + try_remove(*_me, to_remove, *r); + } + // Added + for(const auto& add : added) { + // If active add look up the position and add to the state + try_add(*_me, to_add, *add); + } + // Changed + for(const auto& ch : changed) { + // new order is active and old is not look up piq and add to state + bool is_active = is_active_exchange_order(*ch.current()); + bool was_active = is_active_exchange_order(*ch.previous()); + if (is_active && !was_active) { + try_add(*_me, to_add, *ch.current()); + } + else if (was_active && !is_active) { + try_remove(*_me, to_remove, *ch.current()); + } + // is_active && was_active => don't care + // !is_active && !was_active => don't care + } + + _me->publish(&to_add, &to_change, &to_remove); }); }); } diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index f7f8c9b84..4dc971d81 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -393,25 +393,6 @@ public: } } } - - static std::optional search_orders(const OrderKey oid, const std::vector& orders) { - IntPrice price = {}; - size_t position = 0; - for (size_t x = 0; x < orders.size(); ++x) { - const auto& oo = orders[x]; - if (oo.id == oid) { - return { position }; - } - else if (price == oo.price) { - position += 1; - } - else { - price = oo.price; - position = 0; - } - } - return std::nullopt; - } }; struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index bd943250f..85867b973 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -129,10 +129,10 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con auto* snapshot = _mbo->allocate_snapshot(); // copy from onixs book for (const auto& bid : book.bids()) { - snapshot->bids.push_back({ OrderKey(bid.id()), convert_price(bid.price(), 0), bid.quantity() }); + snapshot->bids.push_back({ ExchangeOrderKey(bid.id()), convert_price(bid.price(), 0), bid.quantity() }); } for (const auto& ask : book.offers()) { - snapshot->asks.push_back({ OrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); + snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); } _mbo->publish_snapshot(SecurityKey(key), snapshot); } From c1789bb4a196936b66c6b33533a90c001496eb8c Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 9 Apr 2019 15:14:35 -0400 Subject: [PATCH 023/100] SPARK-2244 Helps to enable the service --- src/server/sparkd/cmds/Run.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 3dd41a86c..5f255fc56 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -15,6 +15,7 @@ #include "lib/md/SyntheticSecurityService.h" #include "lib/md/relay/QueryServer.h" #include "lib/md/MBOProcessor.h" +#include "lib/md/MBOServiceServer.h" #include "lib/metrics/metrics.h" #include "lib/metrics/system_metrics.h" #include "lib/orderman/order_factory/DynamicOrderRegistry.h" @@ -65,10 +66,11 @@ int Run::execute(std::vector const &) { api::synthetic::SyntheticSecurityService synthetic_service(sc); md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service, &mbo.reader()); - md::QueryServerPtr md_query_server( md::run_query_server(sc, mdclient, stats)); + md::MBOServiceServer mboserver(sc, utils::Dispatcher::background(), mdclient); + // initialize order plugins orderman::DynamicOrderRegistry dyn_order_registry; SparkServices plugin_services(dyn_order_registry); From 5ff9f13e7ede1ba07cb31056170dcb6000e8ccd4 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Apr 2019 10:11:17 -0400 Subject: [PATCH 024/100] Update cmakedeps --- src/lib/CMakeDeps_win.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/CMakeDeps_win.cmake b/src/lib/CMakeDeps_win.cmake index 879f19a67..e6cf6b118 100644 --- a/src/lib/CMakeDeps_win.cmake +++ b/src/lib/CMakeDeps_win.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file From 5ba7cd561cb4168f27adc27ae08a026940763f09 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Apr 2019 10:16:18 -0400 Subject: [PATCH 025/100] SPARK-2244 remove parameterless ActiveOrder constructor --- src/lib/md/MBOServiceServer.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index 6a9218a80..9f1b57a4b 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -74,7 +74,8 @@ class ActiveOrder { int32_t _position; public: - ActiveOrder(): _security(), _id(), _ex_id(), _position(-1) { bts_assert(false); } + // empty constructor needed for unordered map operator[] + // ActiveOrder(): _security(), _id(), _ex_id(), _position(-1) { bts_assert(false); } ActiveOrder(SecurityKey sec, SparkOrderKey s, ExchangeOrderKey ex): _security(sec), _id(s), _ex_id(ex), _position(-1) {} ActiveOrder(SecurityKey sec, SparkOrderKey s, const std::string& ex): _security(sec), _id(s), _ex_id(), _position(-1) { _ex_id = ExchangeOrderKey(std::stoul(ex)); @@ -205,7 +206,7 @@ void try_add(MBOServiceImpl& me, std::vector& to_add, const OrderRecor native->exchange_order_id()); update_position(me.handle_index, order); to_add.push_back(order.to_piq()); - me.active[order.id()] = order; + me.active.insert(std::make_pair(order.id(), order)); } } @@ -257,6 +258,7 @@ MBOServiceServer::MBOServiceServer( } _me->publish(&to_add, &to_change, &to_remove); + if (!_me->timer.running()) _me->timer.start(); }); }); } From a3de83dd04721d755863bef7223787ca07690c66 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Apr 2019 10:24:33 -0400 Subject: [PATCH 026/100] SPARK-2244 Update position on timer tick --- src/lib/md/MBOServiceServer.cpp | 53 +++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index 9f1b57a4b..d4c7e3261 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -27,20 +27,22 @@ class HandleIndex { MDClient& _client; int _sub; std::optional _consumer; - utils::LockBox> _index; public: + utils::LockBox> index; + HandleIndex(md::MDClient& client): _client(client), _sub(0), _consumer(std::nullopt), - _index() { + index() { _sub = _client.on_consumer().subscribe([=](const MDConsumer& c) { on_consumer(c); }); on_consumer(_client.current_consumer()); } ~HandleIndex() { _client.on_consumer().unsubscribe(_sub); } + private: void on_consumer(const MDConsumer& c) { MDConsumer temp(c); @@ -48,7 +50,7 @@ private: auto existing = _consumer ? _consumer->id() : 0; LOG(md,Info, "Updateing MBOService MDConsumer. Old {} new {}", existing, temp.id()); // clear the index - _index.lock([](auto& index) { index.clear(); }); + index.lock([](auto& index) { index.clear(); }); // Swap to new consumer if (_consumer) { _consumer->swap(temp); @@ -58,7 +60,7 @@ private: auto id = _consumer->id(); _consumer->on_security([this,id](const SecurityPtr&, MDConsumerHandle handle) { if (_consumer->id() == id) { - _index.lock([handle](auto& index) { + index.lock([handle](auto& index) { index[SecurityKey(handle.key())] = handle; }); } @@ -81,6 +83,7 @@ public: _ex_id = ExchangeOrderKey(std::stoul(ex)); } + SecurityKey security_id() const { return _security; } SparkOrderKey id() const { return _id; } ExchangeOrderKey exchange_id() const { return _ex_id; } int32_t& position() { return _position; }; @@ -134,12 +137,6 @@ struct MBOServiceImpl { void flush(); }; -void MBOServiceImpl::timer_tick() { - //Dispatcher thread - std::vector rows; - - publish(nullptr, &rows, nullptr); -} /////////////////////// @@ -163,9 +160,39 @@ std::optional search_orders(const ExchangeOrderKey oid, const std::vec return std::nullopt; } -bool update_position(const HandleIndex& , ActiveOrder& order) { - order.set_position(-2); - return true; +bool update_position(const HandleIndex& handles, ActiveOrder& order) { + const auto& index = handles.index; + auto handle = index.lock([id=order.security_id()](const auto& index) { + auto iter = index.find(id); + return iter != index.end() ? iter->second : MDConsumerHandle {}; + }); + + if (handle.is_initialized()) { + auto mbo = handle.get_mbo(); + if (mbo) { + auto search_result = search_orders(order.exchange_id(), mbo->bids); + if (!search_result) { + search_result = search_orders(order.exchange_id(), mbo->asks); + } + if (search_result) { + order.set_position(search_result.value()); + return true; + } + } + } + return false; +} + +void MBOServiceImpl::timer_tick() { + //Dispatcher thread + std::vector rows; + for (auto& pair : active) { + if (update_position(handle_index, pair.second)) { + rows.push_back(pair.second.to_piq()); + } + } + + publish(nullptr, &rows, nullptr); } const NativeOrderState* native_order_state(const OrderRecord& record) { From adec1bdfd06cee6c02334246fe969eb8d0b49018 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Apr 2019 10:57:44 -0400 Subject: [PATCH 027/100] SPARK-678 Dont' crash on null array --- .../api/Channels/QuickBusClientChannel.cs | 28 ++++++++++--------- src/lib/api/core/ApiClientInterop.cpp | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/client/spark_api/api/Channels/QuickBusClientChannel.cs b/src/client/spark_api/api/Channels/QuickBusClientChannel.cs index ac054c2c8..593edf9f7 100644 --- a/src/client/spark_api/api/Channels/QuickBusClientChannel.cs +++ b/src/client/spark_api/api/Channels/QuickBusClientChannel.cs @@ -110,19 +110,21 @@ namespace bts.api.core { the_update.Clear(); try { if (event_type == ApiNativeMethods.EventType.TABLE_UPDATED) { - foreach (var change in changes) { - switch (change.type) { - case ApiNativeMethods.ChangeType.REMOVE: - the_update.removed_keys.Add(change.key); - break; - case ApiNativeMethods.ChangeType.REPLACE: - the_update.changed_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); - the_update.changed_keys.Add(change.key); - break; - case ApiNativeMethods.ChangeType.CREATE: - the_update.added_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); - the_update.added_keys.Add(change.key); - break; + if (changes != null) { + foreach (var change in changes) { + switch (change.type) { + case ApiNativeMethods.ChangeType.REMOVE: + the_update.removed_keys.Add(change.key); + break; + case ApiNativeMethods.ChangeType.REPLACE: + the_update.changed_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); + the_update.changed_keys.Add(change.key); + break; + case ApiNativeMethods.ChangeType.CREATE: + the_update.added_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); + the_update.added_keys.Add(change.key); + break; + } } } observer.OnNext(the_update); diff --git a/src/lib/api/core/ApiClientInterop.cpp b/src/lib/api/core/ApiClientInterop.cpp index 3927bbc3e..487b2eabb 100644 --- a/src/lib/api/core/ApiClientInterop.cpp +++ b/src/lib/api/core/ApiClientInterop.cpp @@ -176,7 +176,7 @@ void safe_call_subscription_callback(raw_subscription_callback cb, EventType typ if (updates.size() > 0) cb(type, &updates[0], updates.size(), extra); else - cb(type, nullptr, updates.size(), extra); + cb(type, nullptr, 0, extra); }); } From f6c0f393bdee701fe83d14ccefc6a9a086701b91 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Apr 2019 11:04:28 -0400 Subject: [PATCH 028/100] SPARK-2244 Suppress empty updates --- src/lib/md/MBOServiceServer.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index d4c7e3261..acc9aa035 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -175,8 +175,7 @@ bool update_position(const HandleIndex& handles, ActiveOrder& order) { search_result = search_orders(order.exchange_id(), mbo->asks); } if (search_result) { - order.set_position(search_result.value()); - return true; + return order.set_position(search_result.value()); } } } @@ -191,8 +190,9 @@ void MBOServiceImpl::timer_tick() { rows.push_back(pair.second.to_piq()); } } - - publish(nullptr, &rows, nullptr); + if (!rows.empty()) { + publish(nullptr, &rows, nullptr); + } } const NativeOrderState* native_order_state(const OrderRecord& record) { From f7fd81f3af8c3f6bb8d66a03f1946a90df5a7f88 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Apr 2019 14:47:50 -0400 Subject: [PATCH 029/100] Improve Servicetail --- src/client/sparkcmd/Cmd/ServiceTail.cs | 20 +++++++++++++++++++- src/client/sparkcmd/sparkcmd.csproj | 2 ++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/client/sparkcmd/Cmd/ServiceTail.cs b/src/client/sparkcmd/Cmd/ServiceTail.cs index 7ebbb0c92..2820bbef6 100644 --- a/src/client/sparkcmd/Cmd/ServiceTail.cs +++ b/src/client/sparkcmd/Cmd/ServiceTail.cs @@ -8,8 +8,15 @@ using System.Text; using System.Threading.Tasks; namespace bts { + + enum PrintFormat { + ToString, + Json + } class TailService : ConsoleCommand { static bool _all = false; + static PrintFormat _format = PrintFormat.ToString; + static bool _timestamp = false; public TailService() { void ignore(string _) { } @@ -19,6 +26,7 @@ namespace bts { // The following are not implemented. If you need them get to work. HasOption("path", "Path inside the row to use for display. Ex: 'Id' or 'Name' ", ignore); HasOption("all", "Invlude 'all' rows on every print", _ => _all = true); + HasOption("json", "print as json", _ => _format = PrintFormat.Json); HasAdditionalArguments(null, " [nothing]"); } public override int Run(string[] remainingArguments) { @@ -46,7 +54,17 @@ namespace bts { Console.Write(" "); } if (_all) { } - Console.WriteLine(value); + + switch(_format) { + case PrintFormat.Json: + dynamic dv = value; + var json = Newtonsoft.Json.JsonConvert.SerializeObject(new { dv.added, dv.removed, dv.changed }); + Console.WriteLine(json); + break; + case PrintFormat.ToString: + Console.WriteLine(value); + break; + } } } private static void tail(string service_name, string method_name) { diff --git a/src/client/sparkcmd/sparkcmd.csproj b/src/client/sparkcmd/sparkcmd.csproj index 9046c4610..b0132eaed 100644 --- a/src/client/sparkcmd/sparkcmd.csproj +++ b/src/client/sparkcmd/sparkcmd.csproj @@ -23,5 +23,7 @@ + + From 5f9a421490d507e3a3d739f649e59bca0295014a Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 11 Apr 2019 10:28:21 -0400 Subject: [PATCH 030/100] Additional logging for QueryServer::get_names error --- src/lib/md/relay/QueryServer.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 7b409662f..306e51372 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -380,9 +380,8 @@ public: google::protobuf::MapPair v(h.handle.key(), security.ice().security_desc()); x->insert(v); } - else - { - LOG(md,Error, "Invalid ExchangeData_case {}", security.ExchangeData_case()); + else { + LOG(md,Error, "Invalid ExchangeData_case {} for security {}", security.ExchangeData_case(), h.handle.key()); } } } From 7381088d6d9b042826b10e09a6b96505da0230a8 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 11 Apr 2019 11:07:49 -0400 Subject: [PATCH 031/100] Remove unnecessary error log --- src/lib/md/relay/QueryServer.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 306e51372..64297f5b0 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -380,9 +380,7 @@ public: google::protobuf::MapPair v(h.handle.key(), security.ice().security_desc()); x->insert(v); } - else { - LOG(md,Error, "Invalid ExchangeData_case {} for security {}", security.ExchangeData_case(), h.handle.key()); - } + // There is currently no ExchangeData for synthetics } } } From a376fe9e22b0b15c77f83203d2a310b496d3f09e Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 12 Apr 2019 09:56:28 -0400 Subject: [PATCH 032/100] Fix pool_stats crash for the pool paupers --- src/lib/md/MBOProcessor.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 4d5f2a33d..4fec20880 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -35,6 +35,8 @@ void print_pool_stats(const utils::ConcurrentPool& pool) { } }); + if (count == 0) return; + std::sort(bid_sz.begin(), bid_sz.end()); std::sort(ask_sz.begin(), ask_sz.end()); size_t bid_tot =0, ask_tot = 0; From d89f93c323cc9d72e76b186a135ec6863bd0df4f Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Mon, 15 Apr 2019 11:05:40 -0500 Subject: [PATCH 033/100] Adding MBOService as an export in gui --- src/client/spark/Daemons/ShellServices.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/spark/Daemons/ShellServices.cs b/src/client/spark/Daemons/ShellServices.cs index 8fa232571..bcfd91e6e 100644 --- a/src/client/spark/Daemons/ShellServices.cs +++ b/src/client/spark/Daemons/ShellServices.cs @@ -6,6 +6,7 @@ using bts.api.persistence; using bts.api.position; using bts.api.synthetic; using bts.co2; +using bts.md; using bts.metrics; using bts.support; using System.ComponentModel.Composition; @@ -50,6 +51,9 @@ namespace Spark { AlgoServiceWrapper _algo_service_wrapper; [Export] public AlgoServiceWrapper algo_service_wrapper => _algo_service_wrapper ?? (_algo_service_wrapper = new AlgoServiceWrapper(context)); + + [Export] + public MBOService mbo_service => context.locate(); } class AlgoServiceWrapper { From 8130d5a0f86832117972c9d703396e14bd5e6b73 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:06:32 -0500 Subject: [PATCH 034/100] adding render_piq method --- src/client/spark/Orders/OrderSecPiq.cs | 17 +++++++++++++ .../spark/view/Spark/Ladder/LaderRender.cs | 24 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 src/client/spark/Orders/OrderSecPiq.cs diff --git a/src/client/spark/Orders/OrderSecPiq.cs b/src/client/spark/Orders/OrderSecPiq.cs new file mode 100644 index 000000000..77f7177d6 --- /dev/null +++ b/src/client/spark/Orders/OrderSecPiq.cs @@ -0,0 +1,17 @@ +using bts.api; +using bts.api.orderman; + + +namespace mbts.spark { + public struct OrderSecPiq { + public SecurityImmutable? sec; + public OrderRecordImmutable order; + public uint? piq; + + public OrderSecPiq(SecurityImmutable? sec_imm, OrderRecordImmutable ori, uint? placement_in_queue) { + sec = sec_imm; + order = ori; + piq = placement_in_queue; + } + } +} diff --git a/src/client/spark/view/Spark/Ladder/LaderRender.cs b/src/client/spark/view/Spark/Ladder/LaderRender.cs index 014176202..dfbc80d07 100644 --- a/src/client/spark/view/Spark/Ladder/LaderRender.cs +++ b/src/client/spark/view/Spark/Ladder/LaderRender.cs @@ -277,6 +277,30 @@ namespace mbts.spark { } } + // this is basically a copy of redner_sizes up above + internal static void render_piqs(DrawingVisual dv, GlyphContext glyph_context, int origin_index, double[] prices, (double price, uint piq)[] piqs, double row_height, ContractSpecificationImmutable spec, LadderColumnSettings settings) { + using (var dc = dv.RenderOpen()) { + using (_draw_with_clip(dc, settings, row_height, prices.Length)) { + if (!prices.within_range(origin_index) || piqs.is_null_or_empty()) + return; + var foreground = settings.foreground.rgb_brush(); + int curr_index = origin_index; + double curr_price = prices[curr_index]; + for (int i = 0; i < piqs.Length; i++) { + double price = piqs[i].price; + // prices are in descending order, so get diff from curr_price - price + int tick_diff = spec.tickdiff_unchecked(curr_price, price); + curr_index += tick_diff; + curr_price = price; + if (curr_index >= 0 && curr_index < prices.Length) { + var gr = format_text(piqs[i].piq.ToString(), curr_index, row_height, settings, glyph_context); + dc.DrawGlyphRun(foreground, gr); + } else break; + } + } + } + } + public static void render_mkt_break(DrawingVisual dv, int index, double row_height, uint color) => render_single_grid_line(dv, index, row_height, color, 0, SystemParameters.FullPrimaryScreenWidth); From 391397692c12430452edf2b6631244e1a03c3df6 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:08:28 -0500 Subject: [PATCH 035/100] adding PIQ case option --- src/client/spark/view/Spark/Ladder/LadderView.xaml.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs b/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs index 273bb4d9a..03e811d34 100644 --- a/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs +++ b/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs @@ -141,6 +141,10 @@ namespace mbts.spark { header = "Last"; width_prop = nameof(LadderVM.last_column_width); break; + case LadderGridHost.ColumnType.PIQ: + header = "PIQ"; + width_prop = nameof(LadderVM.piq_column_width); + break; default: throw ExceptionBuilder.Argument.unexpected_case(type, nameof(type)); } return _create(host, header, $"DataContext.{width_prop}"); From a646f288316787c3f9b5e647180319b1c67b0aae Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:14:05 -0500 Subject: [PATCH 036/100] added min_piq property; now stores (OrderRecordImmutable, uint?) instead of just OrderRecordImmutable --- .../spark/view/Spark/Ladder/LadderOrderVM.cs | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs b/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs index 792b59a84..2768b017b 100644 --- a/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs @@ -46,9 +46,9 @@ namespace mbts.spark { int working_quantity; LadderModel model => _parent.model; - Dictionary _orders = new Dictionary(); + Dictionary _orders = new Dictionary(); - public LadderOrderVM(OrderRecordImmutable o, double real_price, LadderVM parent, IBus bus, IObservable bs) { + public LadderOrderVM((OrderRecordImmutable order, uint? p) op, double real_price, LadderVM parent, IBus bus, IObservable bs) { _parent = parent; _bus = bus; _bs = bs; @@ -73,7 +73,7 @@ namespace mbts.spark { double get_font_size(LadderSettings s) => s.orders_column.font_size; Brush get_foreground(LadderSettings s) => s.orders_column.foreground.rgb_brush(); // - add_order(o); + add_order(op); _dispose = Disposable.Create( _bs.Subscribe(badge => { @@ -109,19 +109,19 @@ namespace mbts.spark { // if this badge only represents a single order, then is_exchange_order and the border (= algo color) // should be set appropriately if (_orders.Count == 1) { - var single_order = _orders.Values.First(); + var single_order = _orders.Values.First().order; var single_order_metadata = _parent.model.order_gateway.factory.get_metadata(single_order); border = single_order_metadata.Color.rgb_brush(); - is_exchange_order = OrderUtils.is_exchange_order(_orders.Values.First()); + is_exchange_order = OrderUtils.is_exchange_order(_orders.Values.First().order); } else { border = new SolidColorBrush(color); is_exchange_order = false; } has_multiple_orders_at_same_level = _orders.Count > 1; - is_hedge_order = _orders.Any(o => o.Value.sub_type() == OrderSubType.HedgingLeg); + is_hedge_order = _orders.Any(op => op.Value.order.sub_type() == OrderSubType.HedgingLeg); if (_orders.Any()) { - summary = _orders.Values.Select(order => order.short_description(_parent.security.ContractSpec)).Aggregate((s1, s2) => $"{s1}{Environment.NewLine}{Environment.NewLine}{s2}"); + summary = _orders.Values.Select(op => op.order.short_description(_parent.security.ContractSpec)).Aggregate((s1, s2) => $"{s1}{Environment.NewLine}{Environment.NewLine}{s2}"); } _set_badge_text(); @@ -132,8 +132,8 @@ namespace mbts.spark { RaisePropertyChanged(nameof(text_lines)); } - public void add_order(OrderRecordImmutable new_order) { - _orders[new_order.Key] = new_order; + public void add_order((OrderRecordImmutable order, uint? p) new_op) { + _orders[new_op.order.Key] = new_op; } public void remove_order(uint order_key) { @@ -149,10 +149,11 @@ namespace mbts.spark { public uint security_key => _parent.security.Id; public double price { get => double.IsNaN(_phantom_price) ? _real_price : _phantom_price; } + public uint? min_piq => _orders.Select(op => op.Value.p).Where(piq => piq.HasValue).Min(); public double real_price => _real_price; public Side side { get { - var distinct_sides = _orders.Values.Select(o => o.side()).Distinct(); + var distinct_sides = _orders.Values.Select(op => op.order.side()).Distinct(); if (distinct_sides.Count() > 1) { return Side.Undefined; } @@ -160,11 +161,11 @@ namespace mbts.spark { } } - IEnumerable _parent_and_child_not_sharing_slot() => _orders.Values.Where(o => !_orders.ContainsKey(o.ParentKey)); + IEnumerable _parent_and_child_not_sharing_slot() => _orders.Values.Select(op => op.order).Where(o => !_orders.ContainsKey(o.ParentKey)); public IReadOnlyList text_lines { get; private set; } - public bool has_parent => _orders.Values.Any( order => order.has_parent()); + public bool has_parent => _orders.Values.Any( op => op.order.has_parent()); public bool has_phantom_price => !double.IsNaN(_phantom_price); public ICommand detach_command => new DelegateCommand(() => _execute_command(_crack_order), () => !has_parent); @@ -191,16 +192,16 @@ namespace mbts.spark { // the setter is public so that the LadderVM can set this field public CustomOrderDefinition.CustomOneofCase parent_algo_type { get => _parent_algo_type.Value; set => _parent_algo_type.Value = value; } - public IEnumerable orders => _orders.Values; + public IEnumerable orders => _orders.Values.Select(o => o.order); void _cancel_order() { - foreach (var order in _orders.Values) { - model.order_gateway.cancel_order(order).ignore(); + foreach (var op in _orders.Values) { + model.order_gateway.cancel_order(op.order).ignore(); } } void _crack_order() { - var distinct_parent_keys = _orders.Values.Where(o => o.has_parent()).Select(x => x.ParentKey).Distinct(); + var distinct_parent_keys = _orders.Values.Where(op => op.order.has_parent()).Select(x => x.order.ParentKey).Distinct(); model.order_gateway.crack_orders(distinct_parent_keys).ignore(); } @@ -279,7 +280,7 @@ namespace mbts.spark { } public override string ToString() { - return $"Price: {this.price} Order Count: {this.orders.Count()}"; + return $"Price: {this.price} Order Count: {this.orders.Count()} PIQ: {(this.min_piq.HasValue ? this.min_piq.Value.ToString() : "")}"; } } } From 096bf3338c49a9a7c59ef1d5d3d3ef6837ac2af0 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:15:06 -0500 Subject: [PATCH 037/100] adding PIQ --- src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs b/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs index 3c4d7869d..43999f483 100644 --- a/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs +++ b/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs @@ -97,13 +97,13 @@ namespace mbts.ui.controls { TotalPnL = "Total PnL", PnLPrice = "PnL Price", Cancel = "Cancel", - MarketDescription = "Market Description", ClearedAlias = "Cleared Alias", ProductDescription = "Product Description", HubAlias = "Hub Alias", StripName = "Strip Name", - MicCode = "Mic Code" + MicCode = "Mic Code", + PositionInQueue = "PIQ" ; } From fa287b14ef266b562efbe2c7b72a631b0787dd1f Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:16:40 -0500 Subject: [PATCH 038/100] OrderSecurity now also has a piq; maybe we change the class name? --- src/client/spark/Orders/OrderSecurity.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/client/spark/Orders/OrderSecurity.cs b/src/client/spark/Orders/OrderSecurity.cs index 174d83344..eb46dc351 100644 --- a/src/client/spark/Orders/OrderSecurity.cs +++ b/src/client/spark/Orders/OrderSecurity.cs @@ -4,21 +4,24 @@ using System; namespace bts.api.orderman { public readonly struct OrderSecurity: IEquatable { - public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order) { + public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order, uint? piq) { this.security = sec; this.order = order; + this.piq = piq; } - public OrderSecurity((SecurityImmutable? sec, OrderRecordImmutable order) t) { + public OrderSecurity(OrderSecPiq t) { security = t.sec; order = t.order; + piq = t.piq; } - public static implicit operator OrderSecurity(ValueTuple x) => new OrderSecurity(x); + public static implicit operator OrderSecurity(OrderSecPiq x) => new OrderSecurity(x); public SecurityImmutable? security { get; } public OrderRecordImmutable order { get; } + public uint? piq { get; } public override bool Equals(object obj) => (obj is OrderSecurity os) && Equals(os); - public bool Equals(OrderSecurity other) => order.Equals(other.order) && security == other.security; - public override int GetHashCode() => (security, order).GetHashCode(); + public bool Equals(OrderSecurity other) => order.Equals(other.order) && security == other.security && piq == other.piq; + public override int GetHashCode() => (security, order, piq).GetHashCode(); public static bool operator == (OrderSecurity left, OrderSecurity right) => left.Equals(right); public static bool operator != (OrderSecurity left, OrderSecurity right) => !left.Equals(right); From d9b4e21c6156d2cdcaa3ed7f0e45a43b20e2a8c0 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:20:09 -0500 Subject: [PATCH 039/100] changing from tuple to simple OrderSecPiq for sanity --- src/client/spark/Daemons/OrderTicketDaemon.cs | 2 +- src/client/spark/Notifications/INotificationService.cs | 2 +- src/client/spark/Notifications/INotificationServiceDataCache.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/spark/Daemons/OrderTicketDaemon.cs b/src/client/spark/Daemons/OrderTicketDaemon.cs index 84e473c46..fa6091f92 100644 --- a/src/client/spark/Daemons/OrderTicketDaemon.cs +++ b/src/client/spark/Daemons/OrderTicketDaemon.cs @@ -19,7 +19,7 @@ namespace bts.shell { readonly OrdersModel _model; readonly ErrorReporter _error_reporter; IReadOnlyDictionary _md_map; - TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> _orders; + TableUpdate _orders; IDisposable _dispose; [ImportingConstructor] diff --git a/src/client/spark/Notifications/INotificationService.cs b/src/client/spark/Notifications/INotificationService.cs index a95ac2048..27086e52c 100644 --- a/src/client/spark/Notifications/INotificationService.cs +++ b/src/client/spark/Notifications/INotificationService.cs @@ -55,7 +55,7 @@ namespace bts.shell { void _on_activity(Activity activity) => _activity_notifications.process_item(activity); - void _on_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable),uint> update) { + void _on_orders(TableUpdate update) { // must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history _data_cache.update(update.as_iincremental()); foreach (var o in update.added.Concat(update.changed)) diff --git a/src/client/spark/Notifications/INotificationServiceDataCache.cs b/src/client/spark/Notifications/INotificationServiceDataCache.cs index fac9deeed..00fb48eb0 100644 --- a/src/client/spark/Notifications/INotificationServiceDataCache.cs +++ b/src/client/spark/Notifications/INotificationServiceDataCache.cs @@ -50,7 +50,7 @@ namespace bts.shell { public DateTime last_order_reset_time { get; private set; } public bool is_first_order_update { get; private set; } - internal void update(IIncrementalUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> data) { + internal void update(IIncrementalUpdate data) { DateTime now = DateTime.Now; if (data.removed.Count == _order_cache.Count) { last_order_reset_time = now; From 9a39d84739e256b73be9bfecc5857866432d6f6e Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:20:42 -0500 Subject: [PATCH 040/100] tuple to OrderSecPiq in here too --- src/client/spark/Orders/OrderActivitySummarizer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/spark/Orders/OrderActivitySummarizer.cs b/src/client/spark/Orders/OrderActivitySummarizer.cs index 3bcb6de13..42e934664 100644 --- a/src/client/spark/Orders/OrderActivitySummarizer.cs +++ b/src/client/spark/Orders/OrderActivitySummarizer.cs @@ -27,7 +27,7 @@ namespace bts.spark { static readonly IEqualityComparer _id_comparer = bts.utils.EqualityComparer.Create(a => a.Key); - public OrderActivitySummarizer(IBus bus, IObservable> orders) { + public OrderActivitySummarizer(IBus bus, IObservable> orders) { _bus = bus; _dispose = Disposable.Create( bus.subscribe(SparkTags.SecuritiesTag).Subscribe(sec => _security = sec), @@ -124,7 +124,7 @@ namespace bts.spark { #endregion - void orders_updated(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) { + void orders_updated(TableUpdate update) { foreach (var o in update.added) { if (!string.IsNullOrEmpty(o.order.State.LastMsg) && o.order.State.Created.ToDateTime().Date == DateTime.Today) { From 567edafe187b271e397d371bffa77c5f73ccf479 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:23:29 -0500 Subject: [PATCH 041/100] including the piq here on this new --- .../spark/Predicates/Data/Events/OrderFilledEventPredicate.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs index f8ca8f2ab..8f542fd5a 100644 --- a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs @@ -82,7 +82,7 @@ namespace mbts { // old order will be a copy of new order, except with filledQuantity set to 0 var old_state = ort.order.State.CloneToMutable(); old_state.FilledQuantity = 0; - return (new OrderSecurity(ort.security, ort.order.with(state: old_state)), ort); + return (new OrderSecurity(ort.security, ort.order.with(state: old_state), ort.piq), ort); } } return null; From b2ed88fb4a200dcfe9f63c8a9a9b013353c21861 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:23:57 -0500 Subject: [PATCH 042/100] adding binding expression for new PIQColumnWidth property --- src/client/spark/view/Spark/Ladder/LadderView.xaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/spark/view/Spark/Ladder/LadderView.xaml b/src/client/spark/view/Spark/Ladder/LadderView.xaml index 889e8e8bd..dc8b3afc2 100644 --- a/src/client/spark/view/Spark/Ladder/LadderView.xaml +++ b/src/client/spark/view/Spark/Ladder/LadderView.xaml @@ -541,6 +541,7 @@ price_column_width="{Binding price_column_width, Mode=TwoWay}" ask_column_width="{Binding ask_column_width, Mode=TwoWay}" LastColumnWidth="{Binding last_column_width, Mode=TwoWay}" + PIQColumnWidth="{Binding piq_column_width, Mode=TwoWay}" VolumeColumnWidth="{Binding volume_column_width, Mode=TwoWay}" > From c312a880a7b992ff38b1cc6689c7592ed7342c4e Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:29:29 -0500 Subject: [PATCH 043/100] adding MBOService for piq, left joining piq on orders --- src/client/spark/Orders/OrdersModel.cs | 32 +++++++++++++++++++++----- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index f8a4cca8c..977eb721f 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -3,6 +3,7 @@ using bts.api; using bts.api.core; using bts.api.orderman; using bts.api.position; +using bts.md; using bts.spark; using bts.utils; using mbts.collections; @@ -16,8 +17,9 @@ using System.ComponentModel.Composition; using System.Linq; using System.Reactive.Concurrency; using System.Reactive.Linq; +using System.Reactive.Subjects; using System.Threading.Tasks; -using OrderUpdate = bts.api.core.TableUpdate<(bts.api.SecurityImmutable? sec, bts.api.orderman.OrderRecordImmutable order), uint>; +using OrderUpdate = bts.api.core.TableUpdate; namespace mbts.spark { [Export] @@ -26,6 +28,7 @@ namespace mbts.spark { readonly OrderFactory _factory; readonly OrderService _service; readonly PositionService _position_service; + readonly MBOService _mbo_service; readonly SpreaderModel _spreader; readonly ErrorReporter _error; readonly DynamicOrderHelper _dynamic_helper; @@ -35,12 +38,13 @@ namespace mbts.spark { IDisposable _dispose; [ImportingConstructor] - OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, + OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, MBOService mbo_service, SpreaderModel spreader, DynamicOrderHelper dynamic_helper, AdminModel admin_model, IAlgoStudioModel algo_model, SecurityStore sstore, ErrorReporter error) { _bus = bus; _factory = factory; _service = service; _position_service = position_service; + _mbo_service = mbo_service; _spreader = spreader; _error = error; _dynamic_helper = dynamic_helper; @@ -48,13 +52,26 @@ namespace mbts.spark { this.algo_model = algo_model; var scheduler = ThreadPoolScheduler.Instance; var interval = TimeSpan.FromMilliseconds(100); - on_orders = + on_orders = _service .subscribei_orders() .Naggle(interval, scheduler) .PermissionFilter(admin_model.on_admin_data) .join(sstore, order => order.security_key()) - .Table().Cache(); + .Table() + .Cache() + .LeftJoin< + (SecurityImmutable?, OrderRecordImmutable), + OrderPIQImmutable, + OrderSecPiq, + uint + >(mbo_service.subscribei_piq().Table(), + si_ort => si_ort.Item2.Key, + p => p.OrderId, + (si_ort, r, p) => new OrderSecPiq(si_ort.Item1, si_ort.Item2, p.Position), + (si_ort, r) => new OrderSecPiq(si_ort.Item1, si_ort.Item2, null), + osp => osp.order.Key); + on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); working_orders = on_orders.Scan( @@ -103,7 +120,9 @@ namespace mbts.spark { f => f.fill.FillKey); on_fill_tuples = fill_order_tuples_join; _dispose = Disposable.Create( - on_orders.Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), + // on_orders changed from an IObservable to an IConnectableObservable, so we call + // RefCount() here to start the connection + on_orders.RefCount().Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), on_positions.Subscribe(_security_positions_demuxer), order_secs_algo_meta_join.Connect(), algo_meta_join.Connect(), @@ -120,12 +139,13 @@ namespace mbts.spark { public OrderFactory factory => _factory; public SpreaderModel spreader => _spreader; public DynamicOrderHelper dynamic_helper => _dynamic_helper; - public IObservable on_orders { get; } + public IConnectableObservable on_orders { get; } public IObservable> on_order_tuples { get; } public IObservable> on_order_securities { get; } public IObservable> on_fill_tuples { get; } public IObservable> working_orders { get; } public IObservable> on_positions { get; } + public AdminModel admin_model { get; } public IAlgoStudioModel algo_model { get; } From d76c2a30ceb4a9d1cf53227929eee176a862fc50 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:32:54 -0500 Subject: [PATCH 044/100] price_vs_piq property on LadderVM; appropriate changes for LadderOrderVM constructor --- src/client/spark/view/Spark/Ladder/LadderVM.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index 5be07f911..0bf612489 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -26,7 +26,7 @@ namespace mbts.spark { readonly LadderModel _model; readonly Property _high, _low, _change, _open, _settlement, _position, _left_panel_width, _zoom_factor; //readonly Property _window_width, _window_height; - readonly Property _orders_column_width, _bid_column_width, _price_column_width, _ask_column_width, _last_column_width, _volume_column_width; + readonly Property _orders_column_width, _bid_column_width, _price_column_width, _ask_column_width, _last_column_width, _volume_column_width, _piq_column_width; readonly Property _volume; readonly Property _auto_clear, _show_day, _show_high, _show_low, _show_open, _show_settle, _show_volume, _show_account, _show_tif, _show_order_type, _show_account_position, _show_size_preset, _show_auto_clear, _show_clear, _show_cancel_buys, @@ -58,6 +58,7 @@ namespace mbts.spark { _price_column_width = backing(nameof(price_column_width), ws.price_column_width); _ask_column_width = backing(nameof(ask_column_width), ws.ask_column_width); _last_column_width = backing(nameof(last_column_width), ws.last_column_width); + _piq_column_width = backing(nameof(piq_column_width), ws.piq_column_width); _volume_column_width = backing(nameof(volume_column_width), ws.volume_column_width); _left_panel_width = backing(nameof(left_panel_width), ws.order_entry_panel_width); _zoom_factor = backing(nameof(zoom_factor), ws.zoom_factor); @@ -157,9 +158,9 @@ namespace mbts.spark { } if (_slot_price_vs_orders.TryGetValue(slot_price_target, out LadderOrderVM slot_vm)) { - slot_vm.add_order(wo); + slot_vm.add_order((wo, osec.piq)); } else { - slot_vm = new LadderOrderVM(wo, slot_price_target, this, _bus, _settings_manager.subscribe_current().Select(s => s.badge_style)); + slot_vm = new LadderOrderVM((wo, osec.piq), slot_price_target, this, _bus, _settings_manager.subscribe_current().Select(s => s.badge_style)); _slot_price_vs_orders[slot_price_target] = slot_vm; } @@ -232,6 +233,7 @@ namespace mbts.spark { public event Action on_tick; public MDConsumerHandle md => _model.md; + public IEnumerable<(double, uint)> price_vs_piq => _slot_price_vs_orders.Values.Where(result => result.min_piq.HasValue).Select(o => (o.price, o.min_piq.Value)); public bool is_synthetic => _model.security.ContractCase == Security.ContractOneofCase.Synthetic; Property _status; @@ -306,13 +308,14 @@ namespace mbts.spark { public uint price_column_width { get => _price_column_width.Value; set => _price_column_width.set(value); } public uint ask_column_width { get => _ask_column_width.Value; set => _ask_column_width.set(value); } public uint last_column_width { get => _last_column_width.Value; set => _last_column_width.set(value); } + public uint piq_column_width { get => _piq_column_width.Value; set => _piq_column_width.set(value); } public uint volume_column_width { get => _volume_column_width.Value; set => _volume_column_width.set(value); } internal LadderModel model => _model; internal IObservable settings => _settings_manager.subscribe_current(); public LadderSettings get_save_state() { var state = _settings_manager.current_settings(); - state = state.with_window_specifics(new LadderSettings.WindowSpecifics(left_panel_width, orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width, zoom_factor)); + state = state.with_window_specifics(new LadderSettings.WindowSpecifics(left_panel_width, orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width, zoom_factor, piq_column_width)); state.use_defaults = _settings_manager.instance_settings == null; return state; } From 42feedfd36e1ebd95aa7ee0b41961c25d4202d5b Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:33:48 -0500 Subject: [PATCH 045/100] adding piq column to settings --- .../spark/view/Spark/Ladder/LadderSettings.cs | 19 +++++++++++++++---- .../view/Spark/Ladder/LadderSettingsVM.cs | 8 +++++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderSettings.cs b/src/client/spark/view/Spark/Ladder/LadderSettings.cs index 6d1784488..e950d5587 100644 --- a/src/client/spark/view/Spark/Ladder/LadderSettings.cs +++ b/src/client/spark/view/Spark/Ladder/LadderSettings.cs @@ -249,7 +249,8 @@ namespace mbts.spark { bid_column = LadderColumnSettings.Default, ask_column = LadderColumnSettings.Default, price_column = LadderColumnSettings.Default, - last_column = LadderColumnSettings.Default; + last_column = LadderColumnSettings.Default, + piq_column = LadderColumnSettings.Default; orders_column.background = BTSColors.LadderSpecial.default_column_background; bid_column.foreground = 0xFFFFFF; bid_column.background = BTSColors.MarketData.bid; @@ -257,6 +258,7 @@ namespace mbts.spark { ask_column.background = BTSColors.MarketData.ask; price_column.background = BTSColors.LadderSpecial.default_column_background; last_column.background = 0xFFFFFF; + piq_column.background = 0xFFFFFF; return new LadderSettings { row_height = 20, grid_line_color = 0x535353, @@ -265,6 +267,7 @@ namespace mbts.spark { ask_column = ask_column, price_column = price_column, last_column = last_column, + piq_column = piq_column, volume_column = VolumeColumnSettings.Default, calculator_buttons = new[] { 1, 5, 10, 25, 50, 100 }, scroll_bar_visible = false, @@ -417,6 +420,9 @@ namespace mbts.spark { [ProtoMember(48)] public NetChangeTerms net_change { get; set; } + [ProtoMember(49)] + public LadderColumnSettings piq_column { get; set; } + public bool Equals(LadderSettings other) => other != null && row_height == other.row_height && @@ -426,6 +432,7 @@ namespace mbts.spark { ask_column.safe_equals(other.ask_column) && price_column.safe_equals(other.price_column) && last_column.safe_equals(other.last_column) && + piq_column.safe_equals(other.piq_column) && calculator_buttons.safe_equals(other.calculator_buttons, (a, b) => a.SequenceEqual(b)) && scroll_bar_visible == other.scroll_bar_visible && default_order_type == other.default_order_type && @@ -476,6 +483,7 @@ namespace mbts.spark { EqualityComparer.safe_get_hash_code(ask_column), EqualityComparer.safe_get_hash_code(price_column), EqualityComparer.safe_get_hash_code(last_column), + EqualityComparer.safe_get_hash_code(piq_column), EqualityComparer.safe_get_hash_code(calculator_buttons), scroll_bar_visible.GetHashCode(), default_order_type.GetHashCode(), @@ -517,12 +525,12 @@ namespace mbts.spark { public struct WindowSpecifics { public readonly double order_entry_panel_width, zoom_factor; - public readonly uint orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width; + public readonly uint orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width, piq_column_width; public WindowSpecifics(LadderSettings s) - : this(s.order_entry_panel_initial_width, s.orders_column.width, s.bid_column.width, s.price_column.width, s.ask_column.width, s.last_column.width, s.volume_column.general.width, s.zoom_factor) { } + : this(s.order_entry_panel_initial_width, s.orders_column.width, s.bid_column.width, s.price_column.width, s.ask_column.width, s.last_column.width, s.volume_column.general.width, s.zoom_factor, s.piq_column.width) { } - public WindowSpecifics(double order_entry_panel_width, uint orders_column_width, uint bid_column_width, uint price_column_width, uint ask_column_width, uint last_column_width, uint volume_column_width, double zoom_factor) { + public WindowSpecifics(double order_entry_panel_width, uint orders_column_width, uint bid_column_width, uint price_column_width, uint ask_column_width, uint last_column_width, uint volume_column_width, double zoom_factor, uint piq_column_width) { this.order_entry_panel_width = order_entry_panel_width; this.orders_column_width = orders_column_width; this.bid_column_width = bid_column_width; @@ -531,6 +539,7 @@ namespace mbts.spark { this.last_column_width = last_column_width; this.volume_column_width = volume_column_width; this.zoom_factor = zoom_factor; + this.piq_column_width = piq_column_width; } } } @@ -543,6 +552,7 @@ namespace mbts.spark { yield return settings.ask_column; yield return settings.volume_column.general; yield return settings.last_column; + yield return settings.piq_column; } public static IEnumerable visible_columns(this LadderSettings settings) => @@ -560,6 +570,7 @@ namespace mbts.spark { source.price_column.width = specifics.price_column_width; source.ask_column.width = specifics.ask_column_width; source.last_column.width = specifics.last_column_width; + source.piq_column.width = specifics.piq_column_width; source.volume_column.general.width = specifics.volume_column_width; source.zoom_factor = specifics.zoom_factor; return source; diff --git a/src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs b/src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs index b3f9bed8b..295aa6739 100644 --- a/src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs @@ -29,7 +29,8 @@ namespace mbts.spark { _bid = new LadderColumnSettingsVM("Bid"), _price = new LadderColumnSettingsVM("Price"), _ask = new LadderColumnSettingsVM("Ask"), - _last = new LadderColumnSettingsVM("Last", true); + _last = new LadderColumnSettingsVM("Last", true), + _piq = new LadderColumnSettingsVM("PIQ", true); VolumeColumnVM _volume = new VolumeColumnVM(); LadderMarketBreakLineVM _bid_mkt_line = new LadderMarketBreakLineVM("Bid"), @@ -82,6 +83,7 @@ namespace mbts.spark { _price.font_size_changed += on_font_size_changed; _ask.font_size_changed += on_font_size_changed; _last.font_size_changed += on_font_size_changed; + _piq.font_size_changed += on_font_size_changed; void on_font_size_changed(double oldval, double newval) => _row_height.refresh(); } @@ -120,6 +122,7 @@ namespace mbts.spark { if (!is_synthetic) { yield return _volume.general; yield return _last; + yield return _piq; } } } @@ -130,6 +133,7 @@ namespace mbts.spark { public LadderColumnSettingsVM ask_column => _ask; public LadderColumnSettingsVM volume_column => _volume.general; public LadderColumnSettingsVM last_column => _last; + public LadderColumnSettingsVM piq_column => _piq; public VolumeColumnVM volume => _volume; public LadderMarketBreakLineVM session_range_fill => _session_range_fill; public ObservableEditableCollection> size_presets { get; } @@ -209,6 +213,7 @@ namespace mbts.spark { _price.assign_from(_current.price_column); _volume.assign_from(_current.volume_column); _last.assign_from(_current.last_column); + _piq.assign_from(_current.piq_column); _bid_mkt_line.assign_from(_current.bid_break_line); _ask_mkt_line.assign_from(_current.ask_break_line); _mid_mkt_line.assign_from(_current.mid_break_line); @@ -254,6 +259,7 @@ namespace mbts.spark { clone.ask_column = _ask.get_settings(); clone.volume_column = _volume.get_settings(); clone.last_column = _last.get_settings(); + clone.piq_column = _piq.get_settings(); clone.bid_break_line = _bid_mkt_line.get_settings(); clone.ask_break_line = _ask_mkt_line.get_settings(); clone.mid_break_line = _mid_mkt_line.get_settings(); From c32eada3aca17bfe39a8293cc39399a86f022750 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:49:03 -0500 Subject: [PATCH 046/100] the big enchilada - adding PIQ column to LadderGridHost --- .../spark/view/Spark/Ladder/LadderGridHost.cs | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderGridHost.cs b/src/client/spark/view/Spark/Ladder/LadderGridHost.cs index 2f91999d2..516ccf0f8 100644 --- a/src/client/spark/view/Spark/Ladder/LadderGridHost.cs +++ b/src/client/spark/view/Spark/Ladder/LadderGridHost.cs @@ -42,7 +42,8 @@ namespace mbts.spark { _ask_vis_tup = new VisualGlyphTuple(), _price_vis_tup = new VisualGlyphTuple(), _last_vis_tup = new VisualGlyphTuple(), - _volume_vis_tup = new VisualGlyphTuple(); + _volume_vis_tup = new VisualGlyphTuple(), + _piq_vis_tup = new VisualGlyphTuple(); readonly RefValueContainer> _volume_container = new RefValueContainer>(); @@ -92,7 +93,7 @@ namespace mbts.spark { }; } - public enum ColumnType { Order, Bid, Price, Ask, Last, Volume, None } + public enum ColumnType { Order, Bid, Price, Ask, Last, Volume, None, PIQ } public event Action visible_columns_changed; @@ -192,6 +193,18 @@ namespace mbts.spark { set => SetValue(LastColumnWidthProperty, value); } + public static readonly DependencyProperty PIQColumnWidthProperty = + WPFHelper.create_dp( + nameof(PIQColumnWidth), + FrameworkPropertyMetadataOptions.BindsTwoWayByDefault, + (o, oldval, newval) => o._adjust_column_width(o.columns.FirstOrDefault(c => c.column_type == ColumnType.PIQ), (double)newval - (double)oldval), + 50u); + + public uint PIQColumnWidth { + get => (uint)GetValue(PIQColumnWidthProperty); + set => SetValue(PIQColumnWidthProperty, value); + } + #region Command public ICommand Command { get => (ICommand)GetValue(CommandProperty); @@ -404,7 +417,8 @@ namespace mbts.spark { double x = 0; var col_list = new List { new Column(_settings.orders_column, ColumnType.Order, null, x), - new Column(_settings.bid_column, ColumnType.Bid, _bid_vis_tup, (x += _settings.orders_column.show ? _settings.orders_column.width : 0)), + new Column(_settings.piq_column, ColumnType.PIQ, _piq_vis_tup, (x += _settings.orders_column.show ? _settings.orders_column.width : 0)), + new Column(_settings.bid_column, ColumnType.Bid, _bid_vis_tup, (x += _settings.piq_column.show ? _settings.piq_column.width : 0)), (_price_column = new Column(_settings.price_column, ColumnType.Price, _price_vis_tup, (x += _settings.bid_column.show ? _settings.bid_column.width : 0))), new Column(_settings.ask_column, ColumnType.Ask, _ask_vis_tup, (x += _settings.price_column.show ? _settings.price_column.width : 0)) }; @@ -509,6 +523,8 @@ namespace mbts.spark { if (depth_changed) _refresh_sizes(); } + // TODO - detect changes in PIQ to selectively redraw + _refresh_piq(); _refresh_volume_at_price(); _last_origin_price = _mkt_break_lines.origin; _price_dirty = false; @@ -545,6 +561,14 @@ namespace mbts.spark { LadderRender.render_last(_last_vis_tup.visual, _last_vis_tup.glyph_context, _last_trade, last_index, _settings.row_height, _settings.last_column); } + void _refresh_piq() { + if (!_settings.piq_column.show) + return; + int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer); + var piq_list = _vm.price_vs_piq; + LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, origin_index, _price_list, piq_list.ToArray(), _settings.row_height, security.ContractSpec, _settings.piq_column); + } + void _refresh_sizes() { Log.assert_d(_price_list != null); int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer); @@ -693,6 +717,7 @@ namespace mbts.spark { _ask_vis_tup.glyph_context = glyph_context(s.ask_column); _price_vis_tup.glyph_context = glyph_context(s.price_column); _last_vis_tup.glyph_context = glyph_context(s.last_column); + _piq_vis_tup.glyph_context = glyph_context(s.piq_column); _volume_vis_tup.glyph_context = glyph_context(s.volume_column.general); GlyphContext glyph_context(LadderColumnSettings cs) => @@ -720,6 +745,7 @@ namespace mbts.spark { case ColumnType.Price: SetCurrentValue(price_column_widthProperty, col.settings.width); break; case ColumnType.Ask: SetCurrentValue(ask_column_widthProperty, col.settings.width); break; case ColumnType.Last: SetCurrentValue(LastColumnWidthProperty, col.settings.width); break; + case ColumnType.PIQ: SetCurrentValue(PIQColumnWidthProperty, col.settings.width); break; case ColumnType.Volume: SetCurrentValue(VolumeColumnWidthProperty, col.settings.width); _volume_container.mark_dirty(); From 6181548902e40239814b607db626eaf8578fbf9c Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:56:31 -0500 Subject: [PATCH 047/100] piq background color now matches orders background color --- src/client/spark/view/Spark/Ladder/LadderSettings.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderSettings.cs b/src/client/spark/view/Spark/Ladder/LadderSettings.cs index e950d5587..329082867 100644 --- a/src/client/spark/view/Spark/Ladder/LadderSettings.cs +++ b/src/client/spark/view/Spark/Ladder/LadderSettings.cs @@ -258,7 +258,7 @@ namespace mbts.spark { ask_column.background = BTSColors.MarketData.ask; price_column.background = BTSColors.LadderSpecial.default_column_background; last_column.background = 0xFFFFFF; - piq_column.background = 0xFFFFFF; + piq_column.background = BTSColors.LadderSpecial.default_column_background; return new LadderSettings { row_height = 20, grid_line_color = 0x535353, From 905c1cab2ce40b09214b60240834702e58b22fe7 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 16 Apr 2019 10:08:50 -0400 Subject: [PATCH 048/100] SPARK-2206 tests --- src/lib/md/test/mbo_tests.cpp | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/lib/md/test/mbo_tests.cpp diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp new file mode 100644 index 000000000..70c4492e9 --- /dev/null +++ b/src/lib/md/test/mbo_tests.cpp @@ -0,0 +1,27 @@ +#ifndef _WIN32 +#include "catch/catch.hpp" +#include "../MBOProcessor.h" +namespace bts { +namespace md { +namespace test { +int mbo_tests = 0; + +TEST_CASE("mbo basic", "[md][mbo]") { + MBOProcessor p; + // publish a snapshot + auto sec = SecurityKey(99); + { + auto s = p.writer().allocate_snapshot(); + s->bids.push_back({ ExchangeOrderKey(1), { 100, 0 }, 42 }); + p.writer().publish_snapshot(sec, s); + } + + auto s = p.reader().acquire_snapshot(sec); + REQUIRE(s); + REQUIRE(s->bids.size() == 1); + REQUIRE(s->bids[0].id.value() == 1); + REQUIRE(s->asks.size() == 0); +} + +}}} +#endif \ No newline at end of file From d7850d6b7040627f1600d5a30d3aa9caa311df27 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 16 Apr 2019 12:14:15 -0400 Subject: [PATCH 049/100] MBO diagnostic api --- src/client/sparkcmd/Cmd/ServiceTestCommand.cs | 56 +++++++++---------- src/lib/api/services/MBOService.proto | 20 ++++++- src/lib/md/MBOServiceServer.cpp | 28 ++++++++++ src/lib/md/MBOServiceServer.h | 2 + 4 files changed, 75 insertions(+), 31 deletions(-) diff --git a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs index 1f449cd4d..daf94bc80 100644 --- a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs +++ b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs @@ -10,6 +10,7 @@ using System.Reactive.Linq; using System.Threading; using System.Threading.Tasks; using System.Linq; +using bts.md; namespace bts { public class ServiceTest : ConsoleCommand { @@ -22,6 +23,7 @@ namespace bts { HasOption("synthetic", "Run synthetic test", _ => _synthetic = true); HasOption("user", "user test", _ => _user = true); HasOption("admin", "admin test", _ => _admin = true); + HasAdditionalArguments(null, " [Keys to check]"); } public override int Run(string[] remainingArguments) { if (_synthetic) { @@ -34,41 +36,37 @@ namespace bts { admin_test(); } else { - // mbo_test(); + mbo_test(remainingArguments); } return 0; } + static string try_get(string[] args, int index) { + if (args.Length < index) { + return args[index]; + } + return ""; + } - // private static void mbo_test() { - // using (var ctx = new ServiceContext()) { - // var relay_svc = ctx.locate(); - // var order_sv = ctx.locate(); + private static void mbo_test(string[] args) { + var key = UInt32.Parse(args.FirstOrDefault()); + var side = try_get(args, 1) == "bid" ? api.Side.Buy : api.Side.Sell; + using (var ctx = new ServiceContext()) { + var mbo = ctx.locate(); + var data = mbo.get_orders_for_security(new MBOSecurityQuery { SecurityId = key }); - // var upd = order_sv.subscribei_orders().FirstAsync().Wait(); /* don't use Wait() */ - // var orders = - // upd.all - // .Where(o => - // o.State.Status == OrderStatus.Working && - // o.State.native_order_state().HasValue) - // .ToList(); - - // var request = new PIQRequest(); - // request.Orders.AddRange(orders.Select(o => - // new OrderQuery { - // SecurityId = o.Def.SecurityKey, - // OrderId = UInt64.Parse(o.State.native_order_state().Value.ExchangeOrderId) })); - - // Console.WriteLine($"Requesting positions for {request.Orders.Count} orders..."); - // var result = relay_svc.get_piq(request); - - // Console.WriteLine($"got response - {result.Position.Count} entries"); - // foreach (var kvp in result.Position) { - // Console.Write($"> {kvp.Key} => "); - // ConsoleEx.WriteLine($"{kvp.Value}", ConsoleColor.Yellow); - // } - // } - // } + Console.WriteLine($"Order data for {data.SecurityId}"); + var vector = side == api.Side.Buy ? data.Bid : data.Ask; + var price = 0.0; + foreach(var e in vector) { + if (e.Price != price) { + price = e.Price; + Console.WriteLine("================"); + } + Console.WriteLine(e); + } + } + } private static void synthetic_test() { using (var ctx = new ServiceContext()) { diff --git a/src/lib/api/services/MBOService.proto b/src/lib/api/services/MBOService.proto index 4a8383f8b..8250375cc 100644 --- a/src/lib/api/services/MBOService.proto +++ b/src/lib/api/services/MBOService.proto @@ -6,10 +6,26 @@ import "lib/api/apioptions.proto"; import "lib/api/apicore.proto"; message OrderPIQ { - uint32 order_id = 1 [(bts.api.key)=true]; - uint32 position = 2; + uint32 order_id = 1 [(bts.api.key)=true]; + uint32 position = 2; +} + +message OrderPosition { + uint64 exchange_id = 1; + float price = 2; + int32 size = 3; +} +message PositionData { + uint32 security_id = 1; + repeated OrderPosition bid = 10; + repeated OrderPosition ask = 20; +} +message MBOSecurityQuery { + uint32 security_id = 1; } service MBOService { rpc piq (bts.api.VoidMessage) returns (stream OrderPIQ); + rpc get_orders_for_security (MBOSecurityQuery) returns (PositionData); + } \ No newline at end of file diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index acc9aa035..a7f8c4bb7 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -293,6 +293,34 @@ MBOServiceServer::MBOServiceServer( MBOServiceServer::~MBOServiceServer() { } +void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request, const api::Response& reply) { + PositionData response; + response.set_security_id(request.security_id()); + auto handle = _me->handle_index.index.lock([id=request.security_id()](const auto& index) { + auto iter = index.find(SecurityKey(id)); + return iter != index.end() ? iter->second : MDConsumerHandle {}; + }); + + if(handle.is_initialized()) { + auto mbo = handle.get_mbo(); + if (mbo) { + for (const auto& o : mbo->bids) { + auto* entry = response.add_bid(); + entry->set_exchange_id(o.id.value()); + entry->set_price(o.price.price()); + entry->set_size(o.size); + } + for (const auto& o : mbo->asks) { + auto* entry = response.add_ask(); + entry->set_exchange_id(o.id.value()); + entry->set_price(o.price.price()); + entry->set_size(o.size); + } + } + } + + reply(response); +} }} //namespace #endif \ No newline at end of file diff --git a/src/lib/md/MBOServiceServer.h b/src/lib/md/MBOServiceServer.h index 9e47aadfa..d0f590a18 100644 --- a/src/lib/md/MBOServiceServer.h +++ b/src/lib/md/MBOServiceServer.h @@ -20,6 +20,8 @@ class MBOServiceServer : public MBOServiceBase { md::MDClient&); ~MBOServiceServer(); + void get_orders_for_security(const MBOSecurityQuery&, const api::Response&) override; + }; }} //namespace From 8716099ac6a94a5ba39eb1a8eeaabf1e90b6df42 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 11:14:58 -0500 Subject: [PATCH 050/100] Removing OrderSecPiq struct since OrderSecurity does everything that did; we still may want to rename it down the line --- src/client/spark/Daemons/OrderTicketDaemon.cs | 6 +++--- .../spark/Notifications/INotificationService.cs | 2 +- .../INotificationServiceDataCache.cs | 2 +- .../spark/Orders/OrderActivitySummarizer.cs | 10 +++++----- src/client/spark/Orders/OrderSecPiq.cs | 17 ----------------- src/client/spark/Orders/OrderSecurity.cs | 5 ++--- src/client/spark/Orders/OrdersModel.cs | 16 ++++++++-------- 7 files changed, 20 insertions(+), 38 deletions(-) delete mode 100644 src/client/spark/Orders/OrderSecPiq.cs diff --git a/src/client/spark/Daemons/OrderTicketDaemon.cs b/src/client/spark/Daemons/OrderTicketDaemon.cs index fa6091f92..568a6b1ac 100644 --- a/src/client/spark/Daemons/OrderTicketDaemon.cs +++ b/src/client/spark/Daemons/OrderTicketDaemon.cs @@ -19,7 +19,7 @@ namespace bts.shell { readonly OrdersModel _model; readonly ErrorReporter _error_reporter; IReadOnlyDictionary _md_map; - TableUpdate _orders; + TableUpdate _orders; IDisposable _dispose; [ImportingConstructor] @@ -60,13 +60,13 @@ namespace bts.shell { } void _show_modify_order(uint order_key) { - if (_orders.state.TryGetValue(order_key, out var ori) && ori.sec.HasValue + if (_orders.state.TryGetValue(order_key, out var ori) && ori.security.HasValue && (ori.order.HasDef && ori.order.Def.HasCustomDef && !(ori.order.Def.CustomDef.HasStop || ori.order.Def.CustomDef.HasStopLimit))) { var md = _md_map.TryGetValue(ori.order.security_key()); _show_order_window(new OrderModificationDialog { DataContext = new ExistingOrderTicketVM(_model, Observable.Return((md,ori.order)), - Observable.Return(_model.factory.default_ladder_settings(ori.sec.Value).calculator_buttons) + Observable.Return(_model.factory.default_ladder_settings(ori.security.Value).calculator_buttons) ) }); } diff --git a/src/client/spark/Notifications/INotificationService.cs b/src/client/spark/Notifications/INotificationService.cs index 27086e52c..820e2ed72 100644 --- a/src/client/spark/Notifications/INotificationService.cs +++ b/src/client/spark/Notifications/INotificationService.cs @@ -55,7 +55,7 @@ namespace bts.shell { void _on_activity(Activity activity) => _activity_notifications.process_item(activity); - void _on_orders(TableUpdate update) { + void _on_orders(TableUpdate update) { // must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history _data_cache.update(update.as_iincremental()); foreach (var o in update.added.Concat(update.changed)) diff --git a/src/client/spark/Notifications/INotificationServiceDataCache.cs b/src/client/spark/Notifications/INotificationServiceDataCache.cs index 00fb48eb0..cf51f6e14 100644 --- a/src/client/spark/Notifications/INotificationServiceDataCache.cs +++ b/src/client/spark/Notifications/INotificationServiceDataCache.cs @@ -50,7 +50,7 @@ namespace bts.shell { public DateTime last_order_reset_time { get; private set; } public bool is_first_order_update { get; private set; } - internal void update(IIncrementalUpdate data) { + internal void update(IIncrementalUpdate data) { DateTime now = DateTime.Now; if (data.removed.Count == _order_cache.Count) { last_order_reset_time = now; diff --git a/src/client/spark/Orders/OrderActivitySummarizer.cs b/src/client/spark/Orders/OrderActivitySummarizer.cs index 42e934664..b085dddc9 100644 --- a/src/client/spark/Orders/OrderActivitySummarizer.cs +++ b/src/client/spark/Orders/OrderActivitySummarizer.cs @@ -27,7 +27,7 @@ namespace bts.spark { static readonly IEqualityComparer _id_comparer = bts.utils.EqualityComparer.Create(a => a.Key); - public OrderActivitySummarizer(IBus bus, IObservable> orders) { + public OrderActivitySummarizer(IBus bus, IObservable> orders) { _bus = bus; _dispose = Disposable.Create( bus.subscribe(SparkTags.SecuritiesTag).Subscribe(sec => _security = sec), @@ -124,12 +124,12 @@ namespace bts.spark { #endregion - void orders_updated(TableUpdate update) { + void orders_updated(TableUpdate update) { foreach (var o in update.added) { if (!string.IsNullOrEmpty(o.order.State.LastMsg) && o.order.State.Created.ToDateTime().Date == DateTime.Today) { - if (o.sec.HasValue) { - log_status(ActivityLogCategory.warn, o.sec.Value, o.order.State.LastMsg); + if (o.security.HasValue) { + log_status(ActivityLogCategory.warn, o.security.Value, o.order.State.LastMsg); } } } @@ -137,7 +137,7 @@ namespace bts.spark { foreach (var order in update.changes) { var neworder = order.New.order; var oldorder = order.Old.order; - var security = order.New.sec ?? order.Old.sec; + var security = order.New.security ?? order.Old.security; // // description var mod_desc = modified_description(oldorder, neworder, security?.ContractSpec); diff --git a/src/client/spark/Orders/OrderSecPiq.cs b/src/client/spark/Orders/OrderSecPiq.cs deleted file mode 100644 index 77f7177d6..000000000 --- a/src/client/spark/Orders/OrderSecPiq.cs +++ /dev/null @@ -1,17 +0,0 @@ -using bts.api; -using bts.api.orderman; - - -namespace mbts.spark { - public struct OrderSecPiq { - public SecurityImmutable? sec; - public OrderRecordImmutable order; - public uint? piq; - - public OrderSecPiq(SecurityImmutable? sec_imm, OrderRecordImmutable ori, uint? placement_in_queue) { - sec = sec_imm; - order = ori; - piq = placement_in_queue; - } - } -} diff --git a/src/client/spark/Orders/OrderSecurity.cs b/src/client/spark/Orders/OrderSecurity.cs index eb46dc351..71889e5a6 100644 --- a/src/client/spark/Orders/OrderSecurity.cs +++ b/src/client/spark/Orders/OrderSecurity.cs @@ -9,12 +9,11 @@ namespace bts.api.orderman { this.order = order; this.piq = piq; } - public OrderSecurity(OrderSecPiq t) { - security = t.sec; + public OrderSecurity(OrderSecurity t) { + security = t.security; order = t.order; piq = t.piq; } - public static implicit operator OrderSecurity(OrderSecPiq x) => new OrderSecurity(x); public SecurityImmutable? security { get; } public OrderRecordImmutable order { get; } public uint? piq { get; } diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 977eb721f..37a7d2d2b 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -19,7 +19,7 @@ using System.Reactive.Concurrency; using System.Reactive.Linq; using System.Reactive.Subjects; using System.Threading.Tasks; -using OrderUpdate = bts.api.core.TableUpdate; +using OrderUpdate = bts.api.core.TableUpdate; namespace mbts.spark { [Export] @@ -63,13 +63,13 @@ namespace mbts.spark { .LeftJoin< (SecurityImmutable?, OrderRecordImmutable), OrderPIQImmutable, - OrderSecPiq, + OrderSecurity, uint >(mbo_service.subscribei_piq().Table(), si_ort => si_ort.Item2.Key, p => p.OrderId, - (si_ort, r, p) => new OrderSecPiq(si_ort.Item1, si_ort.Item2, p.Position), - (si_ort, r) => new OrderSecPiq(si_ort.Item1, si_ort.Item2, null), + (si_ort, r, p) => new OrderSecurity(si_ort.Item1, si_ort.Item2, p.Position), + (si_ort, r) => new OrderSecurity(si_ort.Item1, si_ort.Item2, null), osp => osp.order.Key); on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); @@ -82,8 +82,8 @@ namespace mbts.spark { on_order_securities = on_orders .as_iincremental() - .filter(t => t.sec.HasValue) - .Select(t => t.sec.Value); + .filter(t => t.security.HasValue) + .Select(t => t.security.Value); _activity = new OrderActivitySummarizer(bus, on_orders); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); @@ -100,8 +100,8 @@ namespace mbts.spark { t => t.order.Key, am => am.algo.key, t => new AlgoKey(new bts.persist.algo.AlgoPersistedKey(t.order.Def.AlgoTemplateKey)), - (os, okey, am, mkey) => new OrderSecurityAlgoMeta(os.order, os.sec, am.algo, am.meta), - (os, okey, mkey) => new OrderSecurityAlgoMeta(os.order, os.sec, null, null), + (os, okey, am, mkey) => new OrderSecurityAlgoMeta(os.order, os.security, am.algo, am.meta), + (os, okey, mkey) => new OrderSecurityAlgoMeta(os.order, os.security, null, null), ot => ot.order.Key); on_order_tuples = order_secs_algo_meta_join; var on_fills = From 95500187f0156064e7abe6546262bd1c882dfa1d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 16 Apr 2019 12:43:24 -0400 Subject: [PATCH 051/100] Indicate our orders in diagnostic --- src/lib/api/services/MBOService.proto | 5 +++-- src/lib/md/MBOServiceServer.cpp | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/lib/api/services/MBOService.proto b/src/lib/api/services/MBOService.proto index 8250375cc..7ceabb12c 100644 --- a/src/lib/api/services/MBOService.proto +++ b/src/lib/api/services/MBOService.proto @@ -12,8 +12,9 @@ message OrderPIQ { message OrderPosition { uint64 exchange_id = 1; - float price = 2; - int32 size = 3; + uint32 order_id = 2; + float price = 10; + int32 size = 11; } message PositionData { uint32 security_id = 1; diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index a7f8c4bb7..39c668c95 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -154,7 +154,7 @@ std::optional search_orders(const ExchangeOrderKey oid, const std::vec } else { price = oo.price; - position = 0; + position = 1; } } return std::nullopt; @@ -301,6 +301,11 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request, return iter != index.end() ? iter->second : MDConsumerHandle {}; }); + std::unordered_map reverse; + for (const auto& pair : _me->active) { + reverse.insert(std::make_pair(pair.second.exchange_id(), pair.first)); + } + if(handle.is_initialized()) { auto mbo = handle.get_mbo(); if (mbo) { @@ -309,12 +314,20 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request, entry->set_exchange_id(o.id.value()); entry->set_price(o.price.price()); entry->set_size(o.size); + auto iter = reverse.find(o.id); + if (iter != reverse.end()) { + entry->set_order_id(iter->second.value()); + } } for (const auto& o : mbo->asks) { auto* entry = response.add_ask(); entry->set_exchange_id(o.id.value()); entry->set_price(o.price.price()); entry->set_size(o.size); + auto iter = reverse.find(o.id); + if (iter != reverse.end()) { + entry->set_order_id(iter->second.value()); + } } } } From 05abbc4a503872319b3fb2f57c99de63d196f116 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 12:03:33 -0500 Subject: [PATCH 052/100] don't show uint.Max piqs during server restart --- src/client/spark/view/Spark/Ladder/LadderVM.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index 0bf612489..fadbf56b0 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -233,7 +233,10 @@ namespace mbts.spark { public event Action on_tick; public MDConsumerHandle md => _model.md; - public IEnumerable<(double, uint)> price_vs_piq => _slot_price_vs_orders.Values.Where(result => result.min_piq.HasValue).Select(o => (o.price, o.min_piq.Value)); + // we return only entries who have piqs and whose piqs are not uint.Max + // uint.Max happens when the server is restarted but the client is still up + // maybe long term we don't show this data while the exchange connection is down + public IEnumerable<(double, uint)> price_vs_piq => _slot_price_vs_orders.Values.Where(result => result.min_piq.HasValue && result.min_piq.Value != uint.MaxValue).Select(o => (o.price, o.min_piq.Value)); public bool is_synthetic => _model.security.ContractCase == Security.ContractOneofCase.Synthetic; Property _status; From 4c8d0a4217f035c6782d5cab1c2a10fb3dc5d8ee Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 14:13:06 -0500 Subject: [PATCH 053/100] add null check on r.newval --- src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index 552d080aa..4262c4735 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -64,7 +64,7 @@ namespace mbts.spark { }), _columns, _current_row.SubscribeHookLatest( - r => r.newval.subscribe_data().Subscribe( + r => r.newval?.subscribe_data().Subscribe( data => selected_order_changed?.Invoke(data.order) ) ) From bc35e773cead49f14ae176f0b969c6bc9fd92e40 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 14:19:43 -0500 Subject: [PATCH 054/100] piq_column settings should never return null, which allevites doing the same in sanitize_ladder_settings and another time when dealing with WindowSpecifics --- src/client/spark/view/Spark/Ladder/LadderSettings.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderSettings.cs b/src/client/spark/view/Spark/Ladder/LadderSettings.cs index 329082867..e5a8cf4a7 100644 --- a/src/client/spark/view/Spark/Ladder/LadderSettings.cs +++ b/src/client/spark/view/Spark/Ladder/LadderSettings.cs @@ -401,7 +401,7 @@ namespace mbts.spark { public bool clear_visible { get; set; } // [ProtoMember(43), DefaultValue(_default_order_entry_panel_width)] - public double order_entry_panel_initial_width { get; set; } + public double order_entry_panel_initial_width { get; set; } [ProtoMember(44), DefaultValue(true)] public bool use_defaults { get; set; } @@ -421,7 +421,11 @@ namespace mbts.spark { public NetChangeTerms net_change { get; set; } [ProtoMember(49)] - public LadderColumnSettings piq_column { get; set; } + LadderColumnSettings _piq_column; + public LadderColumnSettings piq_column { + get => _piq_column ?? (_piq_column = LadderSettings.Default.piq_column); + set => _piq_column = value; + } public bool Equals(LadderSettings other) => other != null && From 73c9be3bec68c496ba088554d9b1e23cf3b37088 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 16 Apr 2019 15:45:44 -0400 Subject: [PATCH 055/100] SPARK-2197 bugfix --- src/lib/md/MBOServiceServer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index 39c668c95..e25715ee4 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -147,7 +147,8 @@ std::optional search_orders(const ExchangeOrderKey oid, const std::vec for (size_t x = 0; x < orders.size(); ++x) { const auto& oo = orders[x]; if (oo.id == oid) { - return { position }; + if (oo.price == price) return { position }; + else return 0 ; } else if (price == oo.price) { position += 1; From b059aedbceba6f27c40a0e7376021bf6b25eb0a2 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 17 Apr 2019 08:53:26 -0500 Subject: [PATCH 056/100] adding PlacementInQueue key to toggles so that we have a config option to enable/disable once deployed; defaults to enabled --- src/client/spark/Orders/OrdersModel.cs | 13 +++++++++++-- src/lib/config/protos/toggles.proto | 3 +++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 37a7d2d2b..3835a80d7 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -7,6 +7,7 @@ using bts.md; using bts.spark; using bts.utils; using mbts.collections; +using mbts.config; using mbts.spark.utilities; using mbts.utils; using System; @@ -36,6 +37,7 @@ namespace mbts.spark { readonly TableUpdateDemuxer _security_orders_demuxer; readonly TableUpdateDemuxer _security_positions_demuxer; IDisposable _dispose; + static readonly bool _piq_enabled = Configuration.get().enabled; [ImportingConstructor] OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, MBOService mbo_service, @@ -52,14 +54,16 @@ namespace mbts.spark { this.algo_model = algo_model; var scheduler = ThreadPoolScheduler.Instance; var interval = TimeSpan.FromMilliseconds(100); - on_orders = + var pre_on_orders = _service .subscribei_orders() .Naggle(interval, scheduler) .PermissionFilter(admin_model.on_admin_data) .join(sstore, order => order.security_key()) .Table() - .Cache() + .Cache(); + if (_piq_enabled) { + on_orders = pre_on_orders .LeftJoin< (SecurityImmutable?, OrderRecordImmutable), OrderPIQImmutable, @@ -72,6 +76,11 @@ namespace mbts.spark { (si_ort, r) => new OrderSecurity(si_ort.Item1, si_ort.Item2, null), osp => osp.order.Key); + } else { + on_orders = pre_on_orders.Select((ft) => new OrderSecurity(ft.Item1, ft.Item2, null), os => os.order.Key) + .Publish(); + } + on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); working_orders = on_orders.Scan( diff --git a/src/lib/config/protos/toggles.proto b/src/lib/config/protos/toggles.proto index 6923cfbe9..2925a8721 100644 --- a/src/lib/config/protos/toggles.proto +++ b/src/lib/config/protos/toggles.proto @@ -37,4 +37,7 @@ message VolumeAtPrice{ } message MBOStats { optional bool enabled = 1 [default = true]; +} +message PlacementInQueue { + optional bool enabled = 1 [default = true]; } \ No newline at end of file From 6692797c3f55b71dfce940ce73a97569dae62428 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 17 Apr 2019 13:03:43 -0500 Subject: [PATCH 057/100] changing from .Publish to .Replay b/c apparently Publish doesn't send old values i.e. previously working orders won't show up, but new ones will --- src/client/spark/Orders/OrdersModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 3835a80d7..694260926 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -78,7 +78,7 @@ namespace mbts.spark { } else { on_orders = pre_on_orders.Select((ft) => new OrderSecurity(ft.Item1, ft.Item2, null), os => os.order.Key) - .Publish(); + .Replay(1); } on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); From ff2d150dc463f6c2b3151a169042616c678ef175 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 17 Apr 2019 17:33:46 -0500 Subject: [PATCH 058/100] restricting orders with piqs to just ladders for now --- src/client/spark/Daemons/OrderTicketDaemon.cs | 6 +- .../Notifications/INotificationService.cs | 2 +- .../INotificationServiceDataCache.cs | 2 +- .../spark/Orders/OrderActivitySummarizer.cs | 10 +-- src/client/spark/Orders/OrderSecurity.cs | 21 +++++-- src/client/spark/Orders/OrdersModel.cs | 61 +++++++++++++++---- .../spark/view/Spark/Ladder/LadderVM.cs | 2 +- 7 files changed, 75 insertions(+), 29 deletions(-) diff --git a/src/client/spark/Daemons/OrderTicketDaemon.cs b/src/client/spark/Daemons/OrderTicketDaemon.cs index 568a6b1ac..84e473c46 100644 --- a/src/client/spark/Daemons/OrderTicketDaemon.cs +++ b/src/client/spark/Daemons/OrderTicketDaemon.cs @@ -19,7 +19,7 @@ namespace bts.shell { readonly OrdersModel _model; readonly ErrorReporter _error_reporter; IReadOnlyDictionary _md_map; - TableUpdate _orders; + TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> _orders; IDisposable _dispose; [ImportingConstructor] @@ -60,13 +60,13 @@ namespace bts.shell { } void _show_modify_order(uint order_key) { - if (_orders.state.TryGetValue(order_key, out var ori) && ori.security.HasValue + if (_orders.state.TryGetValue(order_key, out var ori) && ori.sec.HasValue && (ori.order.HasDef && ori.order.Def.HasCustomDef && !(ori.order.Def.CustomDef.HasStop || ori.order.Def.CustomDef.HasStopLimit))) { var md = _md_map.TryGetValue(ori.order.security_key()); _show_order_window(new OrderModificationDialog { DataContext = new ExistingOrderTicketVM(_model, Observable.Return((md,ori.order)), - Observable.Return(_model.factory.default_ladder_settings(ori.security.Value).calculator_buttons) + Observable.Return(_model.factory.default_ladder_settings(ori.sec.Value).calculator_buttons) ) }); } diff --git a/src/client/spark/Notifications/INotificationService.cs b/src/client/spark/Notifications/INotificationService.cs index 820e2ed72..8e294a368 100644 --- a/src/client/spark/Notifications/INotificationService.cs +++ b/src/client/spark/Notifications/INotificationService.cs @@ -55,7 +55,7 @@ namespace bts.shell { void _on_activity(Activity activity) => _activity_notifications.process_item(activity); - void _on_orders(TableUpdate update) { + void _on_orders(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) { // must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history _data_cache.update(update.as_iincremental()); foreach (var o in update.added.Concat(update.changed)) diff --git a/src/client/spark/Notifications/INotificationServiceDataCache.cs b/src/client/spark/Notifications/INotificationServiceDataCache.cs index cf51f6e14..fac9deeed 100644 --- a/src/client/spark/Notifications/INotificationServiceDataCache.cs +++ b/src/client/spark/Notifications/INotificationServiceDataCache.cs @@ -50,7 +50,7 @@ namespace bts.shell { public DateTime last_order_reset_time { get; private set; } public bool is_first_order_update { get; private set; } - internal void update(IIncrementalUpdate data) { + internal void update(IIncrementalUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> data) { DateTime now = DateTime.Now; if (data.removed.Count == _order_cache.Count) { last_order_reset_time = now; diff --git a/src/client/spark/Orders/OrderActivitySummarizer.cs b/src/client/spark/Orders/OrderActivitySummarizer.cs index b085dddc9..b4b8e8b3d 100644 --- a/src/client/spark/Orders/OrderActivitySummarizer.cs +++ b/src/client/spark/Orders/OrderActivitySummarizer.cs @@ -27,7 +27,7 @@ namespace bts.spark { static readonly IEqualityComparer _id_comparer = bts.utils.EqualityComparer.Create(a => a.Key); - public OrderActivitySummarizer(IBus bus, IObservable> orders) { + public OrderActivitySummarizer(IBus bus, IObservable> orders) { _bus = bus; _dispose = Disposable.Create( bus.subscribe(SparkTags.SecuritiesTag).Subscribe(sec => _security = sec), @@ -124,12 +124,12 @@ namespace bts.spark { #endregion - void orders_updated(TableUpdate update) { + void orders_updated(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) { foreach (var o in update.added) { if (!string.IsNullOrEmpty(o.order.State.LastMsg) && o.order.State.Created.ToDateTime().Date == DateTime.Today) { - if (o.security.HasValue) { - log_status(ActivityLogCategory.warn, o.security.Value, o.order.State.LastMsg); + if (o.sec.HasValue) { + log_status(ActivityLogCategory.warn, o.sec.Value, o.order.State.LastMsg); } } } @@ -137,7 +137,7 @@ namespace bts.spark { foreach (var order in update.changes) { var neworder = order.New.order; var oldorder = order.Old.order; - var security = order.New.security ?? order.Old.security; + var security = order.New.sec ?? order.Old.sec; // // description var mod_desc = modified_description(oldorder, neworder, security?.ContractSpec); diff --git a/src/client/spark/Orders/OrderSecurity.cs b/src/client/spark/Orders/OrderSecurity.cs index 71889e5a6..7045f2fe4 100644 --- a/src/client/spark/Orders/OrderSecurity.cs +++ b/src/client/spark/Orders/OrderSecurity.cs @@ -4,16 +4,27 @@ using System; namespace bts.api.orderman { public readonly struct OrderSecurity: IEquatable { + public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order) { + this.security = sec; + this.order = order; + piq = null; + } + public OrderSecurity((SecurityImmutable? sec, OrderRecordImmutable order) t) { + security = t.sec; + order = t.order; + piq = null; + } public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order, uint? piq) { this.security = sec; this.order = order; this.piq = piq; } - public OrderSecurity(OrderSecurity t) { - security = t.security; - order = t.order; - piq = t.piq; - } + //public OrderSecurity(OrderSecurity t) { + // security = t.security; + // order = t.order; + // piq = t.piq; + //} + public static implicit operator OrderSecurity(ValueTuple x) => new OrderSecurity(x); public SecurityImmutable? security { get; } public OrderRecordImmutable order { get; } public uint? piq { get; } diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 694260926..7172a93cf 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -20,7 +20,8 @@ using System.Reactive.Concurrency; using System.Reactive.Linq; using System.Reactive.Subjects; using System.Threading.Tasks; -using OrderUpdate = bts.api.core.TableUpdate; +using OrderUpdate = bts.api.core.TableUpdate<(bts.api.SecurityImmutable? sec, bts.api.orderman.OrderRecordImmutable order), uint>; +using OrderUpdatePiq = bts.api.core.TableUpdate; namespace mbts.spark { [Export] @@ -54,7 +55,7 @@ namespace mbts.spark { this.algo_model = algo_model; var scheduler = ThreadPoolScheduler.Instance; var interval = TimeSpan.FromMilliseconds(100); - var pre_on_orders = + on_orders = _service .subscribei_orders() .Naggle(interval, scheduler) @@ -62,8 +63,9 @@ namespace mbts.spark { .join(sstore, order => order.security_key()) .Table() .Cache(); + if (_piq_enabled) { - on_orders = pre_on_orders + on_orders_with_piq = on_orders .LeftJoin< (SecurityImmutable?, OrderRecordImmutable), OrderPIQImmutable, @@ -75,9 +77,9 @@ namespace mbts.spark { (si_ort, r, p) => new OrderSecurity(si_ort.Item1, si_ort.Item2, p.Position), (si_ort, r) => new OrderSecurity(si_ort.Item1, si_ort.Item2, null), osp => osp.order.Key); - } else { - on_orders = pre_on_orders.Select((ft) => new OrderSecurity(ft.Item1, ft.Item2, null), os => os.order.Key) + on_orders_with_piq = on_orders + .Select((ft) => new OrderSecurity(ft.Item1, ft.Item2, null), os => os.order.Key) .Replay(1); } @@ -88,11 +90,18 @@ namespace mbts.spark { (workers, table) => _update_working_orders(workers, table)) .Replay(1).RefCount(); + working_orders_with_piq = + on_orders_with_piq.Scan( + ImmutableDictionary.Empty, + (workers, table) => _update_working_orders(workers, table)) + .Replay(1).RefCount(); + + on_order_securities = on_orders .as_iincremental() - .filter(t => t.security.HasValue) - .Select(t => t.security.Value); + .filter(t => t.sec.HasValue) + .Select(t => t.sec.Value); _activity = new OrderActivitySummarizer(bus, on_orders); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); @@ -109,8 +118,8 @@ namespace mbts.spark { t => t.order.Key, am => am.algo.key, t => new AlgoKey(new bts.persist.algo.AlgoPersistedKey(t.order.Def.AlgoTemplateKey)), - (os, okey, am, mkey) => new OrderSecurityAlgoMeta(os.order, os.security, am.algo, am.meta), - (os, okey, mkey) => new OrderSecurityAlgoMeta(os.order, os.security, null, null), + (os, okey, am, mkey) => new OrderSecurityAlgoMeta(os.order, os.sec, am.algo, am.meta), + (os, okey, mkey) => new OrderSecurityAlgoMeta(os.order, os.sec, null, null), ot => ot.order.Key); on_order_tuples = order_secs_algo_meta_join; var on_fills = @@ -129,9 +138,8 @@ namespace mbts.spark { f => f.fill.FillKey); on_fill_tuples = fill_order_tuples_join; _dispose = Disposable.Create( - // on_orders changed from an IObservable to an IConnectableObservable, so we call - // RefCount() here to start the connection - on_orders.RefCount().Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), + on_orders.Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), + on_orders_with_piq.RefCount().Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), on_positions.Subscribe(_security_positions_demuxer), order_secs_algo_meta_join.Connect(), algo_meta_join.Connect(), @@ -148,11 +156,13 @@ namespace mbts.spark { public OrderFactory factory => _factory; public SpreaderModel spreader => _spreader; public DynamicOrderHelper dynamic_helper => _dynamic_helper; - public IConnectableObservable on_orders { get; } + public IObservable on_orders { get; } + public IConnectableObservable on_orders_with_piq { get; } public IObservable> on_order_tuples { get; } public IObservable> on_order_securities { get; } public IObservable> on_fill_tuples { get; } public IObservable> working_orders { get; } + public IObservable> working_orders_with_piq { get; } public IObservable> on_positions { get; } public AdminModel admin_model { get; } @@ -249,8 +259,33 @@ namespace mbts.spark { } return clone.ToImmutableDictionary(); } + + static ImmutableDictionary + _update_working_orders(ImmutableDictionary target, OrderUpdatePiq table) { + var clone = target.ToBuilder(); + foreach (var o in table.removed) { + if (o.order.is_active()) { + clone.Remove(o.order.Key); + } + } + foreach (var o in table.changes) { + if (o.Old.order.is_active()) { + clone.Remove(o.Old.order.Key); + } + if (o.New.order.is_active()) { + clone.Add(o.New.order.Key, o.New); + } + } + foreach (var o in table.added) { + if (o.order.is_active()) { + clone.Add(o.order.Key, o); + } + } + return clone.ToImmutableDictionary(); + } } + // usinig new OrdersModel extensions to reduce the amount of code change needed to switch uis over to the new OrdersModel, // without changing any code in the actual new OrdersModel class. static class OrdersModelExtensions { diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index fadbf56b0..b32e25417 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -133,7 +133,7 @@ namespace mbts.spark { account_positions, size_calculator = new CalculatorLogic(0, _settings_manager.subscribe_current().Select(s => s.calculator_buttons)), bus.subscribe(SparkTags.LogActivity).OnDispatcher().Subscribe(status_update), - model.order_gateway.working_orders.OnDispatcher().Subscribe(working => { + model.order_gateway.working_orders_with_piq.OnDispatcher().Subscribe(working => { // Here we manage N orders to 1 price slot in the ladder Dictionary> orders_used_in_this_update = new Dictionary>(_equality); From dcfbb6e62ffa48eb5ffe3a782f4ef529458ad5cb Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Thu, 18 Apr 2019 08:58:56 -0500 Subject: [PATCH 059/100] need a new security order demuxer for the new piq observable --- src/client/spark/Orders/OrdersModel.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 7172a93cf..5be617a10 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -36,6 +36,8 @@ namespace mbts.spark { readonly DynamicOrderHelper _dynamic_helper; readonly OrderActivitySummarizer _activity; readonly TableUpdateDemuxer _security_orders_demuxer; + // we need a new one of these for the time being since the ladder is subscribing to orders with piq + readonly TableUpdateDemuxer _security_orders_piq_demuxer; readonly TableUpdateDemuxer _security_positions_demuxer; IDisposable _dispose; static readonly bool _piq_enabled = Configuration.get().enabled; @@ -105,6 +107,7 @@ namespace mbts.spark { _activity = new OrderActivitySummarizer(bus, on_orders); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); + _security_orders_piq_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); _security_positions_demuxer = new TableUpdateDemuxer(o => o.RecordId, p => p.SecurityId); var algo_meta_join = algo_model.definitions.LeftJoin( algo_model.metadata_service.subscribe_metadata().Table(m => m.id), @@ -139,7 +142,7 @@ namespace mbts.spark { on_fill_tuples = fill_order_tuples_join; _dispose = Disposable.Create( on_orders.Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), - on_orders_with_piq.RefCount().Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), + on_orders_with_piq.RefCount().Select(t => t.order, o => o.Key).Subscribe(_security_orders_piq_demuxer), on_positions.Subscribe(_security_positions_demuxer), order_secs_algo_meta_join.Connect(), algo_meta_join.Connect(), From 36722e27ad3609c9cdc71832d6ba29622084f1e9 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Thu, 18 Apr 2019 11:08:32 -0400 Subject: [PATCH 060/100] Remove SDK docs from the manual --- doc/user/content/bts_spark/Algos/_index.md | 46 +++++++++++++++++++ .../content/bts_spark_sdk/Client/_index.md | 6 --- .../content/bts_spark_sdk/Server/_index.md | 6 --- doc/user/content/bts_spark_sdk/_index.md | 6 --- .../client/spark_sdk/doc}/Overview.md | 0 .../client/spark_sdk/doc}/Threading.md | 0 6 files changed, 46 insertions(+), 18 deletions(-) create mode 100644 doc/user/content/bts_spark/Algos/_index.md delete mode 100644 doc/user/content/bts_spark_sdk/Client/_index.md delete mode 100644 doc/user/content/bts_spark_sdk/Server/_index.md delete mode 100644 doc/user/content/bts_spark_sdk/_index.md rename {doc/user/content/bts_spark_sdk/Client => src/client/spark_sdk/doc}/Overview.md (100%) rename {doc/user/content/bts_spark_sdk/Client => src/client/spark_sdk/doc}/Threading.md (100%) diff --git a/doc/user/content/bts_spark/Algos/_index.md b/doc/user/content/bts_spark/Algos/_index.md new file mode 100644 index 000000000..f3955e979 --- /dev/null +++ b/doc/user/content/bts_spark/Algos/_index.md @@ -0,0 +1,46 @@ ++++ +title = "BTS Basic Algos" +weight = 10 +sort_by = "weight" +insert_anchor_links = "right" ++++ + + +## Lean + +The goal of the lean order is to watch the opposite-inside market size and cross when that size goes below the threshold (this also implies a cross if that size goes to zero and the market trades away from you) If the market trades toward you, you can opt into several different behaviors + +##### Parameters + * Cross Thresh - size thresh in absolute or ratio terms + * Max Chase Ticks - max levels to pay up, in ticks + + +## Join +The goal of the join order is to watch the opposite-inside market and join when the market flips (that is, when the best offer becomes the best bid) and has at least threshold size. + +##### Parameters + * Join Thresh - size thresh in absolute or ratio terms + + +## Top + +The goal of the top order is to not trade by yourself, but the intent at the time of order entry is to get filled.If the order is on the market (say current bid price) AND the quantity on that bid price is below top thresh then cancel the order else remain working. + +##### Parameters + * Top Thresh - size thresh in absolute or ratio terms + + +## Cover +A Cover order’s purpose is to fire a hedge order (of any type) in response to fills from it’s child order on the opposite side of the market N ticks away + +##### Parameters + * Profit Ticks - ticks above or below fill price to place profit / cover order + + +## OCO +An OCO order creates a relationship between a child order and a 'stop' or 'other' order such that when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill. + +Composing this order with Cover simulates scalping functionality + +##### Parameters + * Stop Ticks - ticks below or above main order to place stop / other order \ No newline at end of file diff --git a/doc/user/content/bts_spark_sdk/Client/_index.md b/doc/user/content/bts_spark_sdk/Client/_index.md deleted file mode 100644 index dd7a5550e..000000000 --- a/doc/user/content/bts_spark_sdk/Client/_index.md +++ /dev/null @@ -1,6 +0,0 @@ -+++ -title = "Client" -weight = 1 -sort_by = "weight" -redirect_to = "bts_spark_sdk/Client/overview/" -+++ \ No newline at end of file diff --git a/doc/user/content/bts_spark_sdk/Server/_index.md b/doc/user/content/bts_spark_sdk/Server/_index.md deleted file mode 100644 index c6b0c27ff..000000000 --- a/doc/user/content/bts_spark_sdk/Server/_index.md +++ /dev/null @@ -1,6 +0,0 @@ -+++ -title = "Server" -weight = 2 -sort_by = "weight" -insert_anchor_links = "right" -+++ diff --git a/doc/user/content/bts_spark_sdk/_index.md b/doc/user/content/bts_spark_sdk/_index.md deleted file mode 100644 index 2a698fcda..000000000 --- a/doc/user/content/bts_spark_sdk/_index.md +++ /dev/null @@ -1,6 +0,0 @@ -+++ -title = "BTS Spark SDK" -weight = 2 -sort_by = "weight" -insert_anchor_links = "right" -+++ \ No newline at end of file diff --git a/doc/user/content/bts_spark_sdk/Client/Overview.md b/src/client/spark_sdk/doc/Overview.md similarity index 100% rename from doc/user/content/bts_spark_sdk/Client/Overview.md rename to src/client/spark_sdk/doc/Overview.md diff --git a/doc/user/content/bts_spark_sdk/Client/Threading.md b/src/client/spark_sdk/doc/Threading.md similarity index 100% rename from doc/user/content/bts_spark_sdk/Client/Threading.md rename to src/client/spark_sdk/doc/Threading.md From 02d0f6191ce573fa2bdd2f2a9a715979ca43672e Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 18 Apr 2019 14:06:44 -0400 Subject: [PATCH 061/100] -added a manifest to our csproj that we'd meant to add anyway for win7 subprocess creation -added a fallback for stunnel if we can't create a windows job object --- src/client/spark/App.xaml.cs | 53 +++++++++++++++++++++++------------ src/client/spark/Spark.csproj | 3 +- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/src/client/spark/App.xaml.cs b/src/client/spark/App.xaml.cs index 25aecff08..4fd7024dc 100644 --- a/src/client/spark/App.xaml.cs +++ b/src/client/spark/App.xaml.cs @@ -75,6 +75,37 @@ namespace mbts.ui.shell Environment.Exit(0); } + void start_stunnel(string path, uint job_error) + { + string config_error; + if (!mbts.config.Configuration.try_initialize(out config_error)) + { + _stunnel_process = Process.Start(path + "bin\\stunnel.exe"); + if (_stunnel_process != null) + { + for (int i = 0; i < 10; i++) + { + if (mbts.config.Configuration.try_initialize(out config_error)) + break; + System.Threading.Thread.Sleep(500); + } + } + else + stunnel_job_error = "Stunnel Job Error code: " + job_error + + " could not start stunnel " + (config_error ?? string.Empty); + } + } + + void kill_stunnel() + { + try + { + Process[] ps = Process.GetProcessesByName("stunnel"); + ps.ForEach(p => p.Kill()); + } + catch { } + } + Process _stunnel_process; static internal string stunnel_job_error { get; private set;} void init_stunnel(string[] argv) @@ -85,24 +116,10 @@ namespace mbts.ui.shell if (File.Exists(path + "config\\stunnel.conf")) { uint job_error = BTSUINativeMethods.setup_process_job_object(); - if(job_error == 0) - { - _stunnel_process = Process.Start(path + "bin\\stunnel.exe"); - string config_error; bool success = false; - if(_stunnel_process != null) - { - for(int i = 0; i < 10; i++) - { - if(success = mbts.config.Configuration.try_initialize(out config_error)) - break; - System.Threading.Thread.Sleep(500); - } - } - else - stunnel_job_error = "Stunnel Job Error code: " + job_error + " could not start stunnel"; - } - else - stunnel_job_error = "Stunnel Job Error code: " + job_error; + //if we can't put this process and stunnel into a job, try to stop stunnel then restart + if(job_error != 0) + kill_stunnel(); + start_stunnel(path, job_error); } } catch(Exception ex) diff --git a/src/client/spark/Spark.csproj b/src/client/spark/Spark.csproj index c6eb7e982..dbf62a939 100644 --- a/src/client/spark/Spark.csproj +++ b/src/client/spark/Spark.csproj @@ -4,7 +4,8 @@ Blue Trading Systems Blue Trading Systems Spark - The game has changed. Make your Move. + Advanced Derivatives Trading + spark.manifest true Debug;Release x64 From 546553406ba208b839c6ee855237cc7942a92e67 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 18 Apr 2019 14:13:09 -0400 Subject: [PATCH 062/100] PR comments --- src/client/spark/App.xaml.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/client/spark/App.xaml.cs b/src/client/spark/App.xaml.cs index 4fd7024dc..36810c745 100644 --- a/src/client/spark/App.xaml.cs +++ b/src/client/spark/App.xaml.cs @@ -101,7 +101,14 @@ namespace mbts.ui.shell try { Process[] ps = Process.GetProcessesByName("stunnel"); - ps.ForEach(p => p.Kill()); + ps.ForEach(p => + { + try + { + p.Kill(); + } + catch {} + }); } catch { } } From 7838bdf04f7119c0d65f26a68dcbd5ccbeb43ca0 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 18 Apr 2019 14:16:08 -0400 Subject: [PATCH 063/100] too much whitespace for even me --- src/client/spark/App.xaml.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/client/spark/App.xaml.cs b/src/client/spark/App.xaml.cs index 36810c745..ea2b1de78 100644 --- a/src/client/spark/App.xaml.cs +++ b/src/client/spark/App.xaml.cs @@ -103,10 +103,7 @@ namespace mbts.ui.shell Process[] ps = Process.GetProcessesByName("stunnel"); ps.ForEach(p => { - try - { - p.Kill(); - } + try { p.Kill(); } catch {} }); } From 5392d05ec8fbb840d33aa0071c244ba505b82a67 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 17 Apr 2019 20:22:12 -0400 Subject: [PATCH 064/100] MBO perf work --- .vscode/launch.json | 2 +- src/lib/md/IntPrice.h | 6 + src/lib/md/MBOProcessor.cpp | 91 ++++++++++-- src/lib/md/MBOProcessor.h | 78 ++++++++++- src/lib/md/test/mbo_tests.cpp | 132 +++++++++++++++++- src/server/sparkd/cmds/Run.cpp | 2 +- .../md/platforms/mdp3_onixs/MDP3Listener.h | 42 ++++-- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 4 + 8 files changed, 328 insertions(+), 29 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index fd68a6626..f3c362de0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -215,7 +215,7 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/server/sparkd", - "args": ["test", "[md]"], + "args": ["test", "[mbo]"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/server/", "environment": [], diff --git a/src/lib/md/IntPrice.h b/src/lib/md/IntPrice.h index 1f563548c..dfca4e73f 100644 --- a/src/lib/md/IntPrice.h +++ b/src/lib/md/IntPrice.h @@ -60,6 +60,12 @@ class IntPrice { return delta.mantissa() < 0; } } + bool operator <=(const IntPrice& other) { + return *this == other || *this < other; + } + bool operator >=(const IntPrice& other) { + return !(*this < other); + } IntPrice operator - (const IntPrice& o) const { return op(o, [](int64_t m1, int64_t m2) { return m1 - m2; }); } diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 4fec20880..1079df5bc 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -1,4 +1,4 @@ -#include "MBOProcessor.h" +#include "MBOProcessor.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/ConcurrentPool.h" #include "lib/config/config.h" @@ -12,8 +12,8 @@ #include -namespace bts { -namespace md { +namespace bts { +namespace md { struct SecurityKeyHashCompare { static size_t hash(const SecurityKey& k) { @@ -51,8 +51,8 @@ void print_pool_stats(const utils::ConcurrentPool& pool) { } bid_percentiles << bid_sz[count-1] << " "; ask_percentiles << ask_sz[count-1] << " "; - LOG(md,Info, - "MBO statistics: count: {} - %[50,75,90,95,99,100] bid: [{}] - ask: [{}]", + LOG(md,Info, + "MBO statistics: count: {} - %[50,75,90,95,99,100] bid: [{}] - ask: [{}]", count, bid_percentiles.str(), ask_percentiles.str()); } @@ -60,11 +60,13 @@ typedef tbb::concurrent_hash_map pool; MapType snapshots; + utils::Dispatcher& dispatcher; utils::DispatcherTimer timer; - MBOProcessorImpl() : - pool(2048,512), - snapshots(), - timer(utils::Dispatcher::background(), 600*1000, [&]() { print_pool_stats(pool); }) + MBOProcessorImpl(utils::Dispatcher& d) : + pool(2048,512), + snapshots(), + dispatcher(d), + timer(d, 600*1000, [&]() { print_pool_stats(pool); }) { if (config::get().enabled()) { timer.start(); @@ -75,8 +77,8 @@ struct MBOProcessorImpl { // // Processor -MBOProcessor::MBOProcessor() : - _me(std::make_shared()), +MBOProcessor::MBOProcessor(utils::Dispatcher* d) : + _me(std::make_shared(d == nullptr ? utils::Dispatcher::background() : *d)), _producer(_me), _consumer(_me) {} @@ -86,9 +88,10 @@ MBOReader& MBOProcessor::reader() { return _consumer; } // // Writer + MBOWriter::MBOWriter(std::shared_ptr& me): _me(me) { } MBOWriter::~MBOWriter() { } -MBOSnapshot* MBOWriter::allocate_snapshot() { +MBOSnapshot* MBOWriter::allocate_snapshot(size_t, size_t) { return _me->pool.allocate(); } void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot* snapshot) { @@ -102,6 +105,68 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot* snapshot) { } } +bool MBOWriter::validate_snapshot(SecurityKey key, MBOSnapshot* control) { + #define REQUIRE(x) if (!(x)) { LOG(md,Error, "{}", std::string(#x)); return false; } + // retrieve current snapshot and check they are equal + MBOReader reader(_me); + auto snap = reader.acquire_snapshot(key); + REQUIRE (snap->bids.size() == control->bids.size()) + REQUIRE (snap->asks.size() == control->asks.size()) + for (size_t x = 0; x < snap->bids.size(); ++x) { + REQUIRE (snap->bids[x].id == control->bids[x].id); + REQUIRE (snap->bids[x].price == control->bids[x].price); + REQUIRE (snap->bids[x].size == control->bids[x].size); + } + for (size_t x = 0; x < snap->asks.size(); ++x) { + REQUIRE (snap->asks[x].id == control->asks[x].id); + REQUIRE (snap->asks[x].price == control->asks[x].price); + REQUIRE (snap->asks[x].size == control->asks[x].size); + } + #undef REQUIRE + return true; +} +void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { + _me->dispatcher.post([this,&state,update]() { + this->apply_incremental(state,update); + }); +} +void MBOWriter::apply_incremental(MBOWorkingState& state, const inc::Update& update) { + // fast part + apply_incremental(state, update); + // determine if we need to publish + // slow part + auto snap = allocate_snapshot(state.bid.size(), state.ask.size()); + copy_to_snapshot(state, snap); + publish_snapshot(state.security, snap); +} + +void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { + if (dest == nullptr) return; + // copy working state to snapshot + dest->updated = state.updated; + auto type_map = [](auto pair) { + return SnapshotOrder { pair.second.id, std::get(pair.first), pair.second.size }; + }; + std::transform(state.bid.rbegin(), state.bid.rend(), std::back_inserter(dest->bids), type_map); + std::transform(state.ask.begin(), state.ask.end(), std::back_inserter(dest->asks), type_map); +} + +void apply_incremental(MBOWorkingState& state, const inc::Update& update) { + // apply the update to the working state + MBOWorkingState::Side& collection = (update.type == inc::UpdateType::Bid) ? state.bid : state.ask; + auto key = std::make_tuple(update.price, update.priority); + switch(update.action) { + case inc::UpdateAction::New: + case inc::UpdateAction::Change: + collection[key] = { update.order_id, update.size }; + break; + case inc::UpdateAction::Delete: + collection.erase(key); + break; + } +} + + // // Reader @@ -123,4 +188,4 @@ void MBOReader::release_snapshot(MBOSnapshot**s) const { _me->pool.release(s); } -}} +}} diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 17246fd21..04c3747d7 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -2,16 +2,20 @@ #include "lib/domain_model/SecurityKey.h" #include "lib/domain_model/Security.h" #include "lib/utils/TimeStamp.h" -#include "lib/utils/ValueWrapper.h" +#include "lib/utils/Dispatcher.h" #include "IntPrice.h" #include "ExchangeOrderKey.h" +#include "google/btree/btree_map.h" + #include #include +#include +#include namespace bts { namespace md { - + struct SnapshotOrder { ExchangeOrderKey id; IntPrice price; @@ -39,6 +43,59 @@ struct MBOSnapshot { } }; +namespace inc { + + // according to this link MBO book only has these updates types + // https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP+3.0+-+Market+Data+Snapshot+Full+Refresh#MDP3.0-MarketDataSnapshotFullRefresh-MarketbyOrder(MBO)DataBlock + enum class UpdateType { + Bid, // 0 + Ask, // 1 + // ImpliedBid, // 'E' + // ImpliedAsk, // 'F' + // BookReset // 'J' + }; + + // OrderUpdateAction=37708 + enum class UpdateAction { + New, //0 + Change, //1 + Delete //2 + }; + + typedef int64_t Priority; + struct Update { + ExchangeOrderKey order_id; + Priority priority; + IntPrice price; + int32_t size; + UpdateType type; + UpdateAction action; + }; +} + +struct MBOWorkingState { + typedef std::tuple key_t; + struct Data { + ExchangeOrderKey id; + int32_t size; + }; + //todo: b-tree this + typedef std::map Side; + + SecurityKey security; + utils::TimeStamp updated; + Side bid,ask; +}; +void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* destination); + +void apply_incremental(MBOWorkingState&, const inc::Update&); +template +void apply_incremental(MBOWorkingState& state, const V& vec) { + for (const auto& update : vec) { + apply_incremental(state, update); + } +} + struct MBOProcessorImpl; class MBOProcessor; @@ -47,16 +104,27 @@ class MBOWriter { friend class MBOProcessor; MBOWriter(std::shared_ptr&); ~MBOWriter(); + public: + // + // Snapshot api + MBOSnapshot* allocate_snapshot(size_t bid_count=0, size_t ask_count=0); + void publish_snapshot(SecurityKey, MBOSnapshot*); + bool validate_snapshot(SecurityKey, MBOSnapshot*); - MBOSnapshot* allocate_snapshot(); + // + // Incremental api - void publish_snapshot(SecurityKey key, MBOSnapshot* snapshot); + // Enqueue the update to the mbo work queue. + void enqueue_incremental(MBOWorkingState&, const inc::Update&); + // Update working state and publish directly. + void apply_incremental(MBOWorkingState&, const inc::Update&); }; class MBOReader { std::shared_ptr _me; friend class MBOProcessor; + friend class MBOWriter; MBOReader(std::shared_ptr&); ~MBOReader(); public: @@ -70,7 +138,7 @@ class MBOProcessor { MBOReader _consumer; public: - MBOProcessor(); + MBOProcessor(utils::Dispatcher* _=nullptr); MBOWriter& writer(); MBOReader& reader(); }; diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp index 70c4492e9..95517feb5 100644 --- a/src/lib/md/test/mbo_tests.cpp +++ b/src/lib/md/test/mbo_tests.cpp @@ -1,12 +1,17 @@ #ifndef _WIN32 #include "catch/catch.hpp" +#include "../IntPrice.h" #include "../MBOProcessor.h" + +#include +#include + namespace bts { namespace md { namespace test { int mbo_tests = 0; -TEST_CASE("mbo basic", "[md][mbo]") { +TEST_CASE("MBO publish snapshot", "[md][mbo]") { MBOProcessor p; // publish a snapshot auto sec = SecurityKey(99); @@ -23,5 +28,130 @@ TEST_CASE("mbo basic", "[md][mbo]") { REQUIRE(s->asks.size() == 0); } +std::vector gen_test_orders(uint64_t id=100, inc::UpdateType type=inc::UpdateType::Bid) { + std::vector result; + inc::Priority priority = 10000; + for (int p = 100; p < 501; p+=100) { + // 6 sizes + for (int32_t s = 1; s <= 5; ++s) { + IntPrice price { p + 5 , -2}; + auto order_id = ExchangeOrderKey(++id); + auto size = s; + priority++; + result.push_back( inc::Update { + order_id, + priority, + price, + size, + type, + inc::UpdateAction::New + }); + // std::cout << std::endl << order_id.value() << ": " << price.price() << "-" << size; + } + } + std::random_shuffle(result.begin(), result.end()); + return result; +} + +TEST_CASE("MBO state to snapshot", "[md][mbo]") { + // convert state to snapshot + MBOWorkingState state; + + state.security = SecurityKey(42); + state.updated = utils::TimeStamp::now(); + + auto bids = gen_test_orders(1000, inc::UpdateType::Bid); + apply_incremental(state, bids); + auto asks = gen_test_orders(2000, inc::UpdateType::Ask); + apply_incremental(state, asks); + + MBOSnapshot result; + copy_to_snapshot(state, &result); + + REQUIRE(result.updated == state.updated); + REQUIRE(result.bids.size() == state.bid.size()); + REQUIRE(result.asks.size() == state.ask.size()); + REQUIRE(bids.size() == result.bids.size()); + REQUIRE(asks.size() == result.asks.size()); + for (size_t x = 0; x < result.bids.size() - 1; ++x) { + // std::cout << std::endl << result.bids[x].id.value() << ": " << result.bids[x].price.price() << "-" << result.bids[x].size; + REQUIRE(result.bids[x].price >= result.bids[x+1].price); + } + for (size_t x = 0; x < result.asks.size() - 1; ++x) { + // std::cout << std::endl << result.asks[x].id.value() << ": " << result.asks[x].price.price() << "-" << result.asks[x].size; + REQUIRE(result.asks[x].price <= result.asks[x+1].price); + } + +} + +template +std::vector concat(const std::vector& a, const std::vector& b) { + std::vector res; + std::copy(a.begin(), a.end(), std::back_inserter(res)); + std::copy(b.begin(), b.end(), std::back_inserter(res)); + return res; +} + +TEST_CASE("MBO incremental", "[md][mbo]") { + // basic incremental test + MBOWorkingState state; + + auto orders = + concat( gen_test_orders(128, inc::UpdateType::Bid), + gen_test_orders(256, inc::UpdateType::Ask)); + + std::random_shuffle(orders.begin(), orders.end()); + apply_incremental(state, orders); + + REQUIRE(state.bid.size() + state.ask.size() == orders.size()); + + // generate some updates/deletes + std::vector updates; + // - change all size=2 to size=6 + // - delete all size=3 + size_t removed = 0; + for (const auto& e : orders) { + if (e.size == 2) { + updates.push_back( inc::Update { + e.order_id, + e.priority, + e.price, + 6, + e.type, + inc::UpdateAction::Change + }); + } + else if (e.size == 3) { + ++removed; + updates.push_back( inc::Update { + e.order_id, + e.priority, + e.price, + e.size, + e.type, + inc::UpdateAction::Delete + }); + } + } + apply_incremental(state,updates); + + REQUIRE(state.bid.size() + state.ask.size() == (orders.size() - removed)); + + MBOSnapshot result; + copy_to_snapshot(state, &result); + + auto all = concat (result.bids, result.asks); + std::map counts; + for (const auto& order : all) { + REQUIRE(order.size != 3); + REQUIRE(order.size != 2); + counts[order.size] ++; + } + REQUIRE(counts[6] > 0); + REQUIRE(counts[6] == counts[1]); + REQUIRE(counts[5] == counts[4]); + REQUIRE(counts[6] == counts[5]); +} + }}} #endif \ No newline at end of file diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 5f255fc56..d225abfde 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -60,7 +60,7 @@ int Run::execute(std::vector const &) { persistence::PersistenceServiceServer settings_saver(sc, "settings"); user::AuthServiceServer login_server(sc); - md::MBOProcessor mbo; + md::MBOProcessor mbo(&utils::Dispatcher::get_named("MBO")); md::run_market_datapublishers(&mbo.writer()); api::synthetic::SyntheticSecurityService synthetic_service(sc); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index c18a823d7..cd757f5c6 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -62,16 +62,42 @@ public: // int qty; // a.displayQty(qty); // uint8_t ref; - // a.referenceId(ref); - // const auto& ref_entry = args.message().entries()[(size_t)ref]; - // std::cout - // << a.orderId() - // << qty - // << ref_entry.securityId() - // ; + // if (a.referenceId(ref)) { + // const auto& ref_entry = args.message().entries()[(size_t)ref]; + // LOG(md,Info, "INCREMENTAL32: {} {} {}", + // a.orderId(), + // qty, + // ref_entry.securityId()); + // } // } // }; - // void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args&) override {}; + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args& args) override { + for(const auto& a : args.message().orderIdEntries()) { + int qty; + a.displayQty(qty); + uint8_t ref; + if (a.referenceId(ref)) { + const auto& ref_entry = args.message().entries()[(size_t)ref]; + LOG(md,Info, "INCREMENTAL46: {} {} {}", + a.orderId(), + qty, + ref_entry.securityId()); + } + } + }; + + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook44Args&) override { + if (_mbo) { + LOG(md,Info, "GOT SNAPSHOT44"); + } + } + + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook53Args&) override { + if (_mbo) { + LOG(md,Info, "GOT SNAPSHOT53"); + } + } + // diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index ac8da7ff3..025de1078 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -129,7 +129,11 @@ onixs::SessionSettings session_settings_for_channel(const string& channel_id) { } if (!options.join_snapshot_on_gap()) { + #ifdef DEBUG + settings.gapAndErrorHandling(onixs::GapAndErrorHandlingOptions::RecoverMarketState); + #else settings.gapAndErrorHandling(onixs::GapAndErrorHandlingOptions::ContinueProcessing); + #endif } if (options.use_natural_refresh()) { settings.joinRecovery(onixs::JoinRecoveryOptions::Disabled); From 3cf701a211f5a61e9742195dbd7c82a39eaf617a Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 18 Apr 2019 14:42:26 -0400 Subject: [PATCH 065/100] swap map for btree --- externals | 2 +- src/lib/md/MBOProcessor.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/externals b/externals index d0e8645b5..67a79263b 160000 --- a/externals +++ b/externals @@ -1 +1 @@ -Subproject commit d0e8645b5ef7380314eddc68114a9f0eaa379318 +Subproject commit 67a79263ba116a7646beb8368cef015530bea027 diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 04c3747d7..d1b94fd11 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -79,8 +79,8 @@ struct MBOWorkingState { ExchangeOrderKey id; int32_t size; }; - //todo: b-tree this - typedef std::map Side; + // typedef std::map Side; + typedef btree::btree_map Side; SecurityKey security; utils::TimeStamp updated; From d2a9d2170071ab7fcab7c784848dde129ac79044 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 22 Apr 2019 15:26:50 -0400 Subject: [PATCH 066/100] Handle incremental MBO messages from cme --- src/lib/md/MBOProcessor.cpp | 63 ++++++- src/lib/md/MBOProcessor.h | 24 ++- src/lib/md/test/mbo_tests.cpp | 12 +- .../platforms/mdp3_onixs/CMESecurityIndex.h | 4 +- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 3 +- .../md/platforms/mdp3_onixs/MDP3Listener.h | 172 +++++++++++++++--- 6 files changed, 228 insertions(+), 50 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 1079df5bc..b7909759f 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -94,14 +94,17 @@ MBOWriter::~MBOWriter() { } MBOSnapshot* MBOWriter::allocate_snapshot(size_t, size_t) { return _me->pool.allocate(); } -void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot* snapshot) { - bts_assert(snapshot != nullptr); +void MBOWriter::release_snapshot(MBOSnapshot** s) { + _me->pool.release(s); +} +void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { + bts_assert(*snapshot != nullptr); MapType::accessor accessor; _me->snapshots.insert(accessor, key); - std::swap(accessor->second, snapshot); - if (snapshot != nullptr) { + std::swap(accessor->second, *snapshot); + if (*snapshot != nullptr) { // Note: relase() must happen inside accessor lifetime to ensure correctness - _me->pool.release(&snapshot); + _me->pool.release(snapshot); } } @@ -125,19 +128,27 @@ bool MBOWriter::validate_snapshot(SecurityKey key, MBOSnapshot* control) { #undef REQUIRE return true; } + void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { - _me->dispatcher.post([this,&state,update]() { + std::stringstream ss; + ss << update; + LOG(md,Info, "Update: {}", ss.str()); + + // _me->dispatcher.post([this,&state,update]() { this->apply_incremental(state,update); - }); + // }); } void MBOWriter::apply_incremental(MBOWorkingState& state, const inc::Update& update) { // fast part apply_incremental(state, update); // determine if we need to publish // slow part - auto snap = allocate_snapshot(state.bid.size(), state.ask.size()); - copy_to_snapshot(state, snap); - publish_snapshot(state.security, snap); + auto key = SecurityKey(state.handle.key()); + if (key.value() != invalid_security_key) { + auto snap = allocate_snapshot(state.bid.size(), state.ask.size()); + copy_to_snapshot(state, snap); + publish_snapshot(key, &snap); + } } void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { @@ -166,6 +177,38 @@ void apply_incremental(MBOWorkingState& state, const inc::Update& update) { } } +std::ostream& operator<<(std::ostream& os, const inc::Update& update) { + os << "Update: { " + << update.order_id.value() << " " + << update.priority << " " + << update.price.price() << " " + << update.size << " "; + + switch(update.type) { + case inc::UpdateType::Bid: + os << "bid "; + break; + case inc::UpdateType::Ask: + os << "ask "; + break; + } + + switch(update.action) { + case inc::UpdateAction::New: + os << "NEW"; + break; + case inc::UpdateAction::Change: + os << "CH"; + break; + case inc::UpdateAction::Delete: + os << "DEL"; + break; + } + + os << "}"; + return os; +} + // // Reader diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index d1b94fd11..ceb38f1ea 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -5,6 +5,7 @@ #include "lib/utils/Dispatcher.h" #include "IntPrice.h" #include "ExchangeOrderKey.h" +#include "MDProducerHandle.h" #include "google/btree/btree_map.h" @@ -48,8 +49,8 @@ namespace inc { // according to this link MBO book only has these updates types // https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP+3.0+-+Market+Data+Snapshot+Full+Refresh#MDP3.0-MarketDataSnapshotFullRefresh-MarketbyOrder(MBO)DataBlock enum class UpdateType { - Bid, // 0 - Ask, // 1 + Bid = 0, + Ask = 1, // ImpliedBid, // 'E' // ImpliedAsk, // 'F' // BookReset // 'J' @@ -57,12 +58,12 @@ namespace inc { // OrderUpdateAction=37708 enum class UpdateAction { - New, //0 - Change, //1 - Delete //2 + New = 0, + Change = 1, + Delete = 2 }; - typedef int64_t Priority; + typedef uint64_t Priority; struct Update { ExchangeOrderKey order_id; Priority priority; @@ -73,6 +74,8 @@ namespace inc { }; } +std::ostream& operator<<(std::ostream& os, const inc::Update&); + struct MBOWorkingState { typedef std::tuple key_t; struct Data { @@ -82,10 +85,14 @@ struct MBOWorkingState { // typedef std::map Side; typedef btree::btree_map Side; - SecurityKey security; + MBOWorkingState(MDProducerHandle& h) : handle(h), updated(), bid(),ask() {} + + MDProducerHandle handle; utils::TimeStamp updated; Side bid,ask; }; + + void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* destination); void apply_incremental(MBOWorkingState&, const inc::Update&); @@ -109,7 +116,8 @@ class MBOWriter { // // Snapshot api MBOSnapshot* allocate_snapshot(size_t bid_count=0, size_t ask_count=0); - void publish_snapshot(SecurityKey, MBOSnapshot*); + void release_snapshot(MBOSnapshot**); + void publish_snapshot(SecurityKey, MBOSnapshot**); bool validate_snapshot(SecurityKey, MBOSnapshot*); // diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp index 95517feb5..0a8433de6 100644 --- a/src/lib/md/test/mbo_tests.cpp +++ b/src/lib/md/test/mbo_tests.cpp @@ -2,6 +2,7 @@ #include "catch/catch.hpp" #include "../IntPrice.h" #include "../MBOProcessor.h" +#include "Co2TestInstrument.h" #include #include @@ -18,7 +19,7 @@ TEST_CASE("MBO publish snapshot", "[md][mbo]") { { auto s = p.writer().allocate_snapshot(); s->bids.push_back({ ExchangeOrderKey(1), { 100, 0 }, 42 }); - p.writer().publish_snapshot(sec, s); + p.writer().publish_snapshot(sec, &s); } auto s = p.reader().acquire_snapshot(sec); @@ -55,9 +56,10 @@ std::vector gen_test_orders(uint64_t id=100, inc::UpdateType type=i TEST_CASE("MBO state to snapshot", "[md][mbo]") { // convert state to snapshot - MBOWorkingState state; + Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; + auto handle = instr.producer(); + MBOWorkingState state(handle); - state.security = SecurityKey(42); state.updated = utils::TimeStamp::now(); auto bids = gen_test_orders(1000, inc::UpdateType::Bid); @@ -94,7 +96,9 @@ std::vector concat(const std::vector& a, const std::vector& b) { TEST_CASE("MBO incremental", "[md][mbo]") { // basic incremental test - MBOWorkingState state; + Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; + auto handle = instr.producer(); + MBOWorkingState state(handle); auto orders = concat( gen_test_orders(128, inc::UpdateType::Bid), diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h b/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h index e0e9d3b56..d1915f85d 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h @@ -1,15 +1,17 @@ #pragma once #include "lib/md/SecurityIndex.h" +#include "lib/md/MBOProcessor.h" #include namespace bts { namespace md { struct SecurityData { - SecurityData(uint32_t id, MDProducerHandle& h) : id(id), handle(h), electronic_volume(0) {} + SecurityData(uint32_t id, MDProducerHandle& h) : id(id), handle(h), mbo_state(h), electronic_volume(0) { } uint32_t id; // exchange id MDProducerHandle handle; + MBOWorkingState mbo_state; uint32_t electronic_volume; }; diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index 85867b973..5f541305f 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -134,7 +134,8 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con for (const auto& ask : book.offers()) { snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); } - _mbo->publish_snapshot(SecurityKey(key), snapshot); + // _mbo->publish_snapshot(SecurityKey(key), snapshot); + bts_assert_d(_mbo->validate_snapshot(SecurityKey(key), snapshot)); } } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index cd757f5c6..b7f92544e 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -19,6 +19,52 @@ namespace bts { namespace md { +inline inc::UpdateType mbo_update_type(onixs::EntryTypeBook::Enum otype) { + switch(otype) { + case onixs::EntryTypeBook::Bid: + case onixs::EntryTypeBook::ImpliedBid: + return inc::UpdateType::Bid; + break; + case onixs::EntryTypeBook::Offer: + case onixs::EntryTypeBook::ImpliedOffer: + return inc::UpdateType::Ask; + break; + default: + bts_assert_msg(false, "Unexpected update type"); + } +} + +inline inc::UpdateAction mbo_update_action(onixs::OrderUpdateAction::Enum a) { + switch(a) { + case onixs::OrderUpdateAction::New: + return inc::UpdateAction::New; + case onixs::OrderUpdateAction::Update: + return inc::UpdateAction::Change; + case onixs::OrderUpdateAction::Delete: + return inc::UpdateAction::Delete; + default: + bts_assert_msg(false, "Unexpected update action"); + } +} + +inline inc::UpdateAction mbo_update_action(onixs::UpdateAction::Enum a) { + switch(a) { + case onixs::UpdateAction::New: + return inc::UpdateAction::New; + case onixs::UpdateAction::Change: + return inc::UpdateAction::Change; + case onixs::UpdateAction::Delete: + return inc::UpdateAction::Delete; + case onixs::UpdateAction::DeleteThru: + case onixs::UpdateAction::DeleteFrom: + case onixs::UpdateAction::Overlay: + return inc::UpdateAction::Delete; + default: + bts_assert_msg(false, "Unexpected update action"); + } +} + + class MDP3Listener : public onixs::HandlerListener, public onixs::MarketDataListener, @@ -57,44 +103,118 @@ public: // // Handle MBO incremental updates directly + // void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args& args) override { - // for(const auto& a : args.message().orderIdEntries()) { - // int qty; - // a.displayQty(qty); - // uint8_t ref; - // if (a.referenceId(ref)) { - // const auto& ref_entry = args.message().entries()[(size_t)ref]; - // LOG(md,Info, "INCREMENTAL32: {} {} {}", - // a.orderId(), - // qty, - // ref_entry.securityId()); - // } - // } // }; void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args& args) override { - for(const auto& a : args.message().orderIdEntries()) { + if (!_mbo) return; + + for(const auto& entry : args.message().orderIdEntries()) { int qty; - a.displayQty(qty); + entry.displayQty(qty); uint8_t ref; - if (a.referenceId(ref)) { - const auto& ref_entry = args.message().entries()[(size_t)ref]; - LOG(md,Info, "INCREMENTAL46: {} {} {}", - a.orderId(), - qty, - ref_entry.securityId()); + if (entry.referenceId(ref)) { + const auto& ref_entry = args.message().entries()[(int)ref]; + auto data = _index.get_security(ref_entry.securityId()); + if (!data) continue; + + auto order_id = ExchangeOrderKey(entry.orderId()); + inc::Priority priority; + bts_assert(entry.orderPriority(priority)); + + onixs::Decimal oprice; + ref_entry.entryPx(oprice); + auto price = convert_price(oprice, data->handle.security().display_factor); + + int32_t size; + bts_assert(entry.displayQty(size)); + auto type = mbo_update_type(ref_entry.entryType()); + + auto action = mbo_update_action(entry.orderUpdateAction()); + + _mbo->enqueue_incremental( + data->mbo_state, + inc::Update { + order_id, + priority, + price, + size, + type, + action + }); } } }; - void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook44Args&) override { - if (_mbo) { - LOG(md,Info, "GOT SNAPSHOT44"); + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshOrderBook43Args&) override { + LOG(md,Info, "GOT IROB43"); + } + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshOrderBook47Args& args) override { + if (!_mbo) return; + for (const auto& entry : args.message().entries()) { + auto data = _index.get_security(entry.securityId()); + if (!data) continue; + + // order id + uint64_t temp; + bts_assert( + entry.orderId(temp)); + auto order_id = ExchangeOrderKey(temp); + + inc::Priority priority; + bts_assert( + entry.orderPriority(priority)); + + onixs::Decimal onxprice; + bts_assert( + entry.entryPx(onxprice)); + auto price = convert_price(onxprice, data->handle.security().display_factor); + + int32_t size; + bts_assert( + entry.displayQty(size)); + + auto type = mbo_update_type(entry.entryType()); + auto action = mbo_update_action(entry.updateAction()); + + _mbo->enqueue_incremental( + data->mbo_state, + inc:: Update { + order_id, + priority, + price, + size, + type, + action + }); } } - void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook53Args&) override { - if (_mbo) { - LOG(md,Info, "GOT SNAPSHOT53"); + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook44Args&) override { + LOG(md,Info, "GOT SNAPSHOT44"); + } + + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook53Args& args) override { + if (!_mbo) return; + auto data = _index.get_security(args.message().securityId()); + if (!data) return; + for (const auto& entry : args.message().entries()) { + auto order_id = ExchangeOrderKey(entry.orderId()); + inc::Priority priority; + bts_assert(entry.orderPriority(priority)); + auto price = convert_price(entry.entryPx(), data->handle.security().display_factor); + auto size = entry.displayQty(); + inc::UpdateType type = mbo_update_type(entry.entryType()); + _mbo->enqueue_incremental( + data->mbo_state, + inc::Update { + order_id, + priority, + price, + size, + type, + inc::UpdateAction::New + }); } } From 5b71918a1f465d18af13893035224af8f546a5a8 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 22 Apr 2019 16:41:48 -0400 Subject: [PATCH 067/100] Revert "Make sure every sparkcmd command respects -configserver" This reverts commit dab95d7e91a02fd8ddb1cccdcdafddf3ce8a784f. --- src/client/sparkcmd/Program.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/sparkcmd/Program.cs b/src/client/sparkcmd/Program.cs index 790308e27..c4b3c8cd3 100644 --- a/src/client/sparkcmd/Program.cs +++ b/src/client/sparkcmd/Program.cs @@ -9,7 +9,6 @@ using System.Linq; namespace sparkcmd { class Program { static int Main(string[] args) { - mbts.config.Configuration.initialize(); var commands = ConsoleCommandDispatcher.FindCommandsInSameAssemblyAs(typeof(Program)); return ConsoleCommandDispatcher.DispatchCommand(commands, args, Console.Out); } From 80e337c5996ecc9b61cf77c41051ddf2278159a6 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 22 Apr 2019 16:45:36 -0400 Subject: [PATCH 068/100] sparkcmd should respect '-configserver' --- src/client/sparkcmd/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/sparkcmd/Program.cs b/src/client/sparkcmd/Program.cs index c4b3c8cd3..504ce8daf 100644 --- a/src/client/sparkcmd/Program.cs +++ b/src/client/sparkcmd/Program.cs @@ -9,6 +9,7 @@ using System.Linq; namespace sparkcmd { class Program { static int Main(string[] args) { + mbts.utils.BTSNativeMethods.provide_command_line(args, args.Length); var commands = ConsoleCommandDispatcher.FindCommandsInSameAssemblyAs(typeof(Program)); return ConsoleCommandDispatcher.DispatchCommand(commands, args, Console.Out); } From ca6c0f984c680b87848406f39e51f2d003e6028d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 23 Apr 2019 09:11:30 -0400 Subject: [PATCH 069/100] mbo bugfixes --- src/lib/md/MBOProcessor.cpp | 65 +++++++++++-------- src/lib/md/MBOProcessor.h | 12 +++- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 2 +- 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index b7909759f..6c2efd967 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -109,22 +109,28 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { } bool MBOWriter::validate_snapshot(SecurityKey key, MBOSnapshot* control) { - #define REQUIRE(x) if (!(x)) { LOG(md,Error, "{}", std::string(#x)); return false; } + // #define REQUIRE(x) if (!(x)) { LOG(md,Error, "{}", std::string(#x)); return false; } + #define REQUIRE(x) bts_assert(x); + // retrieve current snapshot and check they are equal - MBOReader reader(_me); - auto snap = reader.acquire_snapshot(key); - REQUIRE (snap->bids.size() == control->bids.size()) - REQUIRE (snap->asks.size() == control->asks.size()) - for (size_t x = 0; x < snap->bids.size(); ++x) { - REQUIRE (snap->bids[x].id == control->bids[x].id); - REQUIRE (snap->bids[x].price == control->bids[x].price); - REQUIRE (snap->bids[x].size == control->bids[x].size); - } - for (size_t x = 0; x < snap->asks.size(); ++x) { - REQUIRE (snap->asks[x].id == control->asks[x].id); - REQUIRE (snap->asks[x].price == control->asks[x].price); - REQUIRE (snap->asks[x].size == control->asks[x].size); - } + _me->dispatcher.post([this,key,control]() { + MBOReader reader(_me); + auto snap = reader.acquire_snapshot(key); + if (snap) { + REQUIRE (snap->bids.size() == control->bids.size()) + REQUIRE (snap->asks.size() == control->asks.size()) + for (size_t x = 0; x < snap->bids.size(); ++x) { + REQUIRE (snap->bids[x].id == control->bids[x].id); + REQUIRE (snap->bids[x].price == control->bids[x].price); + REQUIRE (snap->bids[x].size == control->bids[x].size); + } + for (size_t x = 0; x < snap->asks.size(); ++x) { + REQUIRE (snap->asks[x].id == control->asks[x].id); + REQUIRE (snap->asks[x].price == control->asks[x].price); + REQUIRE (snap->asks[x].size == control->asks[x].size); + } + } + }); #undef REQUIRE return true; } @@ -132,17 +138,17 @@ bool MBOWriter::validate_snapshot(SecurityKey key, MBOSnapshot* control) { void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { std::stringstream ss; ss << update; - LOG(md,Info, "Update: {}", ss.str()); + LOG(md,Info, "security:{} : {}", state.handle.key(), ss.str()); - // _me->dispatcher.post([this,&state,update]() { - this->apply_incremental(state,update); - // }); + _me->dispatcher.post([this,&state,update]() { + // fast part + apply_incremental(state,update); + // slow part + publish_incremental(state); + }); } -void MBOWriter::apply_incremental(MBOWorkingState& state, const inc::Update& update) { - // fast part - apply_incremental(state, update); +void MBOWriter::publish_incremental(MBOWorkingState& state) { // determine if we need to publish - // slow part auto key = SecurityKey(state.handle.key()); if (key.value() != invalid_security_key) { auto snap = allocate_snapshot(state.bid.size(), state.ask.size()); @@ -168,8 +174,11 @@ void apply_incremental(MBOWorkingState& state, const inc::Update& update) { auto key = std::make_tuple(update.price, update.priority); switch(update.action) { case inc::UpdateAction::New: + collection.insert(std::make_pair(key, MBOWorkingState::Data { update.order_id, update.size })); + break; case inc::UpdateAction::Change: collection[key] = { update.order_id, update.size }; + break; break; case inc::UpdateAction::Delete: collection.erase(key); @@ -179,8 +188,8 @@ void apply_incremental(MBOWorkingState& state, const inc::Update& update) { std::ostream& operator<<(std::ostream& os, const inc::Update& update) { os << "Update: { " - << update.order_id.value() << " " - << update.priority << " " + << "id:" << update.order_id.value() << " " + << "pri:" << update.priority << " " << update.price.price() << " " << update.size << " "; @@ -195,13 +204,13 @@ std::ostream& operator<<(std::ostream& os, const inc::Update& update) { switch(update.action) { case inc::UpdateAction::New: - os << "NEW"; + os << "+"; break; case inc::UpdateAction::Change: - os << "CH"; + os << "C"; break; case inc::UpdateAction::Delete: - os << "DEL"; + os << "-"; break; } diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index ceb38f1ea..00c09d888 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -7,7 +7,9 @@ #include "ExchangeOrderKey.h" #include "MDProducerHandle.h" +#ifndef _WIN32 #include "google/btree/btree_map.h" +#endif #include #include @@ -82,8 +84,12 @@ struct MBOWorkingState { ExchangeOrderKey id; int32_t size; }; - // typedef std::map Side; + +#ifdef _WIN32 + typedef std::map Side; +#else typedef btree::btree_map Side; +#endif MBOWorkingState(MDProducerHandle& h) : handle(h), updated(), bid(),ask() {} @@ -94,8 +100,8 @@ struct MBOWorkingState { void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* destination); - void apply_incremental(MBOWorkingState&, const inc::Update&); + template void apply_incremental(MBOWorkingState& state, const V& vec) { for (const auto& update : vec) { @@ -126,7 +132,7 @@ class MBOWriter { // Enqueue the update to the mbo work queue. void enqueue_incremental(MBOWorkingState&, const inc::Update&); // Update working state and publish directly. - void apply_incremental(MBOWorkingState&, const inc::Update&); + void publish_incremental(MBOWorkingState&); }; class MBOReader { diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index 5f541305f..d40b26eb7 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -135,7 +135,7 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); } // _mbo->publish_snapshot(SecurityKey(key), snapshot); - bts_assert_d(_mbo->validate_snapshot(SecurityKey(key), snapshot)); + // bts_assert_d(_mbo->validate_snapshot(SecurityKey(key), snapshot)); } } From 9cc4b5aa87cbb75d1c266ab552f1d2e77dcb3324 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 24 Apr 2019 10:08:53 -0400 Subject: [PATCH 070/100] Update unit/system tests command line parser --- .../sparkcmd/Cmd/ScriptClientCommand.cs | 4 --- src/client/sparkcmd/Cmd/ServiceTail.cs | 1 + src/client/sparkcmd/Cmd/TestCommand.cs | 25 ++++++++----------- src/client/sparkcmd/Program.cs | 7 +++--- src/client/sparkcmd/sparkcmd.csproj | 1 + 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/client/sparkcmd/Cmd/ScriptClientCommand.cs b/src/client/sparkcmd/Cmd/ScriptClientCommand.cs index 5f00cf392..57cf26389 100644 --- a/src/client/sparkcmd/Cmd/ScriptClientCommand.cs +++ b/src/client/sparkcmd/Cmd/ScriptClientCommand.cs @@ -23,10 +23,6 @@ namespace sparkcmd { } public class ScriptClientCommand { - bool _server = false; - string _host; - string _logfile; - public static int Run(ScriptClientOptions o) { if (o.server) { ScriptClient.runserver(); diff --git a/src/client/sparkcmd/Cmd/ServiceTail.cs b/src/client/sparkcmd/Cmd/ServiceTail.cs index 926bc5a6c..d7c47c6d8 100644 --- a/src/client/sparkcmd/Cmd/ServiceTail.cs +++ b/src/client/sparkcmd/Cmd/ServiceTail.cs @@ -8,6 +8,7 @@ using System.Text; using System.Threading.Tasks; namespace bts { + [Verb("tail", HelpText="'console dsqa'")] class TailServiceOptions { [Option('m', "method", Required=true, HelpText = "'[ServiceName].[Method]' to call")] public string method { get;set; } diff --git a/src/client/sparkcmd/Cmd/TestCommand.cs b/src/client/sparkcmd/Cmd/TestCommand.cs index 89dbf7498..b5b919f0a 100644 --- a/src/client/sparkcmd/Cmd/TestCommand.cs +++ b/src/client/sparkcmd/Cmd/TestCommand.cs @@ -8,28 +8,23 @@ using System.Threading.Tasks; using mbts.config; using System.Linq; using CommandLine; +using Newtonsoft.Json; namespace sparkcmd { [Verb("test", HelpText="Run Tests")] public class TestCommandOptions { - [Option('t', "timeout", HelpText="Timeout unit tets", Default=0)] + [Option('t', "timeout", HelpText="Timeout unit tets", Default=0u)] public uint timeout { get; set; } [Option('l', "list", HelpText="List tests")] public bool list { get; set; } [Option('i', "insular", HelpText="Tests run sparkd and use their own config (if specified)")] public bool self_contained { get; set; } - // [Option("unit", HelpText="Run unit tests")] - // public bool unit { get; set; } - // [Option("system", HelpText="Run system tests")] - // public bool system { get; set; } - // [Option("shell", HelpText="Run shell tests")] - // public bool shell { get; set; } + [Option("type", HelpText ="Run all tests of type [Unit|System|Shell]")] + public TestType type { get; set; } - [Value(0, MetaName="TestType")] - public string type { get; set; } [Value(1, MetaName="Tests", HelpText="Test list")] public IEnumerable tests { get; set; } @@ -46,8 +41,10 @@ namespace sparkcmd { } public static int Run(TestCommandOptions o) { + Console.WriteLine(JsonConvert.SerializeObject(o)); + if (o.list) { - list_tests(test_type(o.type)); + list_tests((o.type)); return 0; } @@ -60,20 +57,20 @@ namespace sparkcmd { } } else if (o.tests.Count() > 0) { - return run_tests(Enumerator.match(tests)); + return run_tests(o, Enumerator.match(o.tests)); } else { - return run_tests(Enumerator.get_of_type(test_type)); + return run_tests(o, Enumerator.get_of_type(o.type)); } } static Lazy _enumerator = new Lazy(() => new TestEnumerator("spark_test")); static TestEnumerator Enumerator => _enumerator.Value; - static int run_tests(IEnumerable tests) { + static int run_tests(TestCommandOptions o, IEnumerable tests) { var runner = new TestRunner(); // waits for all child processes to exit or time out - return runner.run_tests(tests, _timeout, self_contained); + return runner.run_tests(tests, o.timeout, o.self_contained); } static void list_tests(TestType test_type) { diff --git a/src/client/sparkcmd/Program.cs b/src/client/sparkcmd/Program.cs index 14b3df9bb..3f73e04c2 100644 --- a/src/client/sparkcmd/Program.cs +++ b/src/client/sparkcmd/Program.cs @@ -24,14 +24,15 @@ namespace sparkcmd { // } static int Main(string[] args) { // note: this can be made to infer the options type list from the list of delegates - return CommandLine.Parser.Default - .ParseArguments(args) - .MapResult( + return Parser.Default + .ParseArguments(args) + .MapResult( Configcheck.Run, ScriptClientCommand.Run, TailService.Run, ServiceTest.Run, SettingsManagerCommand.Run, + TestCommand.Run, errors => 1); } } diff --git a/src/client/sparkcmd/sparkcmd.csproj b/src/client/sparkcmd/sparkcmd.csproj index 1a8c7f607..75fd218c8 100644 --- a/src/client/sparkcmd/sparkcmd.csproj +++ b/src/client/sparkcmd/sparkcmd.csproj @@ -23,6 +23,7 @@ + From 34b7c99c81535d5f83d7d8be43895cb5d34417ae Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 24 Apr 2019 11:05:26 -0400 Subject: [PATCH 071/100] Toggle mbostats off by default --- src/lib/config/protos/toggles.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/config/protos/toggles.proto b/src/lib/config/protos/toggles.proto index 2925a8721..7985bbac7 100644 --- a/src/lib/config/protos/toggles.proto +++ b/src/lib/config/protos/toggles.proto @@ -36,7 +36,7 @@ message VolumeAtPrice{ optional bool enabled = 1 [default = true]; } message MBOStats { - optional bool enabled = 1 [default = true]; + optional bool enabled = 1 [default = false]; } message PlacementInQueue { optional bool enabled = 1 [default = true]; From 34401960f9301c4cc7f9e2571c3876b6dc852133 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 24 Apr 2019 11:08:46 -0400 Subject: [PATCH 072/100] Fix mbo bid sort order --- src/lib/md/MBOProcessor.cpp | 46 +++++++++++++++++++++++++++---- src/lib/md/MBOProcessor.h | 26 +++++++++++++++++- src/lib/md/test/mbo_tests.cpp | 51 +++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 6 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 6c2efd967..f6ef8e2f5 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -164,13 +164,12 @@ void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { auto type_map = [](auto pair) { return SnapshotOrder { pair.second.id, std::get(pair.first), pair.second.size }; }; - std::transform(state.bid.rbegin(), state.bid.rend(), std::back_inserter(dest->bids), type_map); + std::transform(state.bid.begin(), state.bid.end(), std::back_inserter(dest->bids), type_map); std::transform(state.ask.begin(), state.ask.end(), std::back_inserter(dest->asks), type_map); } -void apply_incremental(MBOWorkingState& state, const inc::Update& update) { - // apply the update to the working state - MBOWorkingState::Side& collection = (update.type == inc::UpdateType::Bid) ? state.bid : state.ask; +template +void apply_incremental(C& collection, const inc::Update& update) { auto key = std::make_tuple(update.price, update.priority); switch(update.action) { case inc::UpdateAction::New: @@ -185,8 +184,45 @@ void apply_incremental(MBOWorkingState& state, const inc::Update& update) { break; } } +void apply_incremental(MBOWorkingState& state, const inc::Update& update) { + // apply the update to the working state + if (update.type == inc::UpdateType::Bid) { + apply_incremental(state.bid, update); + } + else { + apply_incremental(state.ask, update); + } +} -std::ostream& operator<<(std::ostream& os, const inc::Update& update) { + +// +// Reader + +MBOReader::MBOReader(std::shared_ptr& me): _me(me) { } +MBOReader::~MBOReader() { } +MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { + // get a result eagerly so we don't have to do it while accessor has the entry locked + auto result = _me->pool.allocate(); + MapType::const_accessor accessor; + if (_me->snapshots.find(accessor, key) && accessor->second != nullptr) { + result->copy_from(*accessor->second); + } + else { + release_snapshot(&result); + } + return result; +} +void MBOReader::release_snapshot(MBOSnapshot**s) const { + _me->pool.release(s); +} + +}} + +namespace std { + using namespace bts; + using namespace bts::md; + +ostream& operator<<(ostream& os, const inc::Update& update) { os << "Update: { " << "id:" << update.order_id.value() << " " << "pri:" << update.priority << " " diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 00c09d888..e6bd0213b 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -78,12 +78,25 @@ namespace inc { std::ostream& operator<<(std::ostream& os, const inc::Update&); + struct MBOWorkingState { typedef std::tuple key_t; struct Data { ExchangeOrderKey id; int32_t size; }; + struct LessBid { + bool operator()(const key_t& left, const key_t& right) const { + const auto& left_price = std::get(left); + const auto& right_price = std::get(right); + if (left_price == right_price) { + return + std::get(left) < + std::get(right); + } + return !(left_price < right_price); + } + }; #ifdef _WIN32 typedef std::map Side; @@ -95,7 +108,14 @@ struct MBOWorkingState { MDProducerHandle handle; utils::TimeStamp updated; - Side bid,ask; + +#ifdef _WIN32 + std::map bid; + std::map ask; +#else + btree::btree_map bid; + btree::btree_map ask; +#endif }; @@ -159,3 +179,7 @@ public: }} + +namespace std { +std::ostream& operator<<(std::ostream& os, const bts::md::inc::Update&); +} diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp index 0a8433de6..d6e11580c 100644 --- a/src/lib/md/test/mbo_tests.cpp +++ b/src/lib/md/test/mbo_tests.cpp @@ -157,5 +157,56 @@ TEST_CASE("MBO incremental", "[md][mbo]") { REQUIRE(counts[6] == counts[5]); } +TEST_CASE("MBO sort order", "[md][mbo]") { + Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; + auto handle = instr.producer(); + MBOWorkingState state(handle); + + state.updated = utils::TimeStamp::now(); + + std::vector updates; + // multiple updates at the same price + for (unsigned x=0; x < 100; ++x) { + // bid + auto p = 1000 + x; + updates.push_back( inc::Update { + ExchangeOrderKey(1000 + x), + 1000 + x, + { 1000, -1 }, + (int)x+1, + inc::UpdateType::Bid, + inc::UpdateAction::New + }); + // ask + if (x < 75) { // ensure bid.size != ask.size + updates.push_back( inc::Update { + ExchangeOrderKey(2000 + x), + 2000 + x, + { 2000, -1 }, + (int)x+1, + inc::UpdateType::Ask, + inc::UpdateAction::New + }); + } + } + std::random_shuffle(updates.begin(), updates.end()); + + apply_incremental(state, updates); + + MBOSnapshot result; + copy_to_snapshot(state, &result); + + // Since all orders are at the same price we can assert they are in priority order + // Since priority is not explicit in the shapshot we set order_id to the same value + + for (size_t x = 0; x < result.bids.size() - 1; ++x) { + REQUIRE(result.bids[x].id.value() < result.bids[x+1].id.value()); + } + for (size_t x = 0; x < result.asks.size() - 1; ++x) { + REQUIRE(result.asks[x].id.value() < result.asks[x+1].id.value()); + } + +} + }}} #endif \ No newline at end of file From f65f34c2fed0f0841540bbd280f3a1e6ec50a83a Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 24 Apr 2019 14:12:50 -0400 Subject: [PATCH 073/100] Limit returned MBOSnapshot size to avoid memory leaking --- src/lib/md/MBOProcessor.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index e6bd0213b..9962208f3 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -41,6 +41,13 @@ struct MBOSnapshot { void reset() { updated = utils::TimeStamp(); + constexpr int max = 256; + if (bids.size() > max) { + bids.resize(max); + } + if (asks.size() > max) { + asks.resize(max); + } bids.clear(); asks.clear(); } From 8a1180835c4611fb96c70f7816d2b185ddc337a3 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 25 Apr 2019 11:06:39 -0400 Subject: [PATCH 074/100] MBO diagnostics/logging --- src/lib/md/MBOProcessor.cpp | 72 +++++++------------ src/lib/md/MBOProcessor.h | 3 +- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 2 +- .../md/platforms/mdp3_onixs/MDP3Listener.h | 23 +++--- 4 files changed, 38 insertions(+), 62 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index f6ef8e2f5..fa1789d5d 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -108,44 +108,42 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { } } -bool MBOWriter::validate_snapshot(SecurityKey key, MBOSnapshot* control) { - // #define REQUIRE(x) if (!(x)) { LOG(md,Error, "{}", std::string(#x)); return false; } - #define REQUIRE(x) bts_assert(x); +bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) { + #define REQUIRE(x,y) \ + if (!(x)) { \ + LOG(md,Error, "Validation error exch:{} sec:{} {} {}", exchid, key.value(), std::string(#x), y); \ + return false; \ + } + // #define REQUIRE(x) bts_assert(x); // retrieve current snapshot and check they are equal - _me->dispatcher.post([this,key,control]() { - MBOReader reader(_me); - auto snap = reader.acquire_snapshot(key); - if (snap) { - REQUIRE (snap->bids.size() == control->bids.size()) - REQUIRE (snap->asks.size() == control->asks.size()) - for (size_t x = 0; x < snap->bids.size(); ++x) { - REQUIRE (snap->bids[x].id == control->bids[x].id); - REQUIRE (snap->bids[x].price == control->bids[x].price); - REQUIRE (snap->bids[x].size == control->bids[x].size); - } - for (size_t x = 0; x < snap->asks.size(); ++x) { - REQUIRE (snap->asks[x].id == control->asks[x].id); - REQUIRE (snap->asks[x].price == control->asks[x].price); - REQUIRE (snap->asks[x].size == control->asks[x].size); - } + MBOReader reader(_me); + auto snap = reader.acquire_snapshot(key); + if (snap) { + REQUIRE (snap->bids.size() == control->bids.size(), (int)snap->bids.size() - (int)control->bids.size()); + REQUIRE (snap->asks.size() == control->asks.size(), (int)snap->asks.size() - (int)control->asks.size()); + for (size_t x = 0; x < snap->bids.size(); ++x) { + REQUIRE (snap->bids[x].id == control->bids[x].id, x); + REQUIRE (snap->bids[x].size == control->bids[x].size, x); } - }); + for (size_t x = 0; x < snap->asks.size(); ++x) { + REQUIRE (snap->asks[x].id == control->asks[x].id, x); + REQUIRE (snap->asks[x].size == control->asks[x].size, x); + } + } #undef REQUIRE return true; } void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { - std::stringstream ss; - ss << update; - LOG(md,Info, "security:{} : {}", state.handle.key(), ss.str()); + LOG(md,Info, "sec:{} : {}", state.handle.key(), update); - _me->dispatcher.post([this,&state,update]() { + // _me->dispatcher.post([this,&state,update]() { // fast part apply_incremental(state,update); // slow part publish_incremental(state); - }); + // }); } void MBOWriter::publish_incremental(MBOWorkingState& state) { // determine if we need to publish @@ -254,26 +252,4 @@ ostream& operator<<(ostream& os, const inc::Update& update) { return os; } - -// -// Reader - -MBOReader::MBOReader(std::shared_ptr& me): _me(me) { } -MBOReader::~MBOReader() { } -MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { - // get a result eagerly so we don't have to do it while accessor has the entry locked - auto result = _me->pool.allocate(); - MapType::const_accessor accessor; - if (_me->snapshots.find(accessor, key) && accessor->second != nullptr) { - result->copy_from(*accessor->second); - } - else { - release_snapshot(&result); - } - return result; -} -void MBOReader::release_snapshot(MBOSnapshot**s) const { - _me->pool.release(s); -} - -}} +} \ No newline at end of file diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 9962208f3..eaf644cbf 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -83,7 +83,6 @@ namespace inc { }; } -std::ostream& operator<<(std::ostream& os, const inc::Update&); struct MBOWorkingState { @@ -151,7 +150,7 @@ class MBOWriter { MBOSnapshot* allocate_snapshot(size_t bid_count=0, size_t ask_count=0); void release_snapshot(MBOSnapshot**); void publish_snapshot(SecurityKey, MBOSnapshot**); - bool validate_snapshot(SecurityKey, MBOSnapshot*); + bool validate_snapshot(uint32_t, SecurityKey, MBOSnapshot*); // // Incremental api diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index d40b26eb7..62a771ee3 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -135,7 +135,7 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); } // _mbo->publish_snapshot(SecurityKey(key), snapshot); - // bts_assert_d(_mbo->validate_snapshot(SecurityKey(key), snapshot)); + _mbo->validate_snapshot(sec.id(), SecurityKey(key), snapshot); } } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index b7f92544e..13b70e1d5 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -104,8 +104,9 @@ public: // // Handle MBO incremental updates directly - // void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args& args) override { - // }; + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args&) override { + LOG(md,Error, "GOT IRB32"); + }; void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args& args) override { if (!_mbo) return; @@ -113,9 +114,13 @@ public: int qty; entry.displayQty(qty); uint8_t ref; + LOG(md,Info, "GOT IRB46:"); if (entry.referenceId(ref)) { - const auto& ref_entry = args.message().entries()[(int)ref]; - auto data = _index.get_security(ref_entry.securityId()); + // entries is indexed by 0 but ref is indexed by 1 :facepalm: + const auto& ref_entry = args.message().entries()[(int)ref-1]; + uint32_t secid = ref_entry.securityId(); + LOG(md,Info, "> GOT IRB46 exch:{} rptSeq:{}", secid, ref_entry.rptSeq()); + auto data = _index.get_security(secid); if (!data) continue; auto order_id = ExchangeOrderKey(entry.orderId()); @@ -395,9 +400,7 @@ void MDP3Listener::onFuture(onixs::Handler&, const onixs::Security& osec, const tracepoint (bts_trace,trace_order,(char*) "onDefinition.future", _latest_sequence, (uint16_t) _latest_sequence, 0); const auto& sec = def.message(); auto exchange_id = sec.securityId(); - // LOG(md, Info, "onDefinition future {} {}", - // std::string(sec.asset().items(), sec.asset().size()), - // std::string(sec.symbol().items(), sec.symbol().size())); + // LOG(md, Info, "onDefinition future exch:{}", exchange_id); if (filter_security(exchange_id, sec) != ADD) return; bts_assert(exchange_id == osec.id()); @@ -425,7 +428,7 @@ void MDP3Listener::onOption(onixs::Handler&, const onixs::Security&, const Instr tracepoint (bts_trace,trace_order,(char*) "onDefinition.option", _latest_sequence, (uint16_t) _latest_sequence, 0); const auto& sec = def.message(); auto exchange_id = sec.securityId(); - // LOG(Info, MarketData) << "onDefinition option " << exchange_id; + // LOG(md, Info, "onDefinition option exch:{}", exchange_id); if (filter_security(exchange_id, sec) != ADD) return; FixedOptionSecurity fsec; @@ -469,9 +472,7 @@ void MDP3Listener::onSpread(onixs::Handler&, const onixs::Security&, const Instr tracepoint (bts_trace,trace_order,(char*) "onDefinition.spread", _latest_sequence, (uint16_t) _latest_sequence, 0); const auto& sec = spread_args.message(); auto exchange_id = sec.securityId(); - // LOG(md, Info, "onDefinition spread {} {}", - // std::string(sec.asset().items(), sec.asset().size()), - // std::string(sec.symbol().items(), sec.symbol().size())); + // LOG(md, Info, "onDefinition spread exch:{}", exchange_id); if (filter_security(exchange_id, sec) != ADD) return; // Filter out any already expired spreads From ad4fd1318923ae8535d69051854e67886856332d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 25 Apr 2019 14:25:49 -0400 Subject: [PATCH 075/100] MBO: Add seconary index to handle mbo change --- src/lib/md/MBOProcessor.cpp | 45 +++++---- src/lib/md/MBOProcessor.h | 9 +- src/lib/md/test/mbo_tests.cpp | 93 ++++++++++++++++++- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 1 - .../md/platforms/mdp3_onixs/MDP3Listener.h | 2 - 5 files changed, 118 insertions(+), 32 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index fa1789d5d..3ec4f31ca 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -109,10 +109,11 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { } bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) { - #define REQUIRE(x,y) \ - if (!(x)) { \ - LOG(md,Error, "Validation error exch:{} sec:{} {} {}", exchid, key.value(), std::string(#x), y); \ - return false; \ + // TO USE THIS VALIDATION YOU MUST MAKE enqueue_incremental syncronous + #define REQUIRE(a,b,y) \ + if ((a) != (b)) { \ + LOG(md,Error, "Validation error exch:{} sec:{} {}!={} other:{}", exchid, key.value(), a, b, y); \ + return false; \ } // #define REQUIRE(x) bts_assert(x); @@ -120,15 +121,15 @@ bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* MBOReader reader(_me); auto snap = reader.acquire_snapshot(key); if (snap) { - REQUIRE (snap->bids.size() == control->bids.size(), (int)snap->bids.size() - (int)control->bids.size()); - REQUIRE (snap->asks.size() == control->asks.size(), (int)snap->asks.size() - (int)control->asks.size()); + REQUIRE (snap->bids.size(), control->bids.size(), '!'); + REQUIRE (snap->asks.size(), control->asks.size(), '!'); for (size_t x = 0; x < snap->bids.size(); ++x) { - REQUIRE (snap->bids[x].id == control->bids[x].id, x); - REQUIRE (snap->bids[x].size == control->bids[x].size, x); + REQUIRE (snap->bids[x].id, control->bids[x].id, x); + REQUIRE (snap->bids[x].size, control->bids[x].size, x); } for (size_t x = 0; x < snap->asks.size(); ++x) { - REQUIRE (snap->asks[x].id == control->asks[x].id, x); - REQUIRE (snap->asks[x].size == control->asks[x].size, x); + REQUIRE (snap->asks[x].id, control->asks[x].id, x); + REQUIRE (snap->asks[x].size, control->asks[x].size, x); } } #undef REQUIRE @@ -167,32 +168,42 @@ void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { } template -void apply_incremental(C& collection, const inc::Update& update) { +void apply_incremental(C& collection, std::unordered_map& index, const inc::Update& update) { auto key = std::make_tuple(update.price, update.priority); switch(update.action) { case inc::UpdateAction::New: collection.insert(std::make_pair(key, MBOWorkingState::Data { update.order_id, update.size })); + index[update.order_id] = key; break; - case inc::UpdateAction::Change: - collection[key] = { update.order_id, update.size }; - break; + case inc::UpdateAction::Change: { + // look up old key + auto r = index.find(update.order_id); + if (r != index.end()) { + auto oldkey = r->second; + collection.erase(oldkey); + index.erase(r); + index[update.order_id] = key; + } + else { bts_assert_d(false); } + collection.insert(std::make_pair(key, MBOWorkingState::Data { update.order_id, update.size })); + } break; case inc::UpdateAction::Delete: collection.erase(key); + index.erase(update.order_id); break; } } void apply_incremental(MBOWorkingState& state, const inc::Update& update) { // apply the update to the working state if (update.type == inc::UpdateType::Bid) { - apply_incremental(state.bid, update); + apply_incremental(state.bid, state.orderid_index, update); } else { - apply_incremental(state.ask, update); + apply_incremental(state.ask, state.orderid_index, update); } } - // // Reader diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index eaf644cbf..f3a92b59b 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -84,7 +84,6 @@ namespace inc { } - struct MBOWorkingState { typedef std::tuple key_t; struct Data { @@ -104,16 +103,12 @@ struct MBOWorkingState { } }; -#ifdef _WIN32 - typedef std::map Side; -#else - typedef btree::btree_map Side; -#endif - MBOWorkingState(MDProducerHandle& h) : handle(h), updated(), bid(),ask() {} MDProducerHandle handle; utils::TimeStamp updated; + // Used for 'change' messages which update price + std::unordered_map orderid_index; #ifdef _WIN32 std::map bid; diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp index d6e11580c..a275a54c0 100644 --- a/src/lib/md/test/mbo_tests.cpp +++ b/src/lib/md/test/mbo_tests.cpp @@ -76,14 +76,11 @@ TEST_CASE("MBO state to snapshot", "[md][mbo]") { REQUIRE(bids.size() == result.bids.size()); REQUIRE(asks.size() == result.asks.size()); for (size_t x = 0; x < result.bids.size() - 1; ++x) { - // std::cout << std::endl << result.bids[x].id.value() << ": " << result.bids[x].price.price() << "-" << result.bids[x].size; REQUIRE(result.bids[x].price >= result.bids[x+1].price); } for (size_t x = 0; x < result.asks.size() - 1; ++x) { - // std::cout << std::endl << result.asks[x].id.value() << ": " << result.asks[x].price.price() << "-" << result.asks[x].size; REQUIRE(result.asks[x].price <= result.asks[x+1].price); } - } template @@ -162,8 +159,6 @@ TEST_CASE("MBO sort order", "[md][mbo]") { auto handle = instr.producer(); MBOWorkingState state(handle); - state.updated = utils::TimeStamp::now(); - std::vector updates; // multiple updates at the same price for (unsigned x=0; x < 100; ++x) { @@ -205,7 +200,95 @@ TEST_CASE("MBO sort order", "[md][mbo]") { for (size_t x = 0; x < result.asks.size() - 1; ++x) { REQUIRE(result.asks[x].id.value() < result.asks[x+1].id.value()); } +} +TEST_CASE("MBO change price and size", "[md][mbo]") { + // When an order's size goes up it gets a new priority + Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; + auto handle = instr.producer(); + MBOWorkingState state(handle); + + constexpr auto bid = inc::UpdateType::Bid; + constexpr auto ask = inc::UpdateType::Ask; + constexpr auto add = inc::UpdateAction::New; + constexpr auto remove = inc::UpdateAction::Delete; + constexpr auto change = inc::UpdateAction::Change; + auto p = [](double price) -> IntPrice { return IntPrice(price * 1000, -3); }; + + // snapshot + std::vector snapshot = { + { ExchangeOrderKey(701267154552), 1160179309, p(2.975), 1, ask, add}, + { ExchangeOrderKey(701267991871), 1161068569, p(2.925), 9, bid, add}, + { ExchangeOrderKey(701267963307), 1161037482, p(2.975), 1, ask, add}, + { ExchangeOrderKey(701267991967), 1161068670, p(2.900), 1, bid, add}, + { ExchangeOrderKey(701267928346), 1160999982, p(3.000), 1, ask, add}, + { ExchangeOrderKey(701267993125), 1161069878, p(2.900), 3, bid, add}, + { ExchangeOrderKey(701267037972), 1160052467, p(2.875), 1, bid, add}, + { ExchangeOrderKey(701267099137), 1160119020, p(2.875), 5, bid, add}, + { ExchangeOrderKey(701267299079), 1160330214, p(2.875), 1, bid, add}, + { ExchangeOrderKey(701267308619), 1160340068, p(2.875), 5, bid, add}, + { ExchangeOrderKey(701267310436), 1160341952, p(2.875), 3, bid, add}, + { ExchangeOrderKey(701267872642), 1160939720, p(2.875), 1, bid, add}, + { ExchangeOrderKey(701267895533), 1160964252, p(2.875), 1, bid, add}, + { ExchangeOrderKey(701267953989), 1161027476, p(2.875), 3, bid, add}, + { ExchangeOrderKey(701267993135), 1161069890, p(2.875), 2, bid, add}, + { ExchangeOrderKey(701266410447), 1159379621, p(2.850), 3, bid, add}, + { ExchangeOrderKey(701267324626), 1160356597, p(2.850), 3, bid, add}, + { ExchangeOrderKey(701267992564), 1161069297, p(2.850), 1, bid, add}, + { ExchangeOrderKey(701266420294), 1159389827, p(2.775), 5, bid, add}, + { ExchangeOrderKey(701267334462), 1160366796, p(2.775), 5, bid, add}, + { ExchangeOrderKey(701266411103), 1159380297, p(2.625), 5, bid, add}, + { ExchangeOrderKey(701267325284), 1160357275, p(2.625), 5, bid, add}, + }; + + apply_incremental(state, snapshot); + REQUIRE(state.bid.size() == 19); + REQUIRE(state.ask.size() == 3); + + std::vector incremental = { + { ExchangeOrderKey(701267993738), 1161070510, p(2.925), 2, bid, add}, + { ExchangeOrderKey(701267993759), 1161070532, p(2.975), 3, ask, add}, + { ExchangeOrderKey(701267993884), 1161070662, p(2.950), 3, ask, add}, + { ExchangeOrderKey(701267994228), 1161071029, p(2.925), 2, bid, add}, + { ExchangeOrderKey(701267994243), 1161071044, p(2.925), 4, bid, add}, + { ExchangeOrderKey(701267994228), 1161071029, p(2.925), 2, bid, remove}, + { ExchangeOrderKey(701267993135), 1161071281, p(2.875), 3, bid, change}, + }; + apply_incremental(state, incremental); + REQUIRE(state.bid.size() == 21); + REQUIRE(state.ask.size() == 5); + + { + auto iter = state.bid.begin(); + auto next = state.bid.begin(); + next++; + for (; next != state.bid.end(); ++iter, ++next) { + auto cur_price = std::get(iter->first); + auto cur_prio = std::get<1>(iter->first); + auto next_price = std::get(next->first); + auto next_prio = std::get<1>(next->first); + REQUIRE(cur_price >= next_price); + if (cur_price == next_price) { + REQUIRE(cur_prio < next_prio); + } + } + } + { + auto iter = state.ask.begin(); + auto next = state.ask.begin(); + next++; + for (; next != state.ask.end(); ++iter, ++next) { + auto cur_price = std::get(iter->first); + auto cur_prio = std::get<1>(iter->first); + auto next_price = std::get(next->first); + auto next_prio = std::get<1>(next->first); + REQUIRE(cur_price <= next_price); + if (cur_price == next_price) { + REQUIRE(cur_prio < next_prio); + } + } + } + } }}} diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index 62a771ee3..b06b185f7 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -134,7 +134,6 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con for (const auto& ask : book.offers()) { snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); } - // _mbo->publish_snapshot(SecurityKey(key), snapshot); _mbo->validate_snapshot(sec.id(), SecurityKey(key), snapshot); } } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index 13b70e1d5..d368c8661 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -114,12 +114,10 @@ public: int qty; entry.displayQty(qty); uint8_t ref; - LOG(md,Info, "GOT IRB46:"); if (entry.referenceId(ref)) { // entries is indexed by 0 but ref is indexed by 1 :facepalm: const auto& ref_entry = args.message().entries()[(int)ref-1]; uint32_t secid = ref_entry.securityId(); - LOG(md,Info, "> GOT IRB46 exch:{} rptSeq:{}", secid, ref_entry.rptSeq()); auto data = _index.get_security(secid); if (!data) continue; From a123ac2ed8d4701c8764f8dab50f1f5b9c9bf6e5 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 25 Apr 2019 15:14:27 -0400 Subject: [PATCH 076/100] Turn on MBO background processing --- src/lib/config/protos/MarketData.proto | 2 +- src/lib/md/MBOProcessor.cpp | 38 ++++++++++--------- src/server/sparkd/cmds/Run.cpp | 3 +- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 8 ++-- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index 607bfd1a2..250936649 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -124,7 +124,7 @@ message CMEOptions optional bool load_security_db = 33 [default = true]; reserved 34; // optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED]; - optional bool maintain_mbo = 35 [default = true]; + optional bool validate_mbo = 35 [default = false]; } optional string onixs_license = 1 [default = "{default}"]; diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 3ec4f31ca..1e0c29dcd 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -109,42 +109,44 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { } bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) { - // TO USE THIS VALIDATION YOU MUST MAKE enqueue_incremental syncronous + // bts_assert_msg(false, "TO USE THIS VALIDATION YOU MUST MAKE enqueue_incremental SYNCRONOUS"); #define REQUIRE(a,b,y) \ if ((a) != (b)) { \ LOG(md,Error, "Validation error exch:{} sec:{} {}!={} other:{}", exchid, key.value(), a, b, y); \ return false; \ } - // #define REQUIRE(x) bts_assert(x); // retrieve current snapshot and check they are equal - MBOReader reader(_me); - auto snap = reader.acquire_snapshot(key); - if (snap) { - REQUIRE (snap->bids.size(), control->bids.size(), '!'); - REQUIRE (snap->asks.size(), control->asks.size(), '!'); - for (size_t x = 0; x < snap->bids.size(); ++x) { - REQUIRE (snap->bids[x].id, control->bids[x].id, x); - REQUIRE (snap->bids[x].size, control->bids[x].size, x); + _me->dispatcher.post([this,exchid,key,control]() { + MBOReader reader(_me); + auto snap = reader.acquire_snapshot(key); + if (snap) { + REQUIRE (snap->bids.size(), control->bids.size(), '!'); + REQUIRE (snap->asks.size(), control->asks.size(), '!'); + for (size_t x = 0; x < snap->bids.size(); ++x) { + REQUIRE (snap->bids[x].id, control->bids[x].id, x); + REQUIRE (snap->bids[x].size, control->bids[x].size, x); + } + for (size_t x = 0; x < snap->asks.size(); ++x) { + REQUIRE (snap->asks[x].id, control->asks[x].id, x); + REQUIRE (snap->asks[x].size, control->asks[x].size, x); + } } - for (size_t x = 0; x < snap->asks.size(); ++x) { - REQUIRE (snap->asks[x].id, control->asks[x].id, x); - REQUIRE (snap->asks[x].size, control->asks[x].size, x); - } - } + return true; + }); #undef REQUIRE return true; } void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { - LOG(md,Info, "sec:{} : {}", state.handle.key(), update); + LOG(md,Diag, "sec:{} : {}", state.handle.key(), update); - // _me->dispatcher.post([this,&state,update]() { + _me->dispatcher.post([this,&state,update]() { // fast part apply_incremental(state,update); // slow part publish_incremental(state); - // }); + }); } void MBOWriter::publish_incremental(MBOWorkingState& state) { // determine if we need to publish diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index d225abfde..547e04fc0 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -61,7 +61,8 @@ int Run::execute(std::vector const &) { user::AuthServiceServer login_server(sc); md::MBOProcessor mbo(&utils::Dispatcher::get_named("MBO")); - md::run_market_datapublishers(&mbo.writer()); + auto* mbowriter = config::get().enabled() ? &mbo.writer() : nullptr; + md::run_market_datapublishers(mbowriter); api::synthetic::SyntheticSecurityService synthetic_service(sc); md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service, &mbo.reader()); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 025de1078..e1880efc4 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -117,11 +117,11 @@ map> create_feed_engines(const vector& symbols, MBOWriter* mbo) { } for (auto& pair : _handlers) { - onixs::SessionSettings settings = session_settings_for_channel(pair.first); + onixs::SessionSettings settings = session_settings_for_channel(pair.first, mbo); pair.second->start(settings); } LOG(md,Info, "** CME MDP3 Publisher started **"); @@ -210,7 +210,7 @@ unique_ptr MDP3Main::prepare_handler(const string& channel_id, M settings.bookManagement().directBooks().maintain(true); settings.bookManagement().impliedBooks().maintain(true); settings.bookManagement().consolidatedBooks().maintain(false); - if (options.maintain_mbo() && mbo != nullptr) { + if (options.validate_mbo() && mbo != nullptr) { settings.bookManagement().mboBooks().maintain(true); } From dfd0cfc2171982a9ab9250739b5a05db94042008 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 25 Apr 2019 17:16:35 -0400 Subject: [PATCH 077/100] templatize mbo callbacks for older templates --- .../md/platforms/mdp3_onixs/MDP3Listener.h | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index d368c8661..1fdb93393 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -104,10 +104,14 @@ public: // // Handle MBO incremental updates directly - void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args&) override { - LOG(md,Error, "GOT IRB32"); + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args& args) override { + onIncrementalRefreshBook(args); }; void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args& args) override { + onIncrementalRefreshBook(args); + } + template + void onIncrementalRefreshBook(const Args& args) { if (!_mbo) return; for(const auto& entry : args.message().orderIdEntries()) { @@ -147,12 +151,16 @@ public: }); } } - }; + } - void onMessage(onixs::Handler&, const onixs::IncrementalRefreshOrderBook43Args&) override { - LOG(md,Info, "GOT IROB43"); + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshOrderBook43Args& args) override { + onIncrementalRefreshOrderBook(args); } void onMessage(onixs::Handler&, const onixs::IncrementalRefreshOrderBook47Args& args) override { + onIncrementalRefreshOrderBook(args); + } + template + void onIncrementalRefreshOrderBook(const Args& args) { if (!_mbo) return; for (const auto& entry : args.message().entries()) { auto data = _index.get_security(entry.securityId()); @@ -193,11 +201,14 @@ public: } } - void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook44Args&) override { - LOG(md,Info, "GOT SNAPSHOT44"); + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook44Args& args) override { + onSnapshotFullRefreshOrderBook(args); } - void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook53Args& args) override { + onSnapshotFullRefreshOrderBook(args); + } + template + void onSnapshotFullRefreshOrderBook(const Args& args) { if (!_mbo) return; auto data = _index.get_security(args.message().securityId()); if (!data) return; From b129bea1296d57f19ebaef6c51a77ccf4ef2766f Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 26 Apr 2019 10:04:07 -0500 Subject: [PATCH 078/100] PR edit - method signature all on one line --- src/client/spark/Orders/OrdersModel.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 5be617a10..9793e8353 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -239,8 +239,7 @@ namespace mbts.spark { public void Dispose() => Disposable.dispose(ref _dispose); - static ImmutableDictionary - _update_working_orders(ImmutableDictionary target, OrderUpdate table) { + static ImmutableDictionary _update_working_orders(ImmutableDictionary target, OrderUpdate table) { var clone = target.ToBuilder(); foreach (var o in table.removed) { if (o.order.is_active()) { @@ -263,8 +262,7 @@ namespace mbts.spark { return clone.ToImmutableDictionary(); } - static ImmutableDictionary - _update_working_orders(ImmutableDictionary target, OrderUpdatePiq table) { + static ImmutableDictionary _update_working_orders(ImmutableDictionary target, OrderUpdatePiq table) { var clone = target.ToBuilder(); foreach (var o in table.removed) { if (o.order.is_active()) { From b102b271b03d2e8c46db630afee4bf17e7595c47 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 26 Apr 2019 10:08:10 -0500 Subject: [PATCH 079/100] PR edit - use single_order instead; this will be gone from attach/detach --- src/client/spark/view/Spark/Ladder/LadderOrderVM.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs b/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs index 2768b017b..a78ff0443 100644 --- a/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs @@ -112,7 +112,7 @@ namespace mbts.spark { var single_order = _orders.Values.First().order; var single_order_metadata = _parent.model.order_gateway.factory.get_metadata(single_order); border = single_order_metadata.Color.rgb_brush(); - is_exchange_order = OrderUtils.is_exchange_order(_orders.Values.First().order); + is_exchange_order = OrderUtils.is_exchange_order(single_order); } else { border = new SolidColorBrush(color); is_exchange_order = false; From 43a18941a40c72393f8804d6df6c027a9b8b3595 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 26 Apr 2019 16:16:59 -0400 Subject: [PATCH 080/100] Change debug assert to logged warning in PIQ --- src/lib/md/MBOProcessor.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 1e0c29dcd..fe1627a74 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -184,10 +184,12 @@ void apply_incremental(C& collection, std::unordered_mapsecond; collection.erase(oldkey); index.erase(r); - index[update.order_id] = key; } - else { bts_assert_d(false); } + else { + LOG(md,Warn, "MBO Update w/out Add for: {}", update); + } collection.insert(std::make_pair(key, MBOWorkingState::Data { update.order_id, update.size })); + index[update.order_id] = key; } break; case inc::UpdateAction::Delete: @@ -242,10 +244,10 @@ ostream& operator<<(ostream& os, const inc::Update& update) { switch(update.type) { case inc::UpdateType::Bid: - os << "bid "; + os << "B "; break; case inc::UpdateType::Ask: - os << "ask "; + os << "A "; break; } From 92f67d1d2d67394b38f7ff2c602cbe24b80ce6f3 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 26 Apr 2019 16:40:59 -0400 Subject: [PATCH 081/100] piq: Add security key to log message --- src/lib/md/MBOProcessor.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index fe1627a74..88cc1f3f2 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -170,7 +170,7 @@ void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { } template -void apply_incremental(C& collection, std::unordered_map& index, const inc::Update& update) { +void apply_incremental(uint32_t security_key, C& collection, std::unordered_map& index, const inc::Update& update) { auto key = std::make_tuple(update.price, update.priority); switch(update.action) { case inc::UpdateAction::New: @@ -186,7 +186,7 @@ void apply_incremental(C& collection, std::unordered_map Date: Mon, 29 Apr 2019 09:13:09 -0400 Subject: [PATCH 082/100] Update release notes --- doc/user/content/release_notes/_index.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index db2b50254..29fc0e4b3 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -4,6 +4,11 @@ weight = 3 sort_by = "weight" insert_anchor_links = "right" +++ +## 1.5.9 +### User Visible Changes + - You can now view your order's position in queue in the ladder + - Fix a bug in the order popup in the ladder which could lead to a crash. + ## 1.5.8 ### User Visible Changes - Fixed a bug where it was possible to crash the UI by entering an order through the order ticket if Enter was used to dismiss the dialog. From 61745d0ed79767e47d688e361502146e4ac00cec Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 29 Apr 2019 09:35:13 -0400 Subject: [PATCH 083/100] Update release notes --- doc/user/content/release_notes/_index.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index 29fc0e4b3..0d403e851 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -4,11 +4,15 @@ weight = 3 sort_by = "weight" insert_anchor_links = "right" +++ -## 1.5.9 +## 1.5.10 ### User Visible Changes - You can now view your order's position in queue in the ladder - Fix a bug in the order popup in the ladder which could lead to a crash. +## 1.5.9 +### User Visible Changes + - Fix for stunnel connections on Windows 7 + ## 1.5.8 ### User Visible Changes - Fixed a bug where it was possible to crash the UI by entering an order through the order ticket if Enter was used to dismiss the dialog. From 5a1cc4cf2b85088c7ac4f2a7d8b7b4291f133a11 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 29 Apr 2019 09:38:02 -0400 Subject: [PATCH 084/100] Update release notes --- doc/user/content/release_notes/_index.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index 0d403e851..3c158addf 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -4,9 +4,11 @@ weight = 3 sort_by = "weight" insert_anchor_links = "right" +++ -## 1.5.10 +## 1.6.0 ### User Visible Changes +#### New Features - You can now view your order's position in queue in the ladder +#### Bug Fixes - Fix a bug in the order popup in the ladder which could lead to a crash. ## 1.5.9 From 221814cc1125dbec6430b9984f6d86edf89f0e08 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 29 Apr 2019 16:30:18 -0400 Subject: [PATCH 085/100] Add OrderStress tool for perf testing --- src/client/sparkcmd/Cmd/OrderStress.cs | 157 +++++++++++++++++++++++++ src/client/sparkcmd/Program.cs | 5 +- 2 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 src/client/sparkcmd/Cmd/OrderStress.cs diff --git a/src/client/sparkcmd/Cmd/OrderStress.cs b/src/client/sparkcmd/Cmd/OrderStress.cs new file mode 100644 index 000000000..43f8d9765 --- /dev/null +++ b/src/client/sparkcmd/Cmd/OrderStress.cs @@ -0,0 +1,157 @@ +using bts.api; +using bts.api.core; +using bts.api.orderman; +using bts.co2; +using bts.test; +using CommandLine; +using mbts.co2; +using mbts.config; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace bts { + + [Verb("orderstress", HelpText = "Create a bunch of orders")] + public class OrderStressOptions { + // Maybe options to create fewer orders or sleep + + [Option('s', "sleep", HelpText = "Sleep ms between order submission", Default = 100)] + public int sleep { get; set; } + + [Option('d', "depth", HelpText = "How many orders per side", Default = 10u)] + public uint depth { get; set; } + + [Option('q', "quanityt", HelpText = "Order quantity", Default = 1u)] + public uint quantity { get; set; } + }; + + public struct Market { + public SecurityImmutable security; + public IntPrice? bid; + public IntPrice? ask; + } + + public static class OrderStress { + static IEnumerable all_markets(ServiceContext ctx) { + var channels = + UserSymbol.get_all() + .Select(u => + new SecurityChannel() { + Time = 0, + Flags = (int)(MDRequest.Types.Flags.Direct | MDRequest.Types.Flags.Implied), + Type = u.type, + Symbol = u.symbol, + Exchange = u.exchange + } + ); + var request = new MDRequest() { Host = "system_test" }; + request.Channels.AddRange(channels); + + var relay = ctx.locate(); + System.Threading.Thread.Sleep(100); + var result = relay.requestmd(request); + + foreach (var ch in result.Channels) { + foreach (var update in ch.Updates) { + if (update.Security != null) { + var m = new Market() { + security = new SecurityImmutable(update.Security), + bid = best( + (a,b) => a > b ? a : b, + update.Direct?.Bids?.FirstOrDefault(), update.Implied?.Bids?.FirstOrDefault()), + ask = best( + (a,b) => a < b ? a : b, + update.Direct?.Asks?.FirstOrDefault(), update.Implied?.Asks?.FirstOrDefault()) + }; + if (m.bid.HasValue || m.ask.HasValue) { + yield return m; + } + } + } + } + } + + private static IntPrice? best(Func comp, params PriceSizeMessage[] ps) { + IntPrice? result = null; + foreach (var p in ps) { + if (p != null) { + var ip = new IntPrice(p.Mantissa, (sbyte)p.Exponent); + if (result == null) { + result = ip; + } + else { + result = comp(result.Value, ip); + } + } + } + return result; + } + + static Create mkorder(uint security, double price, uint size, Side side) { + return new Create { + Price = price, + Quantity = size, + NewOrder = new OrderDefinition { + SecurityKey = security, + Side = side, + TimeInForce = TimeInForce.Day, + ManualOrderIndicator = ManualOrderIndicator.Yes, + CtiCode = CTICode.CtiCode1, + CustomerOrFirm = CustomerOrFirm.Firm, + AccountId = 1, + UserId = 1, + CustomDef = new CustomOrderDefinition { + Limit = new LimitOrderDefinition() + } + } + }; + } + + public static int Run(OrderStressOptions o) { + + var ctx = new ServiceContext(); + ConsoleEx.WriteLine("Please log in to Spark", ConsoleColor.Yellow); + ConsoleEx.Write("Username: ", ConsoleColor.Yellow); + var name = Console.ReadLine(); + ConsoleEx.Write("Password: ", ConsoleColor.Yellow); + var pw = Console.ReadLine(); + + var login_result = ctx.login(name, pw); + if (!login_result.Result.success) { + ConsoleEx.WriteLine("Invalid Credentials", ConsoleColor.Red); + } + + if (Configuration.get().type.ToUpper() == "PROD") { + ConsoleEx.WriteLine("Refusing to run on production system", ConsoleColor.Red); + return -1; + } + + var orders = ctx.locate(); + foreach (var m in all_markets(ctx)) { + ConsoleEx.WriteLine(m.security.security_desc(), ConsoleColor.White); + var tick = m.security.ContractSpec.LowTick; + if (m.bid.HasValue) { + // start at bid and place 1 order at 10 levels + for (var x=0; x < o.depth; ++x) { + var price = m.bid.Value.price() - (tick * x); + ConsoleEx.WriteLine($"Buy {o.quantity} at {price}", ConsoleColor.Blue); + orders.create_order_async(mkorder(m.security.Id, price, o.quantity, Side.Buy)); + System.Threading.Thread.Sleep(o.sleep); + } + } + if (m.ask.HasValue) { + for (var x = 0; x < o.depth; ++x) { + var price = m.ask.Value.price() + (tick * x); + ConsoleEx.WriteLine($"Sell {o.quantity} at {price}", ConsoleColor.Red); + orders.create_order_async(mkorder(m.security.Id, price, o.quantity, Side.Sell)); + System.Threading.Thread.Sleep(o.sleep); + } + } + } + return 3; + } + } + +} \ No newline at end of file diff --git a/src/client/sparkcmd/Program.cs b/src/client/sparkcmd/Program.cs index 9c56c1746..a91fed7f2 100644 --- a/src/client/sparkcmd/Program.cs +++ b/src/client/sparkcmd/Program.cs @@ -26,14 +26,15 @@ namespace sparkcmd { mbts.utils.BTSNativeMethods.provide_command_line(args, args.Length); // note: this can be made to infer the options type list from the list of delegates return Parser.Default - .ParseArguments(args) - .MapResult( + .ParseArguments(args) + .MapResult( Configcheck.Run, ScriptClientCommand.Run, TailService.Run, ServiceTest.Run, SettingsManagerCommand.Run, TestCommand.Run, + OrderStress.Run, errors => 1); } } From 4279e3d388429de7e40359140f67ec655b42c100 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 29 Apr 2019 16:36:16 -0400 Subject: [PATCH 086/100] Fix broken managed md unit test --- src/client/spark_api/Co2/Co2NativeMethods.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/spark_api/Co2/Co2NativeMethods.cs b/src/client/spark_api/Co2/Co2NativeMethods.cs index 495230306..f744b9963 100644 --- a/src/client/spark_api/Co2/Co2NativeMethods.cs +++ b/src/client/spark_api/Co2/Co2NativeMethods.cs @@ -35,6 +35,7 @@ namespace mbts.co2 public readonly IntPtr hot; public readonly IntPtr cold_data; public readonly IntPtr cold_header; + private readonly IntPtr unused_mbo; } [DllImport("libspark")] From 6066e2162c35359f73ea9d7449ccc2cffa934955 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 29 Apr 2019 17:28:23 -0400 Subject: [PATCH 087/100] SPARK-2197 Correction to PIQ calculation per taha (+test coverage) --- src/lib/md/MBOServiceServer.cpp | 6 +++--- src/lib/md/MBOServiceServer.h | 2 ++ src/lib/md/test/mbo_tests.cpp | 25 +++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index e25715ee4..6cabb35a0 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -148,14 +148,14 @@ std::optional search_orders(const ExchangeOrderKey oid, const std::vec const auto& oo = orders[x]; if (oo.id == oid) { if (oo.price == price) return { position }; - else return 0 ; + else return 0; } else if (price == oo.price) { - position += 1; + position += oo.size; } else { price = oo.price; - position = 1; + position = oo.size; } } return std::nullopt; diff --git a/src/lib/md/MBOServiceServer.h b/src/lib/md/MBOServiceServer.h index d0f590a18..6eccab38c 100644 --- a/src/lib/md/MBOServiceServer.h +++ b/src/lib/md/MBOServiceServer.h @@ -2,6 +2,7 @@ #include "lib/api/public/ServiceContext.h" #include "lib/api/services/MBOService.server.pb.h" #include +#include namespace bts { namespace utils { class Dispatcher; } @@ -24,4 +25,5 @@ class MBOServiceServer : public MBOServiceBase { }; + }} //namespace diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp index a275a54c0..69abb194e 100644 --- a/src/lib/md/test/mbo_tests.cpp +++ b/src/lib/md/test/mbo_tests.cpp @@ -9,6 +9,10 @@ namespace bts { namespace md { + +// fwd declare from MBOServiceServer.cpp +std::optional search_orders(const ExchangeOrderKey oid, const std::vector& orders); + namespace test { int mbo_tests = 0; @@ -202,6 +206,7 @@ TEST_CASE("MBO sort order", "[md][mbo]") { } } + TEST_CASE("MBO change price and size", "[md][mbo]") { // When an order's size goes up it gets a new priority Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; @@ -288,7 +293,27 @@ TEST_CASE("MBO change price and size", "[md][mbo]") { } } } + + // Do some tests on the snapshot + MBOSnapshot result; + copy_to_snapshot(state, &result); + + auto ask30 = search_orders(ExchangeOrderKey(701267928346), result.asks); + REQUIRE(ask30); + REQUIRE(*ask30 == 0); + + auto bid2875 = search_orders(ExchangeOrderKey(701267993135), result.bids); + REQUIRE(bid2875); + REQUIRE(*bid2875 == 20); + bid2875 = search_orders(ExchangeOrderKey(701267299079), result.bids); + REQUIRE(bid2875); + REQUIRE(*bid2875 == 6); + + auto not_bid = search_orders(ExchangeOrderKey(701967299879), result.bids); + auto not_ask = search_orders(ExchangeOrderKey(701967299879), result.asks); + REQUIRE(!not_bid); + REQUIRE(!not_ask); } }}} From 6dbeaa14e2b2fa0e7860fa62aad1f1439c124b9d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 30 Apr 2019 10:49:30 -0400 Subject: [PATCH 088/100] SPARK-2343 Update.h does not correctly track 'all' collection breaking limits --- src/client/spark_test/System/LimitsTests.cs | 2 +- src/lib/api/public/Update.h | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/client/spark_test/System/LimitsTests.cs b/src/client/spark_test/System/LimitsTests.cs index 562234dd8..c5f6807cc 100644 --- a/src/client/spark_test/System/LimitsTests.cs +++ b/src/client/spark_test/System/LimitsTests.cs @@ -78,7 +78,7 @@ namespace bts.tests return securities; } - [Test(Timeout = 30)] + [Test(Timeout = 30)] public async Task test_position_limits() { string test_symbol = "CL"; diff --git a/src/lib/api/public/Update.h b/src/lib/api/public/Update.h index c4b615950..bc2885e2f 100644 --- a/src/lib/api/public/Update.h +++ b/src/lib/api/public/Update.h @@ -81,11 +81,6 @@ namespace api { for (size_t x = 0; x < raw.added_keys.size(); ++x) { (*state)[raw.added_keys[x]] = raw.added_items[x]; } - std::vector all; - all.reserve(state->size()); - for (const auto& item : *state) { - all.push_back(item.second); - } // // Change std::vector> changed; @@ -94,6 +89,13 @@ namespace api { changed.push_back(Change{(*state)[raw.changed_keys[x]], raw.changed_items[x]}); (*state)[raw.changed_keys[x]] = raw.changed_items[x]; } + // + // All + std::vector all; + all.reserve(state->size()); + for (const auto& item : *state) { + all.push_back(item.second); + } return Update(all, raw.added_items, changed, removed); } From ad3950e2c545fb3dc7d264f3c3949efdf021d316 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 30 Apr 2019 11:31:38 -0500 Subject: [PATCH 089/100] SPARK-2348 redux - eliminate public orders accessor; fixes order modifies, i.e. drag and drop badge, in ladder --- src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs | 5 +++-- src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs | 2 +- .../spark/Modules/Trading/Ladder/View/LadderOrdersHost.cs | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs b/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs index bcecc88a2..d98014bcb 100644 --- a/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs +++ b/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs @@ -192,7 +192,8 @@ namespace mbts.spark { public bool is_hedge_order { get => _is_hedge_order.Value; private set => _is_hedge_order.Value = value; } public CustomOrderDefinition.CustomOneofCase algo_type { get => _algo_type.Value; private set => _algo_type.Value = value; } - public IEnumerable orders => _orders.Values; + public IReadOnlyList order_keys => _orders.Keys.ToReadOnlyList(); + public IEnumerable root_orders => _root_nodes(); void _cancel_order() { foreach (var order in _root_nodes()) { @@ -278,7 +279,7 @@ namespace mbts.spark { } public override string ToString() { - return $"Price: {this.price} Order Count: {this.orders.Count()}"; + return $"Price: {price} Order Count: {_orders.Values.Count()} Root Order Count: {_root_nodes().Count()}"; } } } diff --git a/src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs b/src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs index 31508a8a3..43e3fd620 100644 --- a/src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs +++ b/src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs @@ -183,7 +183,7 @@ namespace mbts.spark { // remove any orders from the lovm that aren't in here foreach(var kvp in orders_used_in_this_update) { var slot_vm = _slot_price_vs_orders[kvp.Key]; - foreach(var existing_order_key in slot_vm.orders.Select(o => o.Key).ToList()) { + foreach(var existing_order_key in slot_vm.order_keys) { if (!kvp.Value.Contains(existing_order_key)) { slot_vm.remove_order(existing_order_key); } diff --git a/src/client/spark/Modules/Trading/Ladder/View/LadderOrdersHost.cs b/src/client/spark/Modules/Trading/Ladder/View/LadderOrdersHost.cs index 05d66a2c5..51bda9a32 100644 --- a/src/client/spark/Modules/Trading/Ladder/View/LadderOrdersHost.cs +++ b/src/client/spark/Modules/Trading/Ladder/View/LadderOrdersHost.cs @@ -219,7 +219,7 @@ namespace mbts.spark { Log.Ui.info("Drop - " + vm.summary); double price = _parent.price_for_point(e.GetPosition(_parent)); if (price != vm.price) { - foreach (var order in vm.orders) { + foreach (var order in vm.root_orders) { vm.try_set_price(order, price); } } From e4ae42e80d3e13988c1c9aed9007dbf255c27442 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 30 Apr 2019 12:27:31 -0500 Subject: [PATCH 090/100] badge - summary no longer needs to be something that implements IRaiseNotifyPropertyChanged --- .../spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs b/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs index d98014bcb..4adbb9dca 100644 --- a/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs +++ b/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs @@ -37,7 +37,6 @@ namespace mbts.spark { readonly Property _foreground, _background, _hoverbackground, _border; readonly Property _has_multiple_orders_at_same_level, _is_hedge_order; readonly Property _algo_type; - readonly Property _summary; IObservable _bs; LadderSettings.BadgeStyle _current_badge_style; int _n_active_price_change_ops = 0; @@ -62,7 +61,6 @@ namespace mbts.spark { _background = backing(nameof(background), _color_bg_brush); _hoverbackground = backing(nameof(hoverbackground), _color_bg_brush); _border = backing(nameof(border), _color_bg_brush); - _summary = backing(nameof(summary), string.Empty); _has_multiple_orders_at_same_level = backing(nameof(has_multiple_orders_at_same_level), false); _is_hedge_order = backing(nameof(is_hedge_order), false); _algo_type = backing(nameof(algo_type), CustomOrderDefinition.CustomOneofCase.None); @@ -125,10 +123,6 @@ namespace mbts.spark { has_multiple_orders_at_same_level = num_root_orders > 1; is_hedge_order = _root_nodes().Any(o => o.sub_type() == OrderSubType.HedgingLeg); - if (_root_nodes().Any()) { - // This is only used in log messages vs the no-longer-existing badge popup, so simplify string - summary = $"Badge containing root orders: {string.Join(",", _root_nodes().Select(o => o.Key))}"; - } _set_badge_text(); } @@ -187,7 +181,7 @@ namespace mbts.spark { public Brush background { get => _background.Value; private set => _background.Value = value; } public Brush hoverbackground { get => _hoverbackground.Value; private set => _hoverbackground.Value = value; } public Brush border { get => _border.Value; private set => _border.Value = value; } - public string summary { get => _summary.Value; private set => _summary.Value = value; } + public string summary => _root_nodes().Any() ? $"Badge containing root orders: {string.Join(",", _root_nodes().Select(o => o.Key))}" : string.Empty; public bool has_multiple_orders_at_same_level { get => _has_multiple_orders_at_same_level.Value; private set => _has_multiple_orders_at_same_level.Value = value; } public bool is_hedge_order { get => _is_hedge_order.Value; private set => _is_hedge_order.Value = value; } public CustomOrderDefinition.CustomOneofCase algo_type { get => _algo_type.Value; private set => _algo_type.Value = value; } From 69e2bcb164bdba8a476818f849ca93ccaf662124 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 30 Apr 2019 12:51:29 -0500 Subject: [PATCH 091/100] badge - misc refactoring --- .../Trading/Ladder/VM/LadderOrderVM.cs | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs b/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs index 4adbb9dca..3dc2be13e 100644 --- a/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs +++ b/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs @@ -29,7 +29,7 @@ namespace mbts.spark { } #endregion - readonly LadderVM _parent; + readonly LadderVM _owner; readonly IBus _bus; readonly Property _font_family; readonly Property _font_weight; @@ -44,20 +44,20 @@ namespace mbts.spark { uint filled, quantity, num_root_orders; int working_quantity; - LadderModel model => _parent.model; + LadderModel model => _owner.model; Dictionary _orders = new Dictionary(); - public LadderOrderVM(OrderRecordImmutable o, double real_price, LadderVM parent, IBus bus, IObservable bs) { - _parent = parent; + public LadderOrderVM(OrderRecordImmutable o, double real_price, LadderVM owner, IBus bus, IObservable bs) { + _owner = owner; _bus = bus; _bs = bs; _real_price = real_price; // settings var fs = UserSettings.factory_defaults.ladder; - _font_family = backing(nameof(font_family), parent.settings, get_font_family, get_font_family(fs)); - _font_weight = backing(nameof(font_weight), parent.settings, get_font_weight, get_font_weight(fs)); - _font_size = backing(nameof(font_size), parent.settings, get_font_size, get_font_size(fs)); - _foreground = backing(nameof(foreground), parent.settings, get_foreground, get_foreground(fs)); + _font_family = backing(nameof(font_family), owner.settings, get_font_family, get_font_family(fs)); + _font_weight = backing(nameof(font_weight), owner.settings, get_font_weight, get_font_weight(fs)); + _font_size = backing(nameof(font_size), owner.settings, get_font_size, get_font_size(fs)); + _foreground = backing(nameof(foreground), owner.settings, get_foreground, get_foreground(fs)); _background = backing(nameof(background), _color_bg_brush); _hoverbackground = backing(nameof(hoverbackground), _color_bg_brush); _border = backing(nameof(border), _color_bg_brush); @@ -109,7 +109,7 @@ namespace mbts.spark { // should be set appropriately if (num_root_orders == 1) { var single_order = _root_nodes().First(); - var single_order_metadata = _parent.model.order_gateway.factory.get_metadata(single_order); + var single_order_metadata = _owner.model.order_gateway.factory.get_metadata(single_order); border = single_order_metadata.Color.rgb_brush(); if (single_order.order_type().is_bts_algo_type()) { algo_type = single_order.order_type(); @@ -147,7 +147,7 @@ namespace mbts.spark { // } //} - public uint security_key => _parent.security.Id; + public uint security_key => _owner.security.Id; public double price { get => double.IsNaN(_phantom_price) ? _real_price : _phantom_price; } public double real_price => _real_price; public Side side { @@ -160,7 +160,7 @@ namespace mbts.spark { } } - IEnumerable _root_nodes() => _orders.Values.Select(_parent.get_root_at_price).DistinctOn(OrderUtils.RecordKeySelector); + IEnumerable _root_nodes() => _orders.Values.Select(_owner.get_root_at_price).DistinctOn(OrderUtils.RecordKeySelector); public IReadOnlyList text_lines { get; private set; } @@ -202,7 +202,7 @@ namespace mbts.spark { _phantom_price = new_price; RaisePropertyChanged(nameof(price)); ++_n_active_price_change_ops; - if (await _parent.model.order_gateway.update_order(order, new_price, order.State.Quantity)) { + if (await _owner.model.order_gateway.update_order(order, new_price, order.State.Quantity)) { // wait 3 secs, then give up if price change still hasn't taken effect await Task.Delay(3000); } @@ -224,10 +224,10 @@ namespace mbts.spark { price_filter.minimum.value = real_price; price_filter.maximum.value = real_price; - var obvm = new OrderBookVM(null, _parent.bus, null, _parent.model.order_gateway, null, true); + var obvm = new OrderBookVM(null, _owner.bus, null, _owner.model.order_gateway, null, true); var state = new bts.shell.orderbook.OrderBookStartupData( new bts.shell.orderbook.FilterStates { - security_criteria = new SecurityCriteria(_parent.security), + security_criteria = new SecurityCriteria(_owner.security), price = price_filter, statuses = new[] { OrderStatus.Working } }, From 8210f4f74aafbb2716d963b2deb9625220a24023 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 30 Apr 2019 10:19:38 -0500 Subject: [PATCH 092/100] make INode covariant, bc might as well --- src/client/spark_core/Collections/SortedTree.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark_core/Collections/SortedTree.cs b/src/client/spark_core/Collections/SortedTree.cs index f8467669b..92a4ebb4f 100644 --- a/src/client/spark_core/Collections/SortedTree.cs +++ b/src/client/spark_core/Collections/SortedTree.cs @@ -5,7 +5,7 @@ using System.Collections.Generic; using System.Linq; namespace bts.collections { - public interface INode { + public interface INode { T value { get; } IEnumerable> inner { get; } } From 2e49534483cf25cb4e413c35b129eb6477aef030 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 30 Apr 2019 11:14:10 -0500 Subject: [PATCH 093/100] move OrderBookGridSettingsManager into its own file --- .../Settings/OrderBookGridSettingsManager.cs | 115 ++++++++++++++++ .../OrderBook/Settings/OrderBookSettingsVM.cs | 130 ------------------ 2 files changed, 115 insertions(+), 130 deletions(-) create mode 100644 src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookGridSettingsManager.cs diff --git a/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookGridSettingsManager.cs b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookGridSettingsManager.cs new file mode 100644 index 000000000..5101d7fe7 --- /dev/null +++ b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookGridSettingsManager.cs @@ -0,0 +1,115 @@ +using bts.api; +using bts.api.orderman; +using mbts; +using mbts.spark; +using mbts.ui; +using mbts.ui.controls; +using mbts.ui.settings; +using mbts.ui.utility; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using Columns = mbts.spark.OrderBookGridView.ColumnKeys; + +namespace bts.shell.orderbook { + class OrderBookGridSettingsManager : IDisposable, IObserver { + readonly IObserver _core_man_as_observer; + AlgoTreeDisplayMode _algo_tree_display_mode; + + public OrderBookGridSettingsManager( + IObservable owner_title, + IBus bus, + INotificationService notification_service, + OrdersModel model, + IEnumerable rows, + ObservableCollection columns) { + // make font-weight immutable for all columns (the font-weight rule is hard-coded: Regular for root-orders, Light for all others) + // TODO: Remove the immutable font-weight flags when cell animation is ready + var col_settings_flags = EnumHelper.get_values().ToDictionary( + c => c.ToString(), c => GridColumnSettingsFeatureFlags.ImmutableFontWeight); + // make font-color immutable for Side, Size, and Filled columns (the font-color rule is hard-coded: MarketData.bid for Buy orders, MarketData.ask for Sells) + // TODO: Remove the immutable font-color flags when cell animation is ready + foreach (var c in new[] { Columns.Side, Columns.Size, Columns.Filled }) { + col_settings_flags[c.ToString()] = GridColumnSettingsFeatureFlags.ImmutableFontColor | GridColumnSettingsFeatureFlags.ImmutableFontWeight; + } + core_manager = new GridSettingsManager( + owner_title, bus, model, rows, columns, + () => new OrderRecordTuplePredicate(), + () => { + var pvm = PredicateVMWrapper.create( + OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); + pvm.selected_predicate_type = pvm.predicate_types.FirstOrDefault(); + return pvm; + }, + col_id => { + var pvm = PredicateVMWrapper.create( + OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); + pvm.selected_predicate_type = _corresponding_predicate_type(EnumHelper.parse_or_default(col_id)); + return pvm; + }, + (gen, anims) => new OrderBookGridSettings(gen, anims.Select(a => new OrderRecordTuplePredicateAnimations(a.predicate_data, a.cell_animations, a.row_animations, a.sound_notifications))) { algo_tree_display_mode = _algo_tree_display_mode }, + p => p.create_predicate(notification_service.data_cache), + p => p.create_notification_predicate(notification_service.data_cache), + notification_service == null ? new Func, SoundNotification, IDisposable>((p, sn) => Disposable.empty) : notification_service.link_to_order, + notification_service == null ? new Func, IDisposable>(t => Disposable.empty) : notification_service.link_to_order, + col_settings_flags); + _core_man_as_observer = core_manager; + } + + public GridSettingsManager core_manager { get; } + + public void load_from(OrderBookGridSettings gs) { + core_manager.load_from(gs); + _algo_tree_display_mode = gs.algo_tree_display_mode; + } + + public OrderBookGridSettings create(IEnumerable sorting_criterion) { + var gs = core_manager.create(); + gs.general = gs.general.with(sorting_criterion: sorting_criterion); + gs.algo_tree_display_mode = _algo_tree_display_mode; + return gs; + } + + public OrderBookSettingsVM create_vm(OrderBookGridSettings gs, IObservable owner_title) => + new OrderBookSettingsVM(core_manager.create_vm(gs.general, owner_title)) { + algo_tree_display_mode = gs.algo_tree_display_mode + }; + + public void Dispose() => core_manager.Dispose(); + + static OrderRecordTuplePredicate.PredicateTypeOneofCase _corresponding_predicate_type(Columns col) { + switch (col) { + case Columns.Security: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Security; + case Columns.Time: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Time; + case Columns.LastChange: return OrderRecordTuplePredicate.PredicateTypeOneofCase.LastChange; + case Columns.User: return OrderRecordTuplePredicate.PredicateTypeOneofCase.User; + case Columns.Account: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Account; + case Columns.Exchange: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Exchange; + case Columns.TimeInForce: return OrderRecordTuplePredicate.PredicateTypeOneofCase.TimeInForce; + case Columns.OrderType: return OrderRecordTuplePredicate.PredicateTypeOneofCase.OrderType; + case Columns.SubType: return OrderRecordTuplePredicate.PredicateTypeOneofCase.SubType; + case Columns.Side: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Side; + case Columns.Size: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Size; + case Columns.Price: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Price; + case Columns.Filled: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Filled; + case Columns.AvgFilled: return OrderRecordTuplePredicate.PredicateTypeOneofCase.AvgFilled; + case Columns.LastPrice: return OrderRecordTuplePredicate.PredicateTypeOneofCase.LastPrice; + case Columns.Status: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Status; + case Columns.LastMessage: return OrderRecordTuplePredicate.PredicateTypeOneofCase.LastMessage; + case Columns.SecurityKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.SecurityKey; + case Columns.OrderKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.OrderKey; + case Columns.ParentKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.ParentKey; + case Columns.AttachedTargetOrderKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.AttachedTargetOrderKey; + default: return default; + } + } + + void IObserver.OnNext(OrderBookGridSettings value) { + _core_man_as_observer.OnNext(value); + _algo_tree_display_mode = value.algo_tree_display_mode; + } + void IObserver.OnError(Exception error) => _core_man_as_observer.OnError(error); + void IObserver.OnCompleted() => _core_man_as_observer.OnCompleted(); + } +} \ No newline at end of file diff --git a/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettingsVM.cs b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettingsVM.cs index 70c1a4b3e..6791e07c4 100644 --- a/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettingsVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettingsVM.cs @@ -3,118 +3,13 @@ using bts.api.orderman; using mbts; using mbts.spark; using mbts.ui; -using mbts.ui.controls; using mbts.ui.settings; using mbts.ui.shell; using mbts.ui.spark; -using mbts.ui.utility; using System; using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Linq; -using Columns = mbts.spark.OrderBookGridView.ColumnKeys; namespace bts.shell.orderbook { - class OrderBookGridSettingsManager : IDisposable, IObserver { - readonly IObserver _core_man_as_observer; - AlgoTreeDisplayMode _algo_tree_display_mode; - - public OrderBookGridSettingsManager( - IObservable owner_title, - IBus bus, - INotificationService notification_service, - OrdersModel model, - IEnumerable rows, - ObservableCollection columns) { - // make font-weight immutable for all columns (the font-weight rule is hard-coded: Regular for root-orders, Light for all others) - // TODO: Remove the immutable font-weight flags when cell animation is ready - var col_settings_flags = EnumHelper.get_values().ToDictionary( - c => c.ToString(), c => GridColumnSettingsFeatureFlags.ImmutableFontWeight); - // make font-color immutable for Side, Size, and Filled columns (the font-color rule is hard-coded: MarketData.bid for Buy orders, MarketData.ask for Sells) - // TODO: Remove the immutable font-color flags when cell animation is ready - foreach (var c in new[] { Columns.Side, Columns.Size, Columns.Filled }) { - col_settings_flags[c.ToString()] = GridColumnSettingsFeatureFlags.ImmutableFontColor | GridColumnSettingsFeatureFlags.ImmutableFontWeight; - } - core_manager = new GridSettingsManager( - owner_title, bus, model, rows, columns, - () => new OrderRecordTuplePredicate(), - () => { - var pvm = PredicateVMWrapper.create( - OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); - pvm.selected_predicate_type = pvm.predicate_types.FirstOrDefault(); - return pvm; - }, - col_id => { - var pvm = PredicateVMWrapper.create( - OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); - pvm.selected_predicate_type = _corresponding_predicate_type(EnumHelper.parse_or_default(col_id)); - return pvm; - }, - (gen, anims) => new OrderBookGridSettings(gen, anims.Select(a => new OrderRecordTuplePredicateAnimations(a.predicate_data, a.cell_animations, a.row_animations, a.sound_notifications))) { algo_tree_display_mode = _algo_tree_display_mode }, - p => p.create_predicate(notification_service.data_cache), - p => p.create_notification_predicate(notification_service.data_cache), - notification_service == null ? new Func, SoundNotification, IDisposable>((p, sn) => Disposable.empty) : notification_service.link_to_order, - notification_service == null ? new Func, IDisposable>(t => Disposable.empty) : notification_service.link_to_order, - col_settings_flags); - _core_man_as_observer = core_manager; - } - - public GridSettingsManager core_manager { get; } - - public void load_from(OrderBookGridSettings gs) { - core_manager.load_from(gs); - _algo_tree_display_mode = gs.algo_tree_display_mode; - } - - public OrderBookGridSettings create(IEnumerable sorting_criterion) { - var gs = core_manager.create(); - gs.general = gs.general.with(sorting_criterion: sorting_criterion); - gs.algo_tree_display_mode = _algo_tree_display_mode; - return gs; - } - - public OrderBookSettingsVM create_vm(OrderBookGridSettings gs, IObservable owner_title) => - new OrderBookSettingsVM(core_manager.create_vm(gs.general, owner_title)) { - algo_tree_display_mode = gs.algo_tree_display_mode - }; - - public void Dispose() => core_manager.Dispose(); - - static OrderRecordTuplePredicate.PredicateTypeOneofCase _corresponding_predicate_type(Columns col) { - switch (col) { - case Columns.Security: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Security; - case Columns.Time: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Time; - case Columns.LastChange: return OrderRecordTuplePredicate.PredicateTypeOneofCase.LastChange; - case Columns.User: return OrderRecordTuplePredicate.PredicateTypeOneofCase.User; - case Columns.Account: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Account; - case Columns.Exchange: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Exchange; - case Columns.TimeInForce: return OrderRecordTuplePredicate.PredicateTypeOneofCase.TimeInForce; - case Columns.OrderType: return OrderRecordTuplePredicate.PredicateTypeOneofCase.OrderType; - case Columns.SubType: return OrderRecordTuplePredicate.PredicateTypeOneofCase.SubType; - case Columns.Side: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Side; - case Columns.Size: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Size; - case Columns.Price: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Price; - case Columns.Filled: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Filled; - case Columns.AvgFilled: return OrderRecordTuplePredicate.PredicateTypeOneofCase.AvgFilled; - case Columns.LastPrice: return OrderRecordTuplePredicate.PredicateTypeOneofCase.LastPrice; - case Columns.Status: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Status; - case Columns.LastMessage: return OrderRecordTuplePredicate.PredicateTypeOneofCase.LastMessage; - case Columns.SecurityKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.SecurityKey; - case Columns.OrderKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.OrderKey; - case Columns.ParentKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.ParentKey; - case Columns.AttachedTargetOrderKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.AttachedTargetOrderKey; - default: return default; - } - } - - void IObserver.OnNext(OrderBookGridSettings value) { - _core_man_as_observer.OnNext(value); - _algo_tree_display_mode = value.algo_tree_display_mode; - } - void IObserver.OnError(Exception error) => _core_man_as_observer.OnError(error); - void IObserver.OnCompleted() => _core_man_as_observer.OnCompleted(); - } - class OrderBookSettingsVM : DisposableViewModelBase, IWindowSettingsVM, ITemplateSelectorKey { AlgoTreeDisplayMode _algo_tree_display_mode; @@ -147,29 +42,4 @@ namespace bts.shell.orderbook { return gs; } } - //class OrderBookSettingsVM : GridSettingsVM { - // AlgoTreeDisplayMode _algo_tree_display_mode; - - // public OrderBookSettingsVM( - // IObservable owner_title, IBus bus, OrdersModel orders_model, - // Func predicate_data_factory, - // Func> notification_predicate_factory, - // Func> create_predicate_vm, - // Func> create_column_predicate_vm, - // Func>, OrderBookGridSettings> create_settings_obj, - // IReadOnlyDictionary column_flags = null) - // : base(owner_title, bus, orders_model, predicate_data_factory, notification_predicate_factory, create_predicate_vm, create_column_predicate_vm, create_settings_obj, column_flags) { } - - // public AlgoTreeDisplayMode algo_tree_display_mode { get => _algo_tree_display_mode; set => Set(ref _algo_tree_display_mode, value); } - - // public override void load(OrderBookGridSettings grid_settings) { - // base.load(grid_settings); - // algo_tree_display_mode = grid_settings.algo_tree_display_mode; - // } - // public override OrderBookGridSettings get_result() { - // var gs = base.get_result(); - // gs.algo_tree_display_mode = algo_tree_display_mode; - // return gs; - // } - //} } \ No newline at end of file From d803622c01c2237e9b11a4ffa7562559c63e263c Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 30 Apr 2019 12:28:37 -0500 Subject: [PATCH 094/100] SPARK-2315 / SPARK-2338 hook up AlgoTreeDisplayMode setting in OrderBookVM --- .../Modules/Trading/OrderBook/OrderBookVM.cs | 71 +++++++++++++++++-- .../Settings/OrderBookGridSettingsManager.cs | 16 ++++- 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index 94b685ada..b424f037f 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -68,6 +68,7 @@ namespace mbts.spark { HashSet _attachable_selected_orders = new HashSet(), _detachable_selected_orders = new HashSet(); + FilterData _cached_filter_data; public OrderBookVM(IObservable owner_title, IBus bus, INotificationService notification_service, OrdersModel model, IWindowSettingsManager settings_manager, bool rows_are_initially_expanded = false) { _bus = bus; @@ -140,6 +141,7 @@ namespace mbts.spark { return dialog; })); grid_settings_manager = new OrderBookGridSettingsManager(owner_title, bus, notification_service, model, rows, columns); + grid_settings_manager.algo_tree_display_mode_changed += _on_filter_changed; IDisposable attach_vm_hooks = null; begin_attaching += on_begin_attaching; end_attaching += on_end_attaching; @@ -214,6 +216,7 @@ namespace mbts.spark { }), Disposable.Create(() => { _columns.filter_changed -= _on_filter_changed; + grid_settings_manager.algo_tree_display_mode_changed -= _on_filter_changed; _row_collection.CollectionChanged -= _row_collection_CollectionChanged; begin_attaching -= on_begin_attaching; end_attaching -= on_end_attaching; @@ -468,16 +471,28 @@ namespace mbts.spark { (var old_filter, var old_sort) = _columns.create(); // suspend column filter updates _columns.filter_changed -= _on_filter_changed; + grid_settings_manager.algo_tree_display_mode_changed -= _on_filter_changed; #pragma warning disable CS0612 // Type or member is obsolete _columns.load_from((state.filters, (state.orderbook_grid?.general ?? OrderBookStartupData.DefaultGrid.general).sorting_criterion)); #pragma warning restore CS0612 // Type or member is obsolete // resume column filter updates _columns.filter_changed += _on_filter_changed; - (var new_filter, var new_sort) = _columns.create(); - if (!new_sort.Equals(old_sort)) - _row_collection.comparer = _columns.get_sorting_data().comparer; - if (!new_filter.Equals(old_filter)) - _on_filter_changed(); + grid_settings_manager.algo_tree_display_mode_changed += _on_filter_changed; + var new_filter = new FilterData(grid_settings_manager.algo_tree_display_mode, _columns.create_filter_states()); + var new_sort = _columns.get_sorting_data(); + if (!new_filter.Equals(_cached_filter_data)) { + _cached_filter_data = new_filter; + if (!new_sort.Equals(old_sort)) { + /// Both filter and sort have changed, use to avoid syncing twice. + using (_row_collection.defer_refresh()) { + _row_collection.filter = new_filter.build_predicate(); + _row_collection.comparer = new_sort.comparer; + } + } else { + _row_collection.filter = new_filter.build_predicate(); + } + } else if (!new_sort.Equals(old_sort)) + _row_collection.comparer = new_sort.comparer; } public void on_sorting(object sender, DIYGridSortingEventArgs e) { @@ -527,8 +542,13 @@ namespace mbts.spark { } } - void _on_filter_changed() => - _row_collection.filter = _columns.create_filter_states().build_predicate(); + void _on_filter_changed() { + var fd = new FilterData(grid_settings_manager.algo_tree_display_mode, _columns.create_filter_states()); + if (!fd.Equals(_cached_filter_data)) { + _cached_filter_data = fd; + _row_collection.filter = fd.build_predicate(); + } + } async Task _attach_orders(IReadOnlyList rows, OrderBookRowVM target) { Log.Ui.info($"{nameof(OrderBookVM)} begin attaching orders ({rows.Count} total)..."); @@ -818,6 +838,43 @@ namespace mbts.spark { } } + struct FilterData : IEquatable { + public readonly AlgoTreeDisplayMode algo_tree_display_mode; + public readonly FilterStates column_filters; + + public FilterData(AlgoTreeDisplayMode algo_tree_display_mode, FilterStates column_filters) { + this.algo_tree_display_mode = algo_tree_display_mode; + this.column_filters = column_filters; + } + + public Func build_predicate() { + var pred = column_filters.build_predicate(); + if (algo_tree_display_mode == AlgoTreeDisplayMode.RootAndLeafNodes) { + if (pred == null) { + pred = tree_disp_pred; + } else { + var col_pred = pred; + pred = o => tree_disp_pred(o) && col_pred(o); + } + bool tree_disp_pred(OrderSecurityAlgoMeta o) => + !o.order.has_parent() || + o.order.is_exchange_order() || + o.order.order_type() == CustomOrderDefinition.CustomOneofCase.Spreader; + } + return pred; + } + + public bool Equals(FilterData other) => + algo_tree_display_mode == other.algo_tree_display_mode && + column_filters.safe_equals(other.column_filters); + + public override bool Equals(object obj) => + obj is FilterData fd && Equals(fd); + + public override int GetHashCode() => + EqualityComparer.aggregate_hash((int)algo_tree_display_mode, column_filters?.GetHashCode() ?? 0); + } + /// /// Monitors selected rows, and determines whether or not they are all active-orders, with common parameters. /// diff --git a/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookGridSettingsManager.cs b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookGridSettingsManager.cs index 5101d7fe7..96659633e 100644 --- a/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookGridSettingsManager.cs +++ b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookGridSettingsManager.cs @@ -57,11 +57,23 @@ namespace bts.shell.orderbook { _core_man_as_observer = core_manager; } + public event Action algo_tree_display_mode_changed; + public GridSettingsManager core_manager { get; } + public AlgoTreeDisplayMode algo_tree_display_mode { + get => _algo_tree_display_mode; + set { + if (_algo_tree_display_mode != value) { + _algo_tree_display_mode = value; + algo_tree_display_mode_changed?.Invoke(); + } + } + } + public void load_from(OrderBookGridSettings gs) { core_manager.load_from(gs); - _algo_tree_display_mode = gs.algo_tree_display_mode; + algo_tree_display_mode = gs.algo_tree_display_mode; } public OrderBookGridSettings create(IEnumerable sorting_criterion) { @@ -107,7 +119,7 @@ namespace bts.shell.orderbook { void IObserver.OnNext(OrderBookGridSettings value) { _core_man_as_observer.OnNext(value); - _algo_tree_display_mode = value.algo_tree_display_mode; + algo_tree_display_mode = value.algo_tree_display_mode; } void IObserver.OnError(Exception error) => _core_man_as_observer.OnError(error); void IObserver.OnCompleted() => _core_man_as_observer.OnCompleted(); From 0e136637e0668d1fb0f9f6c45e65335607167c41 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 30 Apr 2019 12:29:25 -0500 Subject: [PATCH 095/100] OrderBookVM - remove unused type alias --- src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index b424f037f..38b2b7d7b 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -28,12 +28,6 @@ using System.Reactive.Linq; using System.Threading.Tasks; using System.Windows.Controls; using System.Windows.Input; -using _GridSettingsManager = - mbts.ui.GridSettingsManager< - mbts.spark.OrderBookRowVM, - bts.api.orderman.OrderSecurity, - mbts.OrderRecordTuplePredicate, - bts.shell.orderbook.OrderBookGridSettings>; using Columns = mbts.spark.OrderBookGridView.ColumnKeys; namespace mbts.spark { From 4a69a9204b07477f3bb66f053ad2b251c0959933 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 30 Apr 2019 13:50:35 -0500 Subject: [PATCH 096/100] OperableHierarchicalCollection was not sorting root nodes --- .../OperableHierarchicalCollectionModel.cs | 2 +- .../spark_core/Collections/SortedTree.cs | 38 ++++++++----------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/client/spark_core/Collections/Operable/Hierarchical/OperableHierarchicalCollectionModel.cs b/src/client/spark_core/Collections/Operable/Hierarchical/OperableHierarchicalCollectionModel.cs index 6b0adeda3..e986b732c 100644 --- a/src/client/spark_core/Collections/Operable/Hierarchical/OperableHierarchicalCollectionModel.cs +++ b/src/client/spark_core/Collections/Operable/Hierarchical/OperableHierarchicalCollectionModel.cs @@ -47,7 +47,7 @@ namespace bts.collections { public int total_row_count { get; private set; } IEnumerable> _root_nodes => - _filter == null ? _tree.RootNodes : _cached_filtered ?? (_cached_filtered = _tree.RootNodes.filtered(_filter, _comparer, true).ToList()); + _filter == null ? _tree.RootNodes : _cached_filtered ?? (_cached_filtered = _tree.RootNodes.filtered(_filter).sorted_hard(_comparer)); // collapse public void collapse_row(TKey key) => _toggle_is_expanded(key, false); diff --git a/src/client/spark_core/Collections/SortedTree.cs b/src/client/spark_core/Collections/SortedTree.cs index 92a4ebb4f..b09a7cd65 100644 --- a/src/client/spark_core/Collections/SortedTree.cs +++ b/src/client/spark_core/Collections/SortedTree.cs @@ -19,32 +19,12 @@ namespace bts.collections { } } - public static IEnumerable> filtered(this IEnumerable> root_nodes, Func filter, IComparer comparer, bool check_already_sorted = false) { - var node_comp = comparer.wrap>(n => n.value); + public static IEnumerable> filtered(this IEnumerable> root_nodes, Func filter) { return impl(root_nodes); IEnumerable> impl(IEnumerable> nodes) { foreach (var n in nodes) { if (filter(n.value)) { - // TODO: (better) leverage the fact that the node.inner is already sorted - // (impl may require additional params, custom implementation of INode, etc.) - var inner = new List>(); - if (check_already_sorted) { - using (var en = impl(n.inner).GetEnumerator()) { - if (en.MoveNext()) - inner.Add(en.Current); - var max = en.Current; - while (en.MoveNext()) { - if (node_comp.Compare(en.Current, max) >= 0) { - inner.Add(en.Current); - max = en.Current; - } else inner.InsertSorted(en.Current, node_comp); - } - } - } else { - foreach (var _n in impl(n.inner)) - inner.InsertSorted(_n, node_comp); - } - yield return new Node(n.value, inner); + yield return new Node(n.value, impl(n.inner)); } else { foreach (var _n in impl(n.inner)) yield return _n; @@ -53,6 +33,20 @@ namespace bts.collections { } } + public static List> sorted_hard(this IEnumerable> root_nodes, IComparer comparer) { + var node_comp = comparer.wrap>(n => n.value); + var result = new List>(impl(root_nodes)); + result.Sort(node_comp); + return result; + IEnumerable> impl(IEnumerable> nodes) { + foreach (var n in nodes) { + var inner = new List>(impl(n.inner)); + inner.Sort(node_comp); + yield return new Node(n.value, inner); + } + } + } + struct Node : INode { public Node(T value, IEnumerable> inner) { this.value = value; From 8ceab5122850d370fc8b509031091c1f643f7149 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 30 Apr 2019 14:37:20 -0400 Subject: [PATCH 097/100] Removed obsolete class. --- src/lib/orderman/orders/algo/AlgoOrder.cpp | 780 --------------------- src/lib/orderman/orders/algo/AlgoOrder.h | 208 ------ 2 files changed, 988 deletions(-) delete mode 100644 src/lib/orderman/orders/algo/AlgoOrder.cpp delete mode 100644 src/lib/orderman/orders/algo/AlgoOrder.h diff --git a/src/lib/orderman/orders/algo/AlgoOrder.cpp b/src/lib/orderman/orders/algo/AlgoOrder.cpp deleted file mode 100644 index 94dbb7f2b..000000000 --- a/src/lib/orderman/orders/algo/AlgoOrder.cpp +++ /dev/null @@ -1,780 +0,0 @@ -#include "lib/orderman/orders/algo/AlgoOrder.h" -#include "lib/orderman/actors/order_manager/ChildOrder.h" -#include "lib/orderman/actors/order_manager/OrderBuilder.h" -#include "lib/orderman/actors/order_manager/OrderServices.h" -#include "lib/orderman/orderman_utils/orderman_utils.h" -#include "lib/orderman/orders/PreActivationOrderServices.h" -#include "lib/utils/bts_assert.h" - -#include -#include - -namespace bts { -namespace orderman { - -using namespace api::orderman; -template using Map = boost::container::flat_map; -enum class AlgoOrder::Status { INACTIVE, W_ATTACH, WORKING, COMPLETED }; - -class ChildTemplate { - public: - ChildTemplate() {} - - ChildTemplate(OrderKey parent_key, api::orderman::OrderDefinition const &def, - IOrderFactory *factory, size_t reserve_sz) - : builder(parent_key, def, *factory, reserve_sz) {} - - OrderBuilder builder; -}; - -struct AlgoOrder::ChildRecord { - ChildOrder order; - std::optional state; - bool is_subscribing = false; -}; - -struct AlgoOrder::PImpl { - using Edge = std::pair; - using Action = std::function; - - Status status = Status::INACTIVE; - OrderServices *services = nullptr; - OrderDefinition definition; - const api::Security *security; - OrderState const &state() const { return _state; } - - Map book_subs; - Map children; - bool has_timer_sub = false; - - bool is_state_dirty = false; - - Map transition_actions; - Map invariants; - securities::ISecurityDB const *security_db = nullptr; - IOrderFactory *order_factory = nullptr; - std::vector> child_templates; - OrderKey attach_target = null_order_key(); - - std::optional pending_update; - OrderStatus external_status() const { return _state.status(); } - - void set_external_status(OrderStatus s) { - is_state_dirty = true; - _state.set_status(s); - } - - void set_state(OrderState const &new_state) { - is_state_dirty = true; - _state = new_state; - } - - void set_last_msg(std::string const &msg) { - is_state_dirty = true; - _state.set_last_msg(msg); - } - - void set_price_quantity(Price p, Quantity q) { - is_state_dirty = true; - _state.set_price(p.value()); - _state.set_quantity(q.value()); - } - - void set_fill_quantities(Quantity total_fills, Price last_price, - Price avg_price) { - is_state_dirty = true; - _state.set_filled_quantity(total_fills.value()); - _state.set_last_price(last_price.value()); - _state.set_avg_price(avg_price.value()); - } - - private: - OrderState _state; -}; - -struct AlgoOrder::ReportStateGuard { - ReportStateGuard(AlgoOrder &owner) : owner(owner) {} - ~ReportStateGuard() { - if (owner.self->is_state_dirty) { - owner.self->is_state_dirty = false; - owner.self->services->report_state(owner.self->state()); - } - } - AlgoOrder &owner; -}; - -AlgoOrder::AlgoOrder() : self(new PImpl()) { setup_transitions(); } -AlgoOrder::~AlgoOrder() {} - -// -------------------------------------------------------------------------------- -// Queries -// -------------------------------------------------------------------------------- -api::orderman::OrderDefinition const &AlgoOrder::definition() const { - return *def(); -} - -OrderKey AlgoOrder::order_key() const { return key(); } - -OrderKey AlgoOrder::parent_order_key() const { return parent_key(); } - -api::Security const *AlgoOrder::lookup_security(SecurityKey k) const { - return self->security_db->resolve(k); -} - -Price AlgoOrder::price() const { return Price(self->state().price()); } - -Quantity AlgoOrder::quantity() const { - return Quantity(self->state().quantity()); -} - -Quantity AlgoOrder::total_fills() const { - return Quantity(self->state().filled_quantity()); -} - -bool AlgoOrder::is_attaching() const { - return self->attach_target != null_order_key(); -} - -// -------------------------------------------------------------------------------- -// Commands -// -------------------------------------------------------------------------------- -ChildTemplate & -AlgoOrder::make_child_template(api::orderman::OrderDefinition const &child_def, - size_t reserve) { - auto t = std::make_unique(key(), child_def, - self->order_factory, reserve); - if (self->services) - t->builder.activate(*self->services); - self->child_templates.emplace_back(std::move(t)); - return *(self->child_templates.back()); -} - -OrderKey AlgoOrder::launch_child(ChildTemplate &t, Price p, Quantity q, - bool subscribe_to_changes) { - if (!is_working()) - return null_order_key(); - return index_and_launch_child(t.builder.build(), p, q, subscribe_to_changes); -} - -OrderKey AlgoOrder::launch_child(api::orderman::OrderDefinition const &def, - Price p, Quantity q, - bool subscribe_to_changes) { - if (!is_working()) - return null_order_key(); - return index_and_launch_child(ChildOrder{def, *self->services}, p, q, - subscribe_to_changes); -} - -void AlgoOrder::update_child(OrderKey child, - api::orderman::Update const &update) { - if (!is_working()) - return; - - bts_assert(self->status == Status::WORKING); - if (auto it = self->children.find(child); it != self->children.end()) - it->second.order.update(update); - else - terminate_error(fmt::format("Algo {} tried to update unknown child: {}", - key().value(), child.value())); -} - -void AlgoOrder::cancel_child(OrderKey child) { - if (!is_working()) - return; - - bts_assert(self->status == Status::WORKING); - if (auto it = self->children.find(child); it != self->children.end()) - it->second.order.cancel(); - else - terminate_error(fmt::format("Algo {} tried to cancel unknown child: {}", - key().value(), child.value())); -} - -void AlgoOrder::detach_child(OrderKey child) { - if (!is_working()) - return; - - if (auto it = self->children.find(child); it != self->children.end()) { - it->second.order.mark_detached_from_parent(); - self->children.erase(it); - } else - terminate_error(fmt::format("Algo {} tried to cancel unknown child: {}", - key().value(), child.value())); -} - -void AlgoOrder::update_fill_state(Quantity q, Price p) { - if (!is_working()) - return; - - auto previous_qty = self->state().filled_quantity(); - if (previous_qty > q.value()) { - LOG(OrderMan, Error, - "AlgoOrder {} is reporting a decrease in fill quantity " - "from {} to {}. This should never happen. Canceling {}", - key().value(), previous_qty, q.value(), key().value()); - terminate_error("Internal algo error. Report fill size decreased"); - } else { - auto new_avg_price = Price(0); - if (q.value() > 0) - new_avg_price.value() = - (self->state().filled_quantity() * self->state().avg_price() + - (q.value() - previous_qty) * p.value()) / - q.value(); - self->set_fill_quantities(q, p, new_avg_price); - } -} - -void AlgoOrder::cancel(std::string const &msg) { - if (!is_working()) - return; - terminate(ORDER_STATUS_CANCELED, msg); -} - -void AlgoOrder::terminate_error(std::string const &msg) { - if (!is_working() && self->status != Status::W_ATTACH) - return; - - // If we attached and an error was signaled we do not want to - // cancel our child because that child might be a long running - // order with a good queue position that we want to re-use. - if (is_attaching()) - self->children.erase(self->attach_target); - terminate(ORDER_STATUS_ERROR, msg); -} - -void AlgoOrder::detach_all() { - if (!is_working()) - return; - for (auto &[_, c] : self->children) { - (void)_; - c.order.mark_detached_from_parent(); - } - self->children.clear(); -} - -void AlgoOrder::complete() { - if (!is_working()) - return; - terminate(ORDER_STATUS_COMPLETED); -} - -// -------------------------------------------------------------------------------- -// Subscriptions -// -------------------------------------------------------------------------------- - -void AlgoOrder::subscribe_book(SecurityKey k) { - if (!is_working()) - return; - bts_assert(self->book_subs.find(k) == self->book_subs.end()); - - self->book_subs[k] = self->services->subscribe_to_book( - k, [this, k](auto const &ev) { on_book(k, ev.book); }); -} - -void AlgoOrder::unsubscribe_book(SecurityKey k) { - if (!is_working()) - return; - bts_assert(self->book_subs.find(k) != self->book_subs.end()); - if (auto it = self->book_subs.find(k); it != self->book_subs.end()) - self->services->unsubscribe_from_book(it->second); -} - -void AlgoOrder::subscribe_child_change(OrderKey k) { - if (!is_working()) - return; - - auto it = self->children.find(k); - bts_assert(it != self->children.end()); - auto &child_record = it->second; - bts_assert(!child_record.is_subscribing); - child_record.is_subscribing = true; -} - -void AlgoOrder::unsubscribe_child_change(OrderKey k) { - if (!is_working()) - return; - - auto it = self->children.find(k); - bts_assert(it != self->children.end()); - auto &child_record = it->second; - bts_assert(child_record.is_subscribing); - child_record.is_subscribing = false; -} - -void AlgoOrder::subscribe_timer() { - if (!is_working()) - return; - bts_assert(!self->has_timer_sub); - self->has_timer_sub = true; - self->services->subscribe_to_time([this](auto const &) { on_timer(); }); -} - -void AlgoOrder::unsubscribe_timer() { - if (!is_working()) - return; - bts_assert(self->has_timer_sub); - self->services->unsubscribe_from_time(); - self->has_timer_sub = false; -} - -// Private -void AlgoOrder::on_init_new(OrderDefinition const &def, - const PreActivationOrderServices &serv) { - bts_assert(def.has_custom_def()); - - // cache these so we can resolve securities and create order builders - self->security_db = &serv.sec_db; - self->order_factory = &serv.order_factory; - - // Save our order definition - self->definition = def; - - // If we are consuming the attachment we remove it from our definition so - // that derived orders can use our definition to create their childre - // without triggering an attachment. - if (_is_attaching(self->definition)) { - self->attach_target = - OrderKey(self->definition.attach_target_order_key()); - self->definition.clear_attach_target_order_key(); - } - - self->security = serv.sec_db.resolve(SecurityKey(def.security_key())); - if (self->security == nullptr) - terminate(ORDER_STATUS_REJECTED_BY_SPARK, "Unknown security"); - - auto result = do_before_launch(); - if (result.fail()) - terminate(ORDER_STATUS_REJECTED_BY_SPARK, result.error_msg); -} - -void AlgoOrder::on_init_recovered(OrderRecord const &rec, - const PreActivationOrderServices &serv) { - auto const &def = rec.def(); - self->definition = def; - self->set_state(rec.state()); - self->security_db = &serv.sec_db; - self->order_factory = &serv.order_factory; - - // Infer internal status from persisted external status - switch (self->external_status()) { - - case ORDER_STATUS_WORKING: - case ORDER_STATUS_PENDING_WORKING: - terminate(ORDER_STATUS_ERROR, "System restarted with active algo order"); - return; - - // Terminal states - case ORDER_STATUS_ERROR: - case ORDER_STATUS_REJECTED_BY_SPARK: - case ORDER_STATUS_COMPLETED: - case ORDER_STATUS_CANCELED: - terminate(rec.state().status(), rec.state().last_msg()); - break; - - // It should not be possible to reach these - case ORDER_STATUS_DISCONNECTED: - case ORDER_STATUS_INACTIVE: - case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: - case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: - bts_assert(false); - break; - } -} - -OrderOpResult AlgoOrder::on_activate(Price p, Quantity q, - OrderServices &services) { - ReportStateGuard _(*this); - - bts_assert(!self->services); - self->services = &services; - self->security_db = &services.sec_db(); - self->order_factory = &services.order_factory(); - - switch (self->status) { - - case Status::INACTIVE: { - self->services->subscribe_to_child_updates([this](auto const &u) { - this->on_child_order_update(u.key, u.quick_rebuild_state()); - }); - self->set_external_status(ORDER_STATUS_WORKING); - self->set_price_quantity(p, q); - for (auto &ct : self->child_templates) - ct->builder.activate(*self->services); - if (q.value() == 0) - terminate(ORDER_STATUS_COMPLETED); - else if (is_attaching()) - transition(Status::W_ATTACH); - else - transition(Status::WORKING); - return {}; - } - case Status::COMPLETED: - return {}; - - case Status::W_ATTACH: - case Status::WORKING: - bts_assert(false); - } - - bts_assert(false); - return {}; -} - -OrderOpResult AlgoOrder::on_update(Update const &update) { - ReportStateGuard _(*this); - - switch (self->status) { - case Status::W_ATTACH: - self->pending_update = update; - return {}; - case Status::COMPLETED: - return {}; - case Status::WORKING: { - self->set_price_quantity(Price(update.price()), - Quantity(update.quantity())); - if (update.has_custom()) { - *self->definition.mutable_custom_def() = update.custom(); - self->services->report_definition(self->definition); - } - do_update(update); - return {}; - } - case Status::INACTIVE: - bts_assert(false); - return {}; - } - bts_assert(false); - return {}; -} - -OrderOpResult AlgoOrder::on_update(Price p, Quantity q) { - ReportStateGuard _(*this); - - Update up; - up.set_price(p.value()); - up.set_quantity(q.value()); - return on_update(up); -} - -orderman::OrderOpResult AlgoOrder::on_detach() { - ReportStateGuard _(*this); - - LOG(OrderMan, Info, "AlgoOrder order with key {} received detach", - key().value()); - - switch (self->status) { - case Status::W_ATTACH: - case Status::WORKING: - for (auto &[_, c] : self->children) { - (void)_; - c.order.mark_detached_from_parent(); - } - self->children.clear(); - terminate(ORDER_STATUS_CANCELED); - break; - case Status::COMPLETED: - break; - case Status::INACTIVE: - bts_assert(false); - break; - } - return {}; -} - -orderman::OrderOpResult AlgoOrder::on_cancel() { - ReportStateGuard _(*this); - - switch (self->status) { - case Status::W_ATTACH: - case Status::WORKING: - terminate(ORDER_STATUS_CANCELED); - break; - case Status::COMPLETED: - break; - case Status::INACTIVE: - bts_assert(false); - break; - } - return {}; -} - -void AlgoOrder::on_attach_complete(OrderOpResult r, ChildOrder const &child, - api::orderman::OrderDefinition const &, - api::orderman::OrderState const &state) { - ReportStateGuard _(*this); - - switch (self->status) { - case Status::W_ATTACH: - if (r.ok()) { - bts_assert(state.status() == ORDER_STATUS_WORKING); - - LOG(OrderMan, Info, - "AlgoOrder order with key {} successfully attached to {}", - key().value(), child.key().value()); - - // Store the order - auto &child_record = (self->children)[child.key()]; - child_record.is_subscribing = true; - child_record.order = child; - update_child_record(state, &child_record); - transition(Status::WORKING); - } else { - terminate_error(fmt::format( - "AlgoOrder order with key {} failed to attach to {}: {}", - key().value(), self->attach_target.value(), r.error_msg)); - } - break; - case Status::COMPLETED: - break; - case Status::INACTIVE: - case Status::WORKING: - bts_assert(false); - break; - } -} - -void AlgoOrder::on_child_order_update( - OrderKey child_key, api::orderman::OrderState const &new_child_state) { - ReportStateGuard _(*this); - - switch (self->status) { - case Status::WORKING: - case Status::W_ATTACH: - if (auto it = self->children.find(child_key); - it != self->children.end()) { - auto &child_record = it->second; - switch (new_child_state.status()) { - case ORDER_STATUS_COMPLETED: - case ORDER_STATUS_WORKING: - case ORDER_STATUS_PENDING_WORKING: - case ORDER_STATUS_CANCELED: - if (child_record.state.has_value() && - child_record.state.value().total_fills.value() > - new_child_state.filled_quantity()) { - LOG(OrderMan, Error, - "AlgoOrder {} observed child {} decrease its total fills " - "from {} to {}. This should never happen. Canceling {}", - key().value(), child_key.value(), - child_record.state.value().total_fills.value(), - new_child_state.filled_quantity(), key().value()); - terminate_error("Child order filled quantity decreased"); - } else { - update_child_record(new_child_state, &child_record); - if (child_record.is_subscribing) - on_child_change(child_key, child_record.state.value()); - } - break; - - case ORDER_STATUS_DISCONNECTED: - case ORDER_STATUS_REJECTED_BY_SPARK: - case ORDER_STATUS_ERROR: - terminate_error("Unexpected child order state"); - break; - - case ORDER_STATUS_INACTIVE: - case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: - case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: - bts_assert(false); - break; - } - } else - terminate_error( - fmt::format("Unknown child order update: {}", child_key.value())); - return; - case Status::COMPLETED: - return; - case Status::INACTIVE: - bts_assert(false); - return; - } - bts_assert(false); -} - -void AlgoOrder::setup_transitions() { - // From INACTIVE - self->transition_actions[{Status::INACTIVE, Status::COMPLETED}] = [this]() { - _transition(Status::COMPLETED); - }; - self->transition_actions[{Status::INACTIVE, Status::W_ATTACH}] = [this]() { - self->services->attach( - self->attach_target, - [this](auto result, auto const &child_order, auto const &child_def, - auto const &child_state) { - this->on_attach_complete(result, child_order, child_def, - child_state); - }); - _transition(Status::W_ATTACH); - }; - self->transition_actions[{Status::INACTIVE, Status::WORKING}] = [this]() { - _transition(Status::WORKING); - do_launch(); - }; - - // From W_ATTACH - self->transition_actions[{Status::W_ATTACH, Status::COMPLETED}] = [this]() { - _transition(Status::COMPLETED); - }; - self->transition_actions[{Status::W_ATTACH, Status::WORKING}] = [this]() { - _transition(Status::WORKING); - auto &maybe_state = self->children[self->attach_target].state; - bts_assert(maybe_state.has_value()); - do_launch_attached(self->attach_target, maybe_state.value()); - if (self->status == Status::WORKING && self->pending_update.has_value()) { - auto update = self->pending_update.value(); - self->pending_update.reset(); - do_update(self->pending_update.value()); - } - }; - - // From WORKING - self->transition_actions[{Status::WORKING, Status::COMPLETED}] = [this]() { - cleanup(); - _transition(Status::COMPLETED); - }; -} - -void AlgoOrder::check_invariants() { - bts_assert(self->security_db); - bts_assert(self->order_factory); - - switch (self->status) { - case Status::INACTIVE: - bts_assert(false); - return; - - case Status::W_ATTACH: - bts_assert(self->services); - bts_assert(self->security); - bts_assert(is_attaching()); - bts_assert(self->external_status() == ORDER_STATUS_WORKING); - bts_assert(self->book_subs.empty()); - bts_assert(self->children.empty()); - bts_assert(!self->has_timer_sub); - break; - - case Status::WORKING: - bts_assert(self->services); - bts_assert(self->security); - bts_assert(self->external_status() == ORDER_STATUS_WORKING); - bts_assert(!self->pending_update.has_value()); - break; - - case Status::COMPLETED: - bts_assert(self->children.empty()); - bts_assert(self->external_status() == ORDER_STATUS_CANCELED || - self->external_status() == ORDER_STATUS_COMPLETED || - self->external_status() == ORDER_STATUS_ERROR || - self->external_status() == ORDER_STATUS_REJECTED_BY_SPARK); - bts_assert(self->book_subs.empty()); - bts_assert(!self->has_timer_sub); - bts_assert(!self->pending_update.has_value()); - break; - } -} - -void AlgoOrder::transition(Status new_status) { - if (auto it = self->transition_actions.find({self->status, new_status}); - it != self->transition_actions.end()) { - it->second(); - } else { - LOG(OrderMan, Fatal, "Illegal transition from {} to {}", - state_name(self->status), state_name(new_status)); - bts_assert(false); - } -} - -void AlgoOrder::_transition(Status st) { - LOG(OrderMan, Info, "Transitioninig AlgoOrder with key {} from {} to {}", - key().value(), state_name(self->status), state_name(st)); - self->status = st; - check_invariants(); -} - -void AlgoOrder::terminate(api::orderman::OrderStatus st, - std::string const &error) { - self->set_external_status(st); - if (error.size()) - self->set_last_msg(error); - transition(Status::COMPLETED); -} - -void AlgoOrder::cleanup() { - for (auto const &s : self->book_subs) { - self->services->unsubscribe_from_book(s.second); - self->book_subs.clear(); - } - for (auto &c : self->children) { - c.second.order.cancel(); - self->children.clear(); - } - if (self->has_timer_sub) - unsubscribe_timer(); - self->pending_update.reset(); -} - -void AlgoOrder::update_child_record(api::orderman::OrderState const &new_state, - ChildRecord *child_record) { - if (!child_record->state.has_value()) - child_record->state = ChildState{}; - auto &old_state = child_record->state.value(); - bts_assert(old_state.total_fills.value() <= new_state.filled_quantity()); - - child_record->state = ChildState{ - Price(new_state.price()), - Quantity(new_state.quantity()), - Quantity(new_state.filled_quantity()), - Price(new_state.avg_price()), - Quantity(new_state.filled_quantity() - old_state.total_fills.value()), - Price(new_state.last_price()), - map_child_status(new_state.status())}; -} - -OrderKey AlgoOrder::index_and_launch_child(ChildOrder &&child, Price p, - Quantity q, bool subscribe) { - child.activate(p, q); - auto k = child.key(); - auto &cr = self->children[k]; - cr = {std::move(child), {}, subscribe}; - return k; -} - -bool AlgoOrder::is_working() const { return self->status == Status::WORKING; } - -log::CharPtrWrapper AlgoOrder::state_name(Status st) { - switch (st) { - case Status::COMPLETED: - return {"COMPLETED"}; - case Status::W_ATTACH: - return {"W_ATTACH"}; - case Status::INACTIVE: - return {"INACTIVE"}; - case Status::WORKING: - return {"WORKING"}; - } - bts_assert(false); - return {""}; -} - -ChildStatus AlgoOrder::map_child_status(api::orderman::OrderStatus st) { - switch (st) { - case ORDER_STATUS_COMPLETED: - case ORDER_STATUS_PENDING_WORKING: - return ChildStatus::COMPLETED; - case ORDER_STATUS_WORKING: - return ChildStatus::WORKING; - case ORDER_STATUS_CANCELED: - return ChildStatus::CANCELED; - default: - bts_assert(false); - return ChildStatus::CANCELED; - } -} - -bool AlgoOrder::_is_attaching(api::orderman::OrderDefinition const &def) const { - auto ak = OrderKey(def.attach_target_order_key()); - return (ak != null_order_key() && - !should_forward_attachment(def.custom_def())); -} - -} // namespace orderman -} // namespace bts diff --git a/src/lib/orderman/orders/algo/AlgoOrder.h b/src/lib/orderman/orders/algo/AlgoOrder.h deleted file mode 100644 index 4cc5443eb..000000000 --- a/src/lib/orderman/orders/algo/AlgoOrder.h +++ /dev/null @@ -1,208 +0,0 @@ -#pragma once - -#include "lib/orderman/orders/Order.h" -#include -#include "lib/orderman/orders/algo/IAlgoOrder.h" - -namespace bts { -namespace orderman { - -class Book; -class ChildTemplate; - -class AlgoOrder : private Order { - public: - AlgoOrder(); - ~AlgoOrder(); - - // -------------------------------------------------------------- - // Queries - // -------------------------------------------------------------- - - // Returns the OrderDefinition for this order - api::orderman::OrderDefinition const &definition() const; - - // Returns the key of the order() - OrderKey order_key() const; - - // Returns the parent key of the order() - OrderKey parent_order_key() const; - - // Resolves the security identified by k, returning nullptr if no - // such security exists - api::Security const *lookup_security(SecurityKey k) const; - - // Returns the price for this order. This value is not available - // from do_before_launch() - Price price() const; - - // Returns the quantity for this order. This value is not available - // from do_before_launch(). - Quantity quantity() const; - - // Returns the filled quantity for this order. - Quantity total_fills() const; - - // Returns wheter this order is attaching to an existing order - bool is_attaching() const; - - // -------------------------------------------------------------- - // Commands - // -------------------------------------------------------------- - - // Build a child template that can later can be used to quickly - // launch child orders. The reserve parameter indicates how many - // orders should be created ahead of time, before they are - // launched. This is usefull to remove the relatively expensive - // order creation operation to a background thread. This is the - // only command that can be invoked from the do_before_launch() - // call. - ChildTemplate & - make_child_template(api::orderman::OrderDefinition const &child_def, - size_t reserve = 1); - - // Launch a new child order, based on a previously created template - // at the specified price and quantity. This method can not be - // called from do_before_launch(). By default launching a child - // results in an automatic subscription to its state changes. - OrderKey launch_child(ChildTemplate &, Price, Quantity, - bool subscribe_to_changes = true); - - // Launch a new child order, using the speficied definition, price - // and quantity. This method cannot be called from - // do_before_launch(). By default launching a child results in an - // automatic subscription to its state changes. - OrderKey launch_child(api::orderman::OrderDefinition const &, Price, - Quantity, bool subscribe_to_changes = true); - - // Update previously launched child order. This method cannot be - // called from do_before_launch(). - void update_child(OrderKey child, api::orderman::Update const &); - - // Cancel previously launched child order. This method cannot be - // called from do_before_launch(). - void cancel_child(OrderKey child); - - // Deatch from previously launched child order. This method cannot - // be called from do_before_launch(). - void detach_child(OrderKey child); - - // Update this order's filled quantity and last price. This method - // cannot be called from do_before_launch(). - void update_fill_state(Quantity total_fills, Price last_price); - - // Cancel this order, optionally reporting an explanation. This - // method cannot be called from do_before_launch(). - void cancel(std::string const &msg = ""); - - // Terminate this order, canceling all children, signaling an error - // condition. This method cannot be called from do_before_launch(). - void terminate_error(std::string const &msg); - - // Signal this order completed successfully and cancel any active - // children. This method cannot be called from do_before_launch(). - void complete(); - - // Detach from all children. This method cannot be called from - // do_before_launch(). - void detach_all(); - - // -------------------------------------------------------------- - // Subscriptions - // -------------------------------------------------------------- - - // Subscribe to book updates for the specified security - void subscribe_book(SecurityKey); - void unsubscribe_book(SecurityKey); - - // Subscribe to child order updates for the specified child - // order. This is rarely needed since launch_child() by default - // automatically subscribes to new orders. - void subscribe_child_change(OrderKey); - void unsubscribe_child_change(OrderKey); - - // Subscribe to a periodic timer - void subscribe_timer(); - void unsubscribe_timer(); - - // -------------------------------------------------------------- - // Events - // -------------------------------------------------------------- - - // The child order specified by the order key has changed - virtual void on_child_change(OrderKey, ChildState const &) {} - - // The book for the specified security has changed - virtual void on_book(SecurityKey, Book const &) {} - - // -------------------------------------------------------------- - // Hooks - // -------------------------------------------------------------- - - // Override this to setup and validate an order prior to it being - // launched. This call occurs outside of the latency sensitive path - // so it can be used for any expensive initialization - // operations. Note that at this point the order is not fully - // initialized only a subset of the order functionality is - // available. Note as well that at this point there is no - // guarantee the order will execute at all, since the order might - // simply have been created as part of a pool of orders used to - // speed up order launch. - virtual OrderOpResult do_before_launch() { return {}; } - - // Implement this method to start your order - virtual void do_launch() = 0; - - // Implement this method to start your order as the parent of an - // existing order. Attached child orders are automatically - // subscribed to, so there is no need to call - // subscribe_child_change() for the attached order. - virtual void do_launch_attached(OrderKey /* attached_child */, - ChildState /* attached_child_state */) {} - - // Implement this method to respond to updates to you order. The - // implementation should validate the update and either apply it, - // in case the update is valid or invoke terminate_error() - // otherwise. - virtual void do_update(api::orderman::Update const &) {} - - // Implement this method to receive a periodical call - virtual void on_timer() {} - - private: - friend class OrderFactory; - struct ReportStateGuard; - struct ChildRecord; - enum class Status; - struct PImpl; - std::unique_ptr self; - void on_init_new(api::orderman::OrderDefinition const &, - const PreActivationOrderServices &) override; - void on_init_recovered(api::orderman::OrderRecord const &, - const PreActivationOrderServices &) override; - OrderOpResult on_activate(Price, Quantity, OrderServices &) override; - OrderOpResult on_update(api::orderman::Update const &) override; - OrderOpResult on_update(Price, Quantity) override; - OrderOpResult on_detach() override; - OrderOpResult on_cancel() override; - void on_attach_complete(OrderOpResult, ChildOrder const &, - api::orderman::OrderDefinition const &, - api::orderman::OrderState const &); - void on_child_order_update(OrderKey, api::orderman::OrderState const &); - void setup_transitions(); - void check_invariants(); - void transition(Status); - void _transition(Status); - void terminate(api::orderman::OrderStatus, std::string const &error = ""); - void cleanup(); - void update_child_record(api::orderman::OrderState const &, ChildRecord *); - OrderKey index_and_launch_child(ChildOrder &&child, Price p, Quantity q, - bool subscribe_child_change); - bool is_working() const; - log::CharPtrWrapper state_name(Status); - ChildStatus map_child_status(api::orderman::OrderStatus); - bool _is_attaching(api::orderman::OrderDefinition const &) const; -}; - -} // namespace orderman -} // namespace bts From 99ea02ca40b629e95d2a2c383ea4ebf57bb2cfe0 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 30 Apr 2019 14:57:29 -0400 Subject: [PATCH 098/100] Guarded against the possibilty of algos not supporting attach/detach. --- src/lib/CMakeDeps.cmake | 2 +- .../orderman/orderman_utils/orderman_utils.h | 1 + .../orderman/orders/algo/AlgoWrapperOrder.cpp | 18 +++++++++++++----- src/lib/orderman/orders/algo/IAlgoOrder.cpp | 6 +++++- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index b77880cc0..03ccc1f50 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServices.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/core/IOrderServices.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orderman_utils/orderman_utils.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/algo/AlgoOrder.cpp;orderman/orders/algo/AlgoWrapperOrder.cpp;orderman/orders/algo/IAlgoOrder.cpp;orderman/orders/algo/IAlgoServices.cpp;orderman/orders/algo/IBeforeLaunchAlgoServices.cpp;orderman/orders/algo/algo_orders/CoverAlgo.cpp;orderman/orders/algo/algo_orders/IcebergAlgo.cpp;orderman/orders/algo/algo_orders/JoinAlgo.cpp;orderman/orders/algo/algo_orders/LeanAlgo.cpp;orderman/orders/algo/algo_orders/TopAlgo.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/BookOrderBase.cpp;orderman/orders/book_algos/OCOOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/PricerOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegFillState.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/HedgerManager.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServices.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/core/IOrderServices.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orderman_utils/orderman_utils.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/algo/AlgoWrapperOrder.cpp;orderman/orders/algo/IAlgoOrder.cpp;orderman/orders/algo/IAlgoServices.cpp;orderman/orders/algo/IBeforeLaunchAlgoServices.cpp;orderman/orders/algo/algo_orders/CoverAlgo.cpp;orderman/orders/algo/algo_orders/IcebergAlgo.cpp;orderman/orders/algo/algo_orders/JoinAlgo.cpp;orderman/orders/algo/algo_orders/LeanAlgo.cpp;orderman/orders/algo/algo_orders/TopAlgo.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/BookOrderBase.cpp;orderman/orders/book_algos/OCOOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/PricerOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegFillState.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/HedgerManager.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file diff --git a/src/lib/orderman/orderman_utils/orderman_utils.h b/src/lib/orderman/orderman_utils/orderman_utils.h index 32d5d1939..50b97ce43 100644 --- a/src/lib/orderman/orderman_utils/orderman_utils.h +++ b/src/lib/orderman/orderman_utils/orderman_utils.h @@ -28,6 +28,7 @@ inline bool is_parent_order(const bts::api::orderman::OrderRecord &ord) { } bool should_forward_attachment(const api::orderman::CustomOrderDefinition &def); +bool can_consume_attachment(api::orderman::CustomOrderDefinition::CustomCase) ; } // namespace orderman } // namespace bts diff --git a/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp b/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp index bb884b1de..5522bbcf5 100644 --- a/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp +++ b/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp @@ -129,11 +129,17 @@ void AlgoWrapperOrder::on_init_new(OrderDefinition const &def, me->definition = def; // If we are consuming the attachment we remove it from our definition so - // that derived orders can use our definition to create their childre + // that derived orders can use our definition to create their children // without triggering an attachment. if (_is_attaching(me->definition)) { - me->attach_target = OrderKey(me->definition.attach_target_order_key()); - me->definition.clear_attach_target_order_key(); + if (can_consume_attachment(me->definition.custom_def().custom_case())) { + me->attach_target = OrderKey(me->definition.attach_target_order_key()); + me->definition.clear_attach_target_order_key(); + } + else { + terminate(ORDER_STATUS_REJECTED_BY_SPARK, "Algo does not support attachment"); + return; + } } me->security = serv.sec_db.resolve(SecurityKey(def.security_key())); @@ -574,7 +580,8 @@ void AlgoWrapperOrder::on_child_order_update( } else { update_child_record(new_child_state, &child_record); if (child_record.is_subscribing) - me->algo->on_child_change(child_key, child_record.state.value()); + me->algo->on_child_change(child_key, + child_record.state.value()); } break; @@ -630,7 +637,8 @@ void AlgoWrapperOrder::setup_transitions() { _transition(Status::WORKING); auto &maybe_state = me->children[me->attach_target].state; bts_assert(maybe_state.has_value()); - me->algo->do_launch_attached(*this,me->attach_target, maybe_state.value()); + me->algo->do_launch_attached(*this, me->attach_target, + maybe_state.value()); if (me->status == Status::WORKING && me->pending_update.has_value()) { auto update = me->pending_update.value(); me->pending_update.reset(); diff --git a/src/lib/orderman/orders/algo/IAlgoOrder.cpp b/src/lib/orderman/orders/algo/IAlgoOrder.cpp index c2de7ee7e..11240c91b 100644 --- a/src/lib/orderman/orders/algo/IAlgoOrder.cpp +++ b/src/lib/orderman/orders/algo/IAlgoOrder.cpp @@ -1,4 +1,5 @@ #include "lib/orderman/orders/algo/IAlgoOrder.h" +#include "lib/orderman/orders/algo/IAlgoServices.h" namespace bts { namespace orderman { @@ -8,6 +9,9 @@ OrderOpResult IAlgoOrder::do_before_launch(IBeforeLaunchAlgoServices &) { return {}; } -void IAlgoOrder::do_launch_attached(IAlgoServices &, OrderKey, ChildState) {} +void IAlgoOrder::do_launch_attached(IAlgoServices &serv, OrderKey, ChildState) { + serv.terminate_error("Algo does not implement attachment functionality"); +} + } // namespace orderman } // namespace bts From 0c2943e3da49218128f06aa5b007951a7bde9475 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 30 Apr 2019 16:10:02 -0400 Subject: [PATCH 099/100] Removed obsolete files. --- src/server/sparkd/CMakeDeps.cmake | 2 +- src/server/sparkd/algo/AlgoServiceServer.cpp | 244 ------------------- src/server/sparkd/algo/AlgoServiceServer.h | 54 ---- src/server/sparkd/cmds/Run.cpp | 1 - 4 files changed, 1 insertion(+), 300 deletions(-) delete mode 100644 src/server/sparkd/algo/AlgoServiceServer.cpp delete mode 100644 src/server/sparkd/algo/AlgoServiceServer.h diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index a4aceb09a..e12c685e9 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;algo/AlgoServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IMarginObserver.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/PositionServiceCore.cpp;position/PositionServiceServer.cpp;position/tests/PositionTests.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IMarginObserver.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/PositionServiceCore.cpp;position/PositionServiceServer.cpp;position/tests/PositionTests.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/algo/AlgoServiceServer.cpp b/src/server/sparkd/algo/AlgoServiceServer.cpp deleted file mode 100644 index e444a82a9..000000000 --- a/src/server/sparkd/algo/AlgoServiceServer.cpp +++ /dev/null @@ -1,244 +0,0 @@ -//#include "AlgoServiceServer.h" -//#include "lib/api/service_helpers/AlgoServiceUtils.h" -//#include "lib/api/public/ServiceContext.h" -// -//#include "lib/persistence/SqliteStore.h" -//#include "lib/logging/logging.h" -//#include "lib/utils/bts_assert.h" -//#include "lib/utils/PBUtils.h" -//#include "lib/utils/TimeStamp.h" -// -//#include -//#include -//#include -//#include -//#include -// -//namespace bts { -//namespace orderman { -// -//using namespace std; -//using namespace bts::api::orderman; -// -//const string& algo_key(const CompositeAlgoRecord& row) { -// return row.key(); -//} -//const string& algo_template_key(const AlgoTemplateRecord& row) { -// return row.key(); -//} -// -//struct AlgoServiceServer::PImpl{ -// const std::string template_table_name, -// algo_table_name; -// std::unique_ptr store; -// bts::api::ServiceContext& context; -// AlgoServiceBase::AlgosStatefulPublisher algo_publisher; -// AlgoServiceBase::Algo_templatesStatefulPublisher template_publisher; -// PImpl(bts::api::ServiceContext& context, -// const AlgoServiceBase::AlgosStatefulPublisher& algo_publisher, -// const AlgoServiceBase::Algo_templatesStatefulPublisher& template_publisher) -// : -// template_table_name("templates"), -// algo_table_name("composite_algos"), -// store(new bts::persistence::SqliteStore("algo")), -// context(context), -// algo_publisher(algo_publisher), -// template_publisher(template_publisher){ -// } -//}; -// -//AlgoServiceServer::AlgoServiceServer(bts::api::ServiceContext& context) -//: -//AlgoServiceBase (context), -//me(new PImpl(context, -// bts::api::StatefulPublisher(make_algos_publisher(), algo_key, true), -// bts::api::StatefulPublisher(make_algo_templates_publisher(), algo_template_key, true))) -//{ -// _init_publish_algos(); -// _init_publish_templates(); -//} -//AlgoServiceServer::~AlgoServiceServer(){} -// -//void AlgoServiceServer::create_composite_algo( -// const CompositeAlgoCreateRequest& request, -// const api::Response& response){ -// CompositeAlgoCreateResult result; -// result.set_success(true); -// if (!_has_algo(request.new_algo().name())){ -// CompositeAlgoRecord new_algo_rec; -// new_algo_rec.set_key(request.new_algo().name()); -// new_algo_rec.mutable_common()->CopyFrom(_new_common(request.user_id())); -// new_algo_rec.mutable_algo_definition()->CopyFrom(request.new_algo()); -// if(_store_and_publish_algo(new_algo_rec)) -// result.mutable_record()->CopyFrom(new_algo_rec); -// else{ -// result.set_success(false); -// result.set_err_msg("Could not store algo"); -// } -// } -// else{ -// result.set_success(false); -// result.set_err_msg("Algo with this name already exists"); -// } -// response(result); -//} -// -//void AlgoServiceServer::create_template( -// const TemplateCreateRequest& request, -// const api::Response& response){ -// TemplateCreateResult result; -// result.set_success(true); -// string key = template_key(request.new_template()); -// const AlgoTemplateRecord* ex_rec = _find_template(key); -// if(ex_rec == nullptr){ -// AlgoTemplateRecord new_record; -// new_record.set_key(key); -// new_record.mutable_common()->CopyFrom(_new_common(request.user_id())); -// new_record.mutable_algo_template()->CopyFrom(request.new_template()); -// if(_store_and_publish_template(true,new_record)) -// result.mutable_record()->CopyFrom(new_record); -// else{ -// result.set_success(false); -// result.set_err_msg("Could not store algo"); -// } -// } -// else{ -// result.set_success(false); -// result.set_err_msg("Template already exists"); -// result.mutable_record()->CopyFrom(*ex_rec); -// } -// response(result); -//} -// -//void AlgoServiceServer::update_template( -// const TemplateUpdateRequest& request, -// const api::Response& response){ -// AlgoServiceReply result; -// result.set_success(true); -// const AlgoTemplateRecord* ex_rec = _find_template(request.key()); -// if(ex_rec == nullptr){ -// result.set_success(false); -// result.set_err_msg("Could not locate template"); -// } -// else{ -// AlgoTemplateRecord mrec = AlgoTemplateRecord(*ex_rec); -// mrec.mutable_algo_template()->CopyFrom(request.updated_template()); -// const AlgoTemplateRecord updated_rec = mrec; -// if(!_store_and_publish_template(false,updated_rec)){ -// result.set_success(false); -// result.set_err_msg("Could not serialize template"); -// } -// // TODO: need to make sure -// // a. template's algo is not changed -// // b. template is consistent with the algo -// } -// response(result); -//} -// -//void AlgoServiceServer::remove_template( -// const AlgoServiceRemoveRequest& request, -// const api::Response& response){ -// AlgoServiceReply result; -// result.set_success(true); -// const AlgoTemplateRecord* ex_rec = _find_template(request.key()); -// if(ex_rec == nullptr){ -// result.set_success(false); -// result.set_err_msg("Could not locate template"); -// } -// else{ -// set keys; -// keys.insert(request.key()); -// me->store->remove(me->template_table_name,keys); -// me->template_publisher.erase(request.key()); -// me->template_publisher.flush(); -// } -// response(result); -//} -// -//PersistenceRecordCommon AlgoServiceServer::_new_common(uint32_t user_id){ -// PersistenceRecordCommon rv; -// rv.set_is_active(true); -// rv.set_creator_user_id(user_id); -// return rv; -//} -// -//// algo helper functions -//void AlgoServiceServer::_init_publish_algos(){ -// map> all_data = me->store->retrieve_all(); -// auto table=all_data.find(me->algo_table_name); -// if(table!=all_data.end()){ -// for (const auto& p : table->second){ -// CompositeAlgoRecord algo; -// if(algo.ParseFromString(p.second)) -// me->algo_publisher.add_or_update(algo); -// else -// LOG(Gen, Error, "AlgoServiceServer._load_algos " -// "could not load {}", table->first); -// } -// me->algo_publisher.flush(); -// } -//} -// -//bool AlgoServiceServer::_has_algo(const std::string& key){ -// const auto& algo_map = me->algo_publisher.state_data(); -// return algo_map.find(key) != algo_map.end(); -//} -// -//bool AlgoServiceServer::_store_and_publish_algo ( -// const CompositeAlgoRecord& rec){ -// string ser; -// if(rec.SerializeToString(&ser)){ -// me->store->insert(me->algo_table_name, -// rec.algo_definition().name(), -// ser); -// me->algo_publisher.add_or_update(rec); -// me->algo_publisher.flush(); -// return true; -// } -// else -// return false; -//} -// -//void AlgoServiceServer::_init_publish_templates(){ -// map> all_data = me->store->retrieve_all(); -// auto table=all_data.find(me->template_table_name); -// if(table!=all_data.end()){ -// for (const auto& p : table->second){ -// AlgoTemplateRecord t_record; -// if(t_record.ParseFromString(p.second)) -// me->template_publisher.add_or_update(t_record); -// else -// LOG(Gen, Error, "AlgoServiceServer._load_templates " -// "could not load template {}", table->first); -// } -// me->template_publisher.flush(); -// } -//} -// -//const AlgoTemplateRecord* AlgoServiceServer::_find_template( -// const std::string& key){ -// const auto& template_map = me->template_publisher.state_data(); -// const auto it = template_map.find(key); -// if(it != template_map.end()) -// return &(it->second); -// return nullptr; -//} -// -//bool AlgoServiceServer::_store_and_publish_template ( -// bool insert, -// const AlgoTemplateRecord& rec){ -// string ser; -// if(rec.SerializeToString(&ser)){ -// if(insert) -// me->store->insert(me->template_table_name,rec.key(),ser); -// else -// me->store->update(me->template_table_name,rec.key(),ser); -// me->template_publisher.add_or_update(rec); -// me->template_publisher.flush(); -// return true; -// } -// else -// return false; -//} -// -//}} diff --git a/src/server/sparkd/algo/AlgoServiceServer.h b/src/server/sparkd/algo/AlgoServiceServer.h deleted file mode 100644 index 079a7a36d..000000000 --- a/src/server/sparkd/algo/AlgoServiceServer.h +++ /dev/null @@ -1,54 +0,0 @@ -//#pragma once -// -//#include -//#include "lib/api/services/AlgoService.server.pb.h" -//#include "lib/utils/bts_no_copy.h" -//#include "lib/persistence/IStore.h" -//#include "lib/api/public/ServiceContext.h" -//#include "lib/api/services/OrderService.pb.h" -//#include "lib/api/services/OrderService.client.pb.h" -// -//namespace bts { -//namespace orderman { -// -//class AlgoServiceServer : public api::orderman::AlgoServiceBase -//{ -//private: -// struct PImpl; -// std::unique_ptr me; -//public: -// AlgoServiceServer(bts::api::ServiceContext& context); -// ~AlgoServiceServer(); -// -// void create_composite_algo( -// const bts::api::orderman::CompositeAlgoCreateRequest& request, -// const api::Response& reply) override; -// -// void create_template( -// const bts::api::orderman::TemplateCreateRequest& request, -// const api::Response& reply) override; -// -// void update_template( -// const bts::api::orderman::TemplateUpdateRequest& request, -// const api::Response& reply) override; -// -// void remove_template( -// const bts::api::orderman::AlgoServiceRemoveRequest& request, -// const api::Response& reply) override; -//private: -// bts::api::orderman::PersistenceRecordCommon _new_common(uint32_t user_id); -// // loads algos from the db and publises them -// void _init_publish_algos(); -// bool _has_algo(const std::string& name); -// bool _store_and_publish_algo ( -// const bts::api::orderman::CompositeAlgoRecord& rec); -// // template helpers -// // loads templates from the db and publises them -// void _init_publish_templates(); -// const bts::api::orderman::AlgoTemplateRecord* _find_template( -// const std::string& key); -// bool _store_and_publish_template ( -// bool insert, -// const bts::api::orderman::AlgoTemplateRecord& rec); -//}; -//}} diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index d32604920..de0136eab 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -31,7 +31,6 @@ #include "server/sparkd/support/ShellLatencyServer.h" #include "server/sparkd/user/AdminServiceServer.h" #include "server/sparkd/user/AuthServiceServer.h" -#include "server/sparkd/algo/AlgoServiceServer.h" #include "server/sparkd/algo/AlgoDefinitionServiceServer.h" #include From 48ee5e169947ef1d75fa77aa05d4d57774d323a1 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 30 Apr 2019 17:12:46 -0400 Subject: [PATCH 100/100] advance externals --- externals | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals b/externals index d0e8645b5..67a79263b 160000 --- a/externals +++ b/externals @@ -1 +1 @@ -Subproject commit d0e8645b5ef7380314eddc68114a9f0eaa379318 +Subproject commit 67a79263ba116a7646beb8368cef015530bea027