From be6c9166911f8998e216179a0452b81832b4f61c Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 28 Mar 2019 16:49:59 -0400 Subject: [PATCH 001/131] Allow std::function to move into Dispatcher --- src/lib/utils/Dispatcher.cpp | 2 +- src/lib/utils/Dispatcher.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/utils/Dispatcher.cpp b/src/lib/utils/Dispatcher.cpp index 9c504ee8c..1d7ab64d3 100755 --- a/src/lib/utils/Dispatcher.cpp +++ b/src/lib/utils/Dispatcher.cpp @@ -200,7 +200,7 @@ void Dispatcher::invoke(Action action) } } -void Dispatcher::post(Action action, int64_t delay_ms) +void Dispatcher::post(Action&& action, int64_t delay_ms) { if (delay_ms > 0) { diff --git a/src/lib/utils/Dispatcher.h b/src/lib/utils/Dispatcher.h index 4a4470fb8..2adc656af 100755 --- a/src/lib/utils/Dispatcher.h +++ b/src/lib/utils/Dispatcher.h @@ -87,7 +87,7 @@ public: // Perform an action on the dispatcher thread. Block until the action is complete. void invoke(Action action); // Queue an action on the dispatcher thread and return immediately. - void post(Action action, int64_t delay_ms=0); + void post(Action&& action, int64_t delay_ms=0); boost::asio::io_service* io_service(); From e9da3384024e6ad00fa90f3beb315ab72e117cac Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 28 Mar 2019 17:08:01 -0400 Subject: [PATCH 002/131] optimize IntPrice lessthan --- src/lib/md/IntPrice.h | 10 +++++++--- src/server/sparkd/cmds/Benchmark.cpp | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/lib/md/IntPrice.h b/src/lib/md/IntPrice.h index a66806eb3..191988fe3 100644 --- a/src/lib/md/IntPrice.h +++ b/src/lib/md/IntPrice.h @@ -43,7 +43,9 @@ class IntPrice { return _mantissa == other._mantissa; } else { - return price()==other.price(); + // return price()==other.price(); + auto delta = *this - other; + return delta.mantissa() == 0; } } bool operator!=(const IntPrice& other) const { @@ -54,7 +56,10 @@ class IntPrice { return _mantissa < other._mantissa; } else { - return price() < other.price(); + // avoiding conversion to double is 4-5% faster + // return price() < other.price(); + auto delta = *this - other; + return delta.mantissa() < 0; } } IntPrice operator - (const IntPrice& o) const { @@ -169,7 +174,6 @@ inline md::IntPrice round_to_quote_price(const IntPrice& tick, const IntPrice& p return price; } - inline std::ostream& operator<<(std::ostream& os, const IntPrice& p) { os << p.price(); return os; diff --git a/src/server/sparkd/cmds/Benchmark.cpp b/src/server/sparkd/cmds/Benchmark.cpp index 360f29f89..d62e01f9e 100644 --- a/src/server/sparkd/cmds/Benchmark.cpp +++ b/src/server/sparkd/cmds/Benchmark.cpp @@ -640,6 +640,21 @@ static void perf_log(benchmark::State &state) { } BENCHMARK(perf_log); +void int_price_less(benchmark::State& state) { + for (auto _ : state) { + state.PauseTiming(); + int64_t count = state.range(0); + std::vector prices; + prices.reserve((int)count); + for (int64_t x = 0; x < count; ++x) { + prices.push_back({x % 100, (int8_t)-(x%5)}); + } + state.ResumeTiming(); + std::sort(prices.begin(), prices.end()); + } +} +BENCHMARK(int_price_less)->Range(8, 8 << 11); + std::string Benchmark::help() { return "Run micro-benchmarks. Use --benchmark_filter=regex to run only " "specific benchmarks."; From bad5bc30975a05b042a00e35f1c40f3c044bbd17 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 11:08:37 -0400 Subject: [PATCH 003/131] SPARK-2202 MBOProcessor producer side --- src/lib/config/protos/MarketData.proto | 3 +- src/lib/md/MBOProcessor.cpp | 81 ++++++++++++++++++ src/lib/md/MBOProcessor.h | 82 +++++++++++++++++++ src/lib/utils/ConcurrentPool.h | 55 +++++++++++++ src/server/sparkd/cmds/Run.cpp | 7 +- src/server/sparkd/cmds/Web.cpp | 2 +- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 25 +++++- .../md/platforms/mdp3_onixs/MDP3Listener.h | 27 +++++- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 16 ++-- .../sparkd/md/platforms/mdp3_onixs/MDP3Main.h | 6 +- src/server/sparkd/md/run_md.cpp | 25 +++--- src/server/sparkd/md/run_md.h | 4 +- 12 files changed, 302 insertions(+), 31 deletions(-) create mode 100644 src/lib/md/MBOProcessor.cpp create mode 100644 src/lib/md/MBOProcessor.h create mode 100644 src/lib/utils/ConcurrentPool.h diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index c52377235..3efb46c77 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -113,7 +113,8 @@ message CMEOptions optional bool load_security_db = 33 [default = true]; reserved 34; - // optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED]; + // optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED]; + optional bool maintain_mbo = 35 [default = true]; } optional string onixs_license = 1 [default = "{default}"]; diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp new file mode 100644 index 000000000..4366fb528 --- /dev/null +++ b/src/lib/md/MBOProcessor.cpp @@ -0,0 +1,81 @@ +#include "MBOProcessor.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/ConcurrentPool.h" + +#include +#include + +#include + + +namespace bts { +namespace md { + +struct SecurityKeyHashCompare { + static size_t hash(const SecurityKey& k) { + return k.value(); + } + static bool equal(const SecurityKey& a, const SecurityKey& b) { + return a.value() == b.value(); + } +}; + +typedef tbb::concurrent_hash_map MapType; +struct MBOProcessorImpl { + utils::ConcurrentPool pool; + MapType snapshots; + MBOProcessorImpl() : pool(2048,512), snapshots() {} +}; + +// +// Processor + +MBOProcessor::MBOProcessor() : + _me(std::make_shared()), + _producer(_me), + _consumer(_me) +{} + +MBOWriter& MBOProcessor::writer() { return _producer;} +MBOReader& MBOProcessor::reader() { return _consumer; } + +// +// Writer +MBOWriter::MBOWriter(std::shared_ptr& me): _me(me) { } +MBOWriter::~MBOWriter() { } +snapshot::MBOSnapshot* MBOWriter::allocate_snapshot() { + return _me->pool.allocate(); +} +void MBOWriter::publish_snapshot(SecurityKey key, snapshot::MBOSnapshot* snapshot) { + bts_assert(snapshot != nullptr); + MapType::accessor accessor; + _me->snapshots.insert(accessor, key); + std::swap(accessor->second, snapshot); + if (snapshot != nullptr) { + // Note: relase() should happen inside accessor lifetime + _me->pool.release(&snapshot); + } +} + +// +// Reader + +MBOReader::MBOReader(std::shared_ptr& me): _me(me) { } +MBOReader::~MBOReader() { } +snapshot::MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) { + // get a result eagerly so we don't have to do it while accessor has the entry locked + auto result = _me->pool.allocate(); + MapType::const_accessor accessor; + if (_me->snapshots.find(accessor, key) && accessor->second != nullptr) { + result->copy_from(*accessor->second); + } + else { + release_snapshot(&result); + } + return result; +} +void MBOReader::release_snapshot(snapshot::MBOSnapshot** s) { + _me->pool.release(s); +} + +}} diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h new file mode 100644 index 000000000..854c47a6d --- /dev/null +++ b/src/lib/md/MBOProcessor.h @@ -0,0 +1,82 @@ +#pragma once +#include "lib/domain_model/SecurityKey.h" +#include "lib/domain_model/Security.h" +#include "lib/utils/TimeStamp.h" +#include "lib/utils/ValueWrapper.h" +#include "IntPrice.h" + +#include +#include + +namespace bts { +namespace md { + +using OrderKey = utils::ValueWrapper; + +namespace snapshot { + struct SnapshotOrder { + OrderKey id; + int32_t size; + }; + struct MBOSnapshot { + MBOSnapshot() = default; + MBOSnapshot(const MBOSnapshot&) = delete; + MBOSnapshot(MBOSnapshot&& other) = delete; + + utils::TimeStamp updated; + std::vector bids; + std::vector asks; + + void copy_from(const MBOSnapshot& src) { + bids = src.bids; + asks = src.asks; + updated = src.updated; + } + + void reset() { + updated = utils::TimeStamp(); + bids.clear(); + asks.clear(); + } + }; + +} + +struct MBOProcessorImpl; +class MBOProcessor; + +class MBOWriter { + std::shared_ptr _me; + friend class MBOProcessor; + MBOWriter(std::shared_ptr&); + ~MBOWriter(); + public: + + snapshot::MBOSnapshot* allocate_snapshot(); + + void publish_snapshot(SecurityKey key, snapshot::MBOSnapshot* snapshot); +}; + +class MBOReader { + std::shared_ptr _me; + friend class MBOProcessor; + MBOReader(std::shared_ptr&); + ~MBOReader(); + public: + snapshot::MBOSnapshot* acquire_snapshot(SecurityKey key); + void release_snapshot(snapshot::MBOSnapshot**); +}; + +class MBOProcessor { + std::shared_ptr _me; + MBOWriter _producer; + MBOReader _consumer; + +public: + MBOProcessor(); + MBOWriter& writer(); + MBOReader& reader(); +}; + + +}} diff --git a/src/lib/utils/ConcurrentPool.h b/src/lib/utils/ConcurrentPool.h new file mode 100644 index 000000000..91ad99095 --- /dev/null +++ b/src/lib/utils/ConcurrentPool.h @@ -0,0 +1,55 @@ +#pragma once +#include +#include +#include + +namespace bts { +namespace utils { + +template +class ConcurrentPool { + tbb::concurrent_vector> _all; + tbb::concurrent_queue _free; + size_t _grow_size; + std::atomic _outstanding; + public: + ConcurrentPool(size_t initial_size=256, size_t grow_size=32): + _all(), _free(), _grow_size(grow_size), _outstanding(0) { + reserve(initial_size); + } + + ~ConcurrentPool() { } + + T* allocate() { + T* result = nullptr; + ++_outstanding; + if (_free.try_pop(result)) { + return result; + } + else { + reserve(_grow_size); + return allocate(); + } + } + + void reserve(size_t count) { + std::unique_ptr buffer {new T[count]}; + for (size_t x = 0; x < count; ++x) { + T* temp = &buffer[x]; + release(&temp); + } + _all.push_back(std::move(buffer)); + } + + void release(T** item) { + if (*item) { + (*item)->reset(); + _free.push(*item); + *item = nullptr; + --_outstanding; + } + } + +}; + +}} //namespace diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 2dc515483..3b788c4a2 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -14,6 +14,7 @@ #include "lib/md/MDClient.h" #include "lib/md/SyntheticSecurityService.h" #include "lib/md/relay/QueryServer.h" +#include "lib/md/MBOProcessor.h" #include "lib/metrics/metrics.h" #include "lib/metrics/system_metrics.h" #include "lib/orderman/order_factory/DynamicOrderRegistry.h" @@ -57,10 +58,12 @@ int Run::execute(std::vector const &) { persistence::PersistenceServiceServer settings_saver(sc, "settings"); user::AuthServiceServer login_server(sc); - md::run_market_datapublishers(); + + md::MBOProcessor mbo; + md::run_market_datapublishers(&mbo.writer()); api::synthetic::SyntheticSecurityService synthetic_service(sc); - md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service); + md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service /*, mbo*/); md::QueryServerPtr md_query_server( diff --git a/src/server/sparkd/cmds/Web.cpp b/src/server/sparkd/cmds/Web.cpp index 029ae34bc..abbe80ed6 100644 --- a/src/server/sparkd/cmds/Web.cpp +++ b/src/server/sparkd/cmds/Web.cpp @@ -37,7 +37,7 @@ int Web::execute(const std::vector& args) { user::AuthServiceServer login_server(sc); if (std::find(args.begin(), args.end(), "-md") != args.end()) { - md::run_market_datapublishers(); + md::run_market_datapublishers(nullptr); } api::synthetic::SyntheticSecurityService synthetic_service(sc); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index 6049e982b..f25450a6d 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -15,7 +15,7 @@ namespace md { MDP3Listener::MDP3Listener( CMESecurityIndex& index, MDProducer& producer, - // co2::SettlementRecorder& , + MBOWriter* mbo, const std::vector channels) : _index(index), _producer(producer), @@ -27,7 +27,8 @@ MDP3Listener::MDP3Listener( _symbol_filter(), _channels(channels), _concurrency_monitor(0), - _book_updates() + _book_updates(), + _mbo(mbo) { _book_updates.reserve(32); } @@ -109,7 +110,7 @@ void MDP3Listener::onEndOfPacket(onixs::Handler&, const onixs::PacketArgs&) { } // -// Security discovery +// Book updates void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) { onBookUpdate(sec, book, book_type_direct); @@ -119,6 +120,24 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con onBookUpdate(sec, book, book_type_implied); } +void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::MboBook& book) { + if (_mbo) { + auto data = _index.get_security(sec.id()); + if (data == nullptr) return; + auto key = data->handle.key(); + if (key == invalid_security_key) return; + auto* snapshot = _mbo->allocate_snapshot(); + // copy from onixs book + for (const auto& bid : book.bids()) { + snapshot->bids.push_back({ OrderKey(bid.id()), bid.quantity() }); + } + for (const auto& ask : book.offers()) { + snapshot->asks.push_back({ OrderKey(ask.id()), ask.quantity() }); + } + _mbo->publish_snapshot(SecurityKey(key), snapshot); + } +} + // // Trades diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index fc20add4c..c18a823d7 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -2,6 +2,7 @@ #include "lib/utils/bts_no_copy.h" #include "lib/md/coconut2.h" +#include "lib/md/MBOProcessor.h" #include "CMESecurityIndex.h" #include "lib/tracing/bts_trace.h" @@ -13,11 +14,11 @@ #include #include #include +#include namespace bts { namespace md { - class MDP3Listener : public onixs::HandlerListener, public onixs::MarketDataListener, @@ -31,7 +32,7 @@ class MDP3Listener : public: MDP3Listener(CMESecurityIndex& index, MDProducer&, - // co2::SettlementRecorder&, + MBOWriter* mbo, const std::vector); virtual ~MDP3Listener(); @@ -54,6 +55,25 @@ public: void onPacket(onixs::Handler&, const onixs::PacketArgs&) override; void onEndOfPacket(onixs::Handler&, const onixs::PacketArgs&) override; + // + // Handle MBO incremental updates directly + // void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args& args) override { + // for(const auto& a : args.message().orderIdEntries()) { + // int qty; + // a.displayQty(qty); + // uint8_t ref; + // a.referenceId(ref); + // const auto& ref_entry = args.message().entries()[(size_t)ref]; + // std::cout + // << a.orderId() + // << qty + // << ref_entry.securityId() + // ; + // } + // }; + // void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args&) override {}; + + // // Security discovery template @@ -74,9 +94,9 @@ public: // // Book - // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override; void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override; void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ImpliedBook&) override; + void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override; // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override; template @@ -199,6 +219,7 @@ private: std::pair, std::array> asks; }; std::vector _book_updates; + MBOWriter* _mbo; }; ////////////////////////////////////////// diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 7c0db2832..c711993ac 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -1,6 +1,7 @@ #include "MDP3Main.h" #include "lib/md/cme/MDP3Utils.h" +#include "lib/md/MBOProcessor.h" #include "lib/config/config.h" #include "lib/md/coconut2.h" #include "lib/os/os.h" @@ -130,14 +131,14 @@ onixs::SessionSettings session_settings_for_channel(const string& channel_id) { return settings; } -void MDP3Main::run(std::vector& symbols) { +void MDP3Main::run(std::vector& symbols, MBOWriter* mbo) { await_cme_config(); initialize(); const auto& channels = parse_channels(symbols); _engines = create_feed_engines(channels); for (const auto& channel : channels) { - unique_ptr phandler = prepare_handler(channel.id); - _listeners.push_back(prepare_listener(*phandler, channel)); + unique_ptr phandler = prepare_handler(channel.id, mbo); + _listeners.push_back(prepare_listener(*phandler, channel, mbo)); _handlers.insert(std::make_pair(channel.id, move(phandler))); std::stringstream symstr; for (const auto& sym : channel.symbols) { symstr << sym.symbol() << "-" << SecurityType_Name(sym.type()) << " "; } @@ -196,7 +197,7 @@ void MDP3Main::initialize() { // // // Initialize HandlerSettings and allocate the channel Handler -unique_ptr MDP3Main::prepare_handler(const string& channel_id) { +unique_ptr MDP3Main::prepare_handler(const string& channel_id, MBOWriter* mbo) { const auto& pub_options = config::get(); const auto& options = channel_settings_for(channel_id); LOG(md,Info,"Loaded channel options for channel {}:\n{}",channel_id ,options.DebugString()); @@ -209,6 +210,9 @@ unique_ptr MDP3Main::prepare_handler(const string& channel_id) { settings.bookManagement().directBooks().maintain(true); settings.bookManagement().impliedBooks().maintain(true); settings.bookManagement().consolidatedBooks().maintain(false); + if (options.maintain_mbo() && mbo != nullptr) { + settings.bookManagement().mboBooks().maintain(true); + } // 10k default is *far* too low settings.recoveryQueueMaxSize(options.recovery_queue_size()); @@ -331,14 +335,14 @@ unique_ptr MDP3Main::prepare_handler(const string& channel_id) { // // Create a listener for this channel and register callbacks -unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, const CMEChannel& channel) { +unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, const CMEChannel& channel, MBOWriter* mbo) { // We need to get a handler to the producer channels so we can produce heartbeats vector producer_channels; for (auto& symbol : channel.symbols) { producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } - unique_ptr listener(new MDP3Listener(_index, _producer, /*_settlement_recorder,*/ producer_channels)); + unique_ptr listener(new MDP3Listener(_index, _producer, mbo, producer_channels)); for (auto& symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index 4eb1fbf62..eb321087f 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -15,6 +15,8 @@ namespace md { // void run_onixs_mdp3_server(UserSymbol* sym, size_t symcount); +class MBOWriter; + struct CMEChannel; /* @@ -29,10 +31,10 @@ public: MDP3Main(); ~MDP3Main(); - void run(std::vector&); + void run(std::vector&, MBOWriter*); void initialize(); std::unique_ptr prepare_handler(const std::string& channel); - std::unique_ptr prepare_listener(onixs::Handler& handler, const CMEChannel& channel); + std::unique_ptr prepare_listener(onixs::Handler& handler, const CMEChannel& channel, MBOWriter*); private: BTS_NO_COPY(MDP3Main); diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index 56e2c2762..bf216d7ec 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -20,6 +20,7 @@ #include #include #include +#include using namespace bts; using namespace std; @@ -65,12 +66,12 @@ void run_replay() { replay_recording(file, naggle_ms, fallback_path); } -void run_onixs() { +void run_onixs(MBOWriter* mbo) { // get all glbx symbols std::vector symbols; get_system_md_symbols(api::VENUE_GLBX, api::SECURITYTYPE_UNSET, &symbols); MDP3Main main; - main.run(symbols); + main.run(symbols, mbo); } void run_cleanup() { @@ -78,15 +79,14 @@ void run_cleanup() { } -std::thread* _md_thread; -void run_market_data_publishers_forever() { - os::set_thread_name("md publisher"); +std::optional _md_thread; +void run_market_data_publishers_forever(MBOWriter* mbo) { const auto& conf = config::get(); if (conf.use_replay()) { run_replay(); } else if (conf.use_onixs()) { - run_onixs(); + run_onixs(mbo); } } @@ -110,14 +110,15 @@ void boot_sleep() { } } -void run_market_datapublishers() { - bts_assert(_md_thread == nullptr); +void run_market_datapublishers(MBOWriter* mbo) { + bts_assert(_md_thread == std::nullopt); run_cleanup(); if (!config::get().use_md()) return; // start a thread for the md - _md_thread = new std::thread([]() { - run_market_data_publishers_forever(); - }); + _md_thread = std::optional( + threading::make_thread("md publisher", [mbo]() { + run_market_data_publishers_forever(mbo); + })); if (config::get().use_sim()) { threading::make_thread("ExchangeSim", []() { sim::start_exchange_sim(); }) @@ -327,7 +328,7 @@ void md_tool(std::vector const &args) { else if (cmd == "cleanup") run_cleanup(); else if (cmd == "dump") run_dump(vector(args.begin()+1, args.end())); else if (cmd == "replay") run_replay(); - else if (cmd == "run") run_market_data_publishers_forever(); + else if (cmd == "run") run_market_data_publishers_forever(nullptr); else if (cmd == "relay") relay_everything(); else if (cmd == "test_relay") relay_everything(); else if (cmd == "dumpsecurity" || cmd == "dumpsecurities") dump_db(vector(args.begin()+1, args.end())); diff --git a/src/server/sparkd/md/run_md.h b/src/server/sparkd/md/run_md.h index 587a6e0c1..dc771b0bf 100644 --- a/src/server/sparkd/md/run_md.h +++ b/src/server/sparkd/md/run_md.h @@ -5,7 +5,9 @@ namespace bts { namespace md { -void run_market_datapublishers(); +class MBOWriter; + +void run_market_datapublishers(MBOWriter*); void md_tool(std::vector const& args); }} From ce438e93a63fdef4ff23287156449e442007a101 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 14:00:26 -0400 Subject: [PATCH 004/131] Revert IntPrice regression (unit test failure) --- src/lib/md/IntPrice.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib/md/IntPrice.h b/src/lib/md/IntPrice.h index 191988fe3..1f563548c 100644 --- a/src/lib/md/IntPrice.h +++ b/src/lib/md/IntPrice.h @@ -43,9 +43,7 @@ class IntPrice { return _mantissa == other._mantissa; } else { - // return price()==other.price(); - auto delta = *this - other; - return delta.mantissa() == 0; + return price()==other.price(); } } bool operator!=(const IntPrice& other) const { From 2020221e652ad6ee3552a0e40ce969f895caf35f Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 14:02:29 -0400 Subject: [PATCH 005/131] MBOProducer cleanup and const correctness --- src/lib/md/MBOProcessor.cpp | 14 ++++----- src/lib/md/MBOProcessor.h | 55 ++++++++++++++++------------------ src/lib/utils/ConcurrentPool.h | 2 +- 3 files changed, 34 insertions(+), 37 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 4366fb528..de7fd5363 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -20,9 +20,9 @@ struct SecurityKeyHashCompare { } }; -typedef tbb::concurrent_hash_map MapType; +typedef tbb::concurrent_hash_map MapType; struct MBOProcessorImpl { - utils::ConcurrentPool pool; + utils::ConcurrentPool pool; MapType snapshots; MBOProcessorImpl() : pool(2048,512), snapshots() {} }; @@ -43,16 +43,16 @@ MBOReader& MBOProcessor::reader() { return _consumer; } // Writer MBOWriter::MBOWriter(std::shared_ptr& me): _me(me) { } MBOWriter::~MBOWriter() { } -snapshot::MBOSnapshot* MBOWriter::allocate_snapshot() { +MBOSnapshot* MBOWriter::allocate_snapshot() { return _me->pool.allocate(); } -void MBOWriter::publish_snapshot(SecurityKey key, snapshot::MBOSnapshot* snapshot) { +void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot* snapshot) { bts_assert(snapshot != nullptr); MapType::accessor accessor; _me->snapshots.insert(accessor, key); std::swap(accessor->second, snapshot); if (snapshot != nullptr) { - // Note: relase() should happen inside accessor lifetime + // Note: relase() must happen inside accessor lifetime to ensure correctness _me->pool.release(&snapshot); } } @@ -62,7 +62,7 @@ void MBOWriter::publish_snapshot(SecurityKey key, snapshot::MBOSnapshot* snapsho MBOReader::MBOReader(std::shared_ptr& me): _me(me) { } MBOReader::~MBOReader() { } -snapshot::MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) { +const MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { // get a result eagerly so we don't have to do it while accessor has the entry locked auto result = _me->pool.allocate(); MapType::const_accessor accessor; @@ -74,7 +74,7 @@ snapshot::MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) { } return result; } -void MBOReader::release_snapshot(snapshot::MBOSnapshot** s) { +void MBOReader::release_snapshot(MBOSnapshot**const s) const { _me->pool.release(s); } diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 854c47a6d..0c02f10dc 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -13,34 +13,31 @@ namespace md { using OrderKey = utils::ValueWrapper; -namespace snapshot { - struct SnapshotOrder { - OrderKey id; - int32_t size; - }; - struct MBOSnapshot { - MBOSnapshot() = default; - MBOSnapshot(const MBOSnapshot&) = delete; - MBOSnapshot(MBOSnapshot&& other) = delete; +struct SnapshotOrder { + OrderKey id; + int32_t size; +}; +struct MBOSnapshot { + MBOSnapshot() = default; + MBOSnapshot(const MBOSnapshot&) = delete; + MBOSnapshot(MBOSnapshot&& other) = delete; - utils::TimeStamp updated; - std::vector bids; - std::vector asks; + utils::TimeStamp updated; + std::vector bids; + std::vector asks; - void copy_from(const MBOSnapshot& src) { - bids = src.bids; - asks = src.asks; - updated = src.updated; - } + void copy_from(const MBOSnapshot& src) { + bids = src.bids; + asks = src.asks; + updated = src.updated; + } - void reset() { - updated = utils::TimeStamp(); - bids.clear(); - asks.clear(); - } - }; - -} + void reset() { + updated = utils::TimeStamp(); + bids.clear(); + asks.clear(); + } +}; struct MBOProcessorImpl; class MBOProcessor; @@ -52,9 +49,9 @@ class MBOWriter { ~MBOWriter(); public: - snapshot::MBOSnapshot* allocate_snapshot(); + MBOSnapshot* allocate_snapshot(); - void publish_snapshot(SecurityKey key, snapshot::MBOSnapshot* snapshot); + void publish_snapshot(SecurityKey key, MBOSnapshot* snapshot); }; class MBOReader { @@ -63,8 +60,8 @@ class MBOReader { MBOReader(std::shared_ptr&); ~MBOReader(); public: - snapshot::MBOSnapshot* acquire_snapshot(SecurityKey key); - void release_snapshot(snapshot::MBOSnapshot**); + const MBOSnapshot* acquire_snapshot(SecurityKey key) const; + void release_snapshot(MBOSnapshot**const) const; }; class MBOProcessor { diff --git a/src/lib/utils/ConcurrentPool.h b/src/lib/utils/ConcurrentPool.h index 91ad99095..96802c5dc 100644 --- a/src/lib/utils/ConcurrentPool.h +++ b/src/lib/utils/ConcurrentPool.h @@ -41,7 +41,7 @@ class ConcurrentPool { _all.push_back(std::move(buffer)); } - void release(T** item) { + void release(T**const item) { if (*item) { (*item)->reset(); _free.push(*item); From e1cfa7bf0bbe09172781fbf3f38c1ae4ba64c772 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 14:06:25 -0400 Subject: [PATCH 006/131] Fix build break --- src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index eb321087f..6cb777afe 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -33,7 +33,7 @@ public: void run(std::vector&, MBOWriter*); void initialize(); - std::unique_ptr prepare_handler(const std::string& channel); + std::unique_ptr prepare_handler(const std::string& channel, MBOWriter*); std::unique_ptr prepare_listener(onixs::Handler& handler, const CMEChannel& channel, MBOWriter*); private: From dc1a80afd215686da7833ed02320a141adbadf63 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 14:07:48 -0400 Subject: [PATCH 007/131] Make MBOReader available to MDConsumerHandle --- src/lib/md/Diag/MDPeriodicDumper.cpp | 2 +- src/lib/md/MDClient.cpp | 15 +++++++++----- src/lib/md/MDClient.h | 3 +++ src/lib/md/MDConsumer.cpp | 19 +++++++++++++---- src/lib/md/MDConsumer.h | 2 ++ src/lib/md/MDConsumerHandle.cpp | 5 +++-- src/lib/md/MDConsumerHandle.h | 23 +++++++++++++++++++- src/lib/md/test/Co2TestInstrument.cpp | 2 +- src/lib/md/test/md2_tests.cpp | 30 +++++++++++++-------------- src/server/sparkd/cmds/Run.cpp | 2 +- 10 files changed, 73 insertions(+), 30 deletions(-) diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index 46e5bfbcd..d25ab3d5f 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -160,7 +160,7 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, threading::configure_current_thread(thread_name); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols); + MDConsumer consumer(cache, nullptr, nullptr, &symbols); // read all securities into a vector std::vector entries; diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index 1cb454971..3695640ba 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -3,6 +3,7 @@ #include "SecurityCache.h" #include "SecurityOracle.h" #include "lib/utils/TimeStamp.h" +#include "MBOProcessor.h" #include namespace bts { @@ -15,19 +16,22 @@ struct MDClientImpl{ std::unique_ptr _security_timer; std::unique_ptr _recovery_timer; std::set _symbols; + MBOReader* _mbo; MDConsumer _consumer; utils::Event _event; MDClientImpl( utils::Dispatcher* d, api::ISyntheticSecurityService* s, + MBOReader* mbo, const std::set* symbols): _dispatcher(d), _ss(s), _security_timer(), _recovery_timer(), _symbols(symbols ? *symbols : get_system_md_symbols()), - _consumer(_cache, _ss, &_symbols), + _mbo(mbo), + _consumer(_cache, _ss, mbo, &_symbols), _event() { if (d) { @@ -42,7 +46,7 @@ struct MDClientImpl{ void check_heartbeats() { if (!_consumer.all_heartbeats_ok()) { - MDConsumer temp(_cache,_ss); + MDConsumer temp(_cache,_ss,_mbo,&_symbols); if (temp.all_heartbeats_ok()) { _consumer.swap(temp); LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); @@ -71,11 +75,12 @@ struct MDClientImpl{ MDClient::MDClient( utils::Dispatcher* d, api::ISyntheticSecurityService* s, + MBOReader* mbo, const std::set* symbols) : - _self(new MDClientImpl(d,s, symbols)) + _self(new MDClientImpl(d,s,mbo,symbols)) { } -MDClient::~MDClient() { -} + +MDClient::~MDClient() { } size_t MDClient::scan_securities() { return _self->scan_securities(); } void MDClient::check_heartbeats() { return _self->check_heartbeats(); } diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h index 47925bc88..51b03a777 100644 --- a/src/lib/md/MDClient.h +++ b/src/lib/md/MDClient.h @@ -11,7 +11,9 @@ namespace bts { namespace md { class SecurityOracle; +class MBOReader; struct MDClientImpl; + class MDClient { std::unique_ptr _self; @@ -20,6 +22,7 @@ public: MDClient( utils::Dispatcher* d=nullptr, api::ISyntheticSecurityService* ss=nullptr, + MBOReader* mbo=nullptr, const std::set* s=nullptr); ~MDClient(); diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 20a4f73da..a303c34ad 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -3,6 +3,7 @@ #include "MemorySegmentUtils.h" #include "PlatformLoader.h" #include "SecurityCache.h" +#include "MBOProcessor.h" #include "lib/config/config.h" #include "lib/utils/SharedList.h" #include "lib/utils/WaitFreeQueue.h" @@ -535,6 +536,7 @@ class MDConsumerImpl { Mutex _workmutex; vector> _instances; std::shared_ptr _sci; + MBOReader* _mbo; map> _deferred_handles; vector _callbacks; mutable SecurityCache _cache; @@ -545,10 +547,12 @@ public: const set& symbols, SecurityCache& cache, api::ISyntheticSecurityService* ss, + MBOReader* mbo, bool shared, co2::Platform platform) : _workmutex(), _instances(), _sci(), + _mbo(mbo), _deferred_handles(), _callbacks(), _cache(cache) @@ -625,7 +629,7 @@ public: for (auto& inst : _instances) { if (inst->find_handle(key, &cold, &hot)) { MDConsumerHandle handle; - handle.init(inst->cold_header(), *cold, hot); + handle.init(inst->cold_header(), *cold, hot, _mbo); return handle; } } @@ -657,9 +661,15 @@ public: } // Erase anything that wasn't found from the resulting map + // Set the mbo pointer on anything else for (auto it = result.begin(); it != result.end(); ) { - if (!it->second.is_initialized()) it = result.erase(it); - else ++it; + if (!it->second.is_initialized()) { + it = result.erase(it); + } + else { + it->second.set_mbo(_mbo); + ++it; + } } return result; @@ -910,6 +920,7 @@ public: MDConsumer::MDConsumer( SecurityCache& cache, api::ISyntheticSecurityService* ss, + MBOReader* mbo, const std::set* symbols, bool shared, co2::Platform platform) : _pimpl() { @@ -920,7 +931,7 @@ MDConsumer::MDConsumer( symbols = &all_symbols; } - _pimpl.reset(new MDConsumerImpl(*symbols, cache, ss, shared, platform)); + _pimpl.reset(new MDConsumerImpl(*symbols, cache, ss, mbo, shared, platform)); } MDConsumer::~MDConsumer() {} diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 02c762a8f..103e8bb19 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -26,6 +26,7 @@ class MDConsumerImpl; class MDConsumerInst; class MDConsumerChannel; class SecurityCache; +class MBOReader; class MDConsumer { @@ -33,6 +34,7 @@ public: MDConsumer( SecurityCache&, api::ISyntheticSecurityService* ss, + MBOReader* mbo=nullptr, const std::set* sym = nullptr, bool shared=true, co2::Platform platform=co2::PLATFORM_UNSET); diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index 0218be0eb..616480af4 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -14,7 +14,7 @@ namespace md { MDConsumerHandle::MDConsumerHandle() { memset(this, 0, sizeof(MDConsumerHandle)); _type = HANDLETYPE_VOID; - static_assert(sizeof(MDConsumerHandle) <= 32, ""); + static_assert(sizeof(MDConsumerHandle) <= 40, ""); } MDConsumerHandle::MDConsumerHandle(const std::shared_ptr& def) { memset(this, 0, sizeof(MDConsumerHandle)); @@ -33,9 +33,10 @@ MDConsumerHandle::MDConsumerHandle(const MDConsumerHandle& other) { MDConsumerHandle::~MDConsumerHandle() {} -void MDConsumerHandle::init(const ColdHeader& cheader, const ColdEntry& centry, const void* pvhot) { +void MDConsumerHandle::init(const ColdHeader& cheader, const ColdEntry& centry, const void* pvhot, const MBOReader* mbo) { set_cold(&cheader, ¢ry); set_hot(pvhot); + set_mbo(mbo); } TimeStamp copy_bbo(const HotEntry& hot, PriceSize& out_bid, PriceSize& out_ask) { diff --git a/src/lib/md/MDConsumerHandle.h b/src/lib/md/MDConsumerHandle.h index 38a7772eb..e85b6b93c 100644 --- a/src/lib/md/MDConsumerHandle.h +++ b/src/lib/md/MDConsumerHandle.h @@ -13,6 +13,8 @@ namespace bts { namespace md { class MDConsumerHandle; +class MBOReader; + struct SyntheticStrategy { std::unique_ptr strategy; api::Security security; @@ -141,6 +143,9 @@ TimeStamp optimistic_copy( return t; } +// +// MDConsumerHandle +// // Can be an invalid handle if you request a security which has not been discovered yet. class BTS_EXPORT MDConsumerHandle @@ -151,7 +156,7 @@ public: ~MDConsumerHandle(); MDConsumerHandle(const MDConsumerHandle& other); - void init(const ColdHeader&, const ColdEntry&, const void* hot); + void init(const ColdHeader&, const ColdEntry&, const void* hot, const MBOReader*); bool is_initialized() const { return _type != HANDLETYPE_VOID; } bool is_synthetic() const { return _type == HANDLETYPE_SYNTHETIC; } @@ -269,6 +274,7 @@ public: private: friend class MDConsumerInst; + friend class MDConsumerImpl; template HandleReadResult get_book(Read read, TimeStamp& time, uint32_t copydepth, PriceSize* bids, PriceSize* asks) const { @@ -326,6 +332,19 @@ private: _type = HANDLETYPE_BBO; } } + void set_mbo(const MBOReader* mbo) { + switch(_type) { + case HANDLETYPE_DEPTH: + book.mbo = mbo; + break; + case HANDLETYPE_BBO: + bbo.mbo = mbo; + break; + default: + //ok + break; + } + } enum HandleType : uint8_t { HANDLETYPE_VOID, @@ -337,11 +356,13 @@ private: const HotDepthEntry* hot; const ColdEntry* cdata; const ColdHeader* cheader; + const MBOReader* mbo; }; struct BBOHandle { const HotEntry* hot; const ColdEntry* cdata; const ColdHeader* cheader; + const MBOReader* mbo; }; struct SyntheticHandle { std::shared_ptr def; diff --git a/src/lib/md/test/Co2TestInstrument.cpp b/src/lib/md/test/Co2TestInstrument.cpp index 06d037b1c..0c2c2274a 100644 --- a/src/lib/md/test/Co2TestInstrument.cpp +++ b/src/lib/md/test/Co2TestInstrument.cpp @@ -43,7 +43,7 @@ MDProducerHandle Co2TestInstrument::producer() { MDConsumerHandle Co2TestInstrument::consumer() { MDConsumerHandle handle; - handle.init(*_cheader, *_cold, _hot.get()); + handle.init(*_cheader, *_cold, _hot.get(), nullptr/*mbo*/); return handle; } diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 3a9a5dcdd..d945fea8a 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -370,7 +370,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); vector consumer_securities; // Poll for security list - none visible until they have keys @@ -441,7 +441,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); // Add the on_security callback vector securities; @@ -481,7 +481,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -539,7 +539,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -600,7 +600,7 @@ public: // Symbol list for the consumer auto symbols = get_test_symbols(); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); // Add the on_security callback - callback fires for existing securities vector securities; @@ -628,7 +628,7 @@ public: // Symbol list for the consumer SecurityCache cache; auto symbols = get_test_symbols(); - MDClient client(nullptr, nullptr, &symbols); + MDClient client(nullptr, nullptr, nullptr, &symbols); MDConsumer consumer = client.current_consumer(); map products; consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { @@ -761,7 +761,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); set keys; consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) { keys.insert(security->id()); }); @@ -806,7 +806,7 @@ public: csymbols.insert(UserSymbol("S3", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S4", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &csymbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &csymbols, true, TEST_PLATFORM); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -853,7 +853,7 @@ public: csymbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); csymbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &csymbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &csymbols, true, TEST_PLATFORM); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -905,7 +905,7 @@ public: consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); consumer_symbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &consumer_symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &consumer_symbols, true, TEST_PLATFORM); size_t security_count = 0; // Indirection for check function @@ -974,7 +974,7 @@ public: set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &consumer_symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &consumer_symbols, true, TEST_PLATFORM); auto h1 = consumer.get_handle(k1); REQUIRE(h1.is_initialized()); auto h2 = consumer.get_handle(k2); @@ -1260,7 +1260,7 @@ struct Co2RelayTests { UserSymbol option_symbol("FOOX", VENUE_BTS, SECURITYTYPE_OPTION); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols); + MDConsumer consumer(cache, nullptr, nullptr, &symbols); return consumer; } @@ -1327,7 +1327,7 @@ struct Co2RelayTests { // todo: setup_query_server should use client.symbols() instead of it's own list // const auto symbols = setup_symbols(); - MDClient client(nullptr, nullptr, &symbols); + MDClient client(nullptr, nullptr, nullptr, &symbols); auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices @@ -1460,7 +1460,7 @@ struct Co2RelayTests { vector producers; Entitlements entitlements; const auto symbols = setup_symbols(); - MDClient client(nullptr, nullptr, &symbols); + MDClient client(nullptr, nullptr, nullptr, &symbols); auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices to all levels @@ -1638,7 +1638,7 @@ namespace ImpliedTests { UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols); + MDConsumer consumer(cache, nullptr, nullptr, &symbols); vector consumer_securities; // Poll for security list - none visible until they have keys size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 3b788c4a2..3dd41a86c 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -63,7 +63,7 @@ int Run::execute(std::vector const &) { md::run_market_datapublishers(&mbo.writer()); api::synthetic::SyntheticSecurityService synthetic_service(sc); - md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service /*, mbo*/); + md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service, &mbo.reader()); md::QueryServerPtr md_query_server( From a4575e3fa315503176916e39510e63603c6bb5b9 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 15:38:26 -0400 Subject: [PATCH 008/131] Fix windows build --- src/lib/md/MDConsumerInterop.cpp | 2 +- src/lib/md/test/Co2ManagedTestHelper.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/md/MDConsumerInterop.cpp b/src/lib/md/MDConsumerInterop.cpp index 5027f3d75..17a36b072 100644 --- a/src/lib/md/MDConsumerInterop.cpp +++ b/src/lib/md/MDConsumerInterop.cpp @@ -32,7 +32,7 @@ CAPI MDConsumer* create_md_consumer(BlitUserSymbol* usersymbols, uint32_t count, for (size_t x = 0; x < count; ++x) { symbols.insert(UserSymbol(usersymbols[x].symbol, usersymbols[x].exchange, usersymbols[x].type)); } - return new MDConsumer(cache, nullptr/*synthetics*/, &symbols, shared); + return new MDConsumer(cache, nullptr/*synthetics*/, nullptr /*mbo*/, &symbols, shared); } CAPI void destroy_md_consumer(MDConsumer* consumer) { diff --git a/src/lib/md/test/Co2ManagedTestHelper.cpp b/src/lib/md/test/Co2ManagedTestHelper.cpp index 4d69543ac..30caf3bde 100644 --- a/src/lib/md/test/Co2ManagedTestHelper.cpp +++ b/src/lib/md/test/Co2ManagedTestHelper.cpp @@ -26,7 +26,7 @@ extern "C" BTS_EXPORT void co2_get_1234_handle(size_t& size, MDConsumerHandle& auto pheader = reinterpret_cast(&buffer[0]); pheader->has_depth = false; size = sizeof(MDConsumerHandle); - handle.init(*pheader, *pcold, phot.get()); + handle.init(*pheader, *pcold, phot.get(), nullptr); // Copy the addresses for comparison *hotaddr = (uintptr_t)phot.get(); *coldaddr = (uintptr_t)pcold.get(); From ad6fc6d23b66da40ed5db9f89be2e455e41dbf26 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 15:38:54 -0400 Subject: [PATCH 009/131] Cannot have a const pointer returned to the pool --- src/lib/md/MBOProcessor.cpp | 4 ++-- src/lib/md/MBOProcessor.h | 4 ++-- src/lib/utils/ConcurrentPool.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index de7fd5363..4f89023a6 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -62,7 +62,7 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot* snapshot) { MBOReader::MBOReader(std::shared_ptr& me): _me(me) { } MBOReader::~MBOReader() { } -const MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { +MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { // get a result eagerly so we don't have to do it while accessor has the entry locked auto result = _me->pool.allocate(); MapType::const_accessor accessor; @@ -74,7 +74,7 @@ const MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { } return result; } -void MBOReader::release_snapshot(MBOSnapshot**const s) const { +void MBOReader::release_snapshot(MBOSnapshot**s) const { _me->pool.release(s); } diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 0c02f10dc..04270e8bd 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -60,8 +60,8 @@ class MBOReader { MBOReader(std::shared_ptr&); ~MBOReader(); public: - const MBOSnapshot* acquire_snapshot(SecurityKey key) const; - void release_snapshot(MBOSnapshot**const) const; + MBOSnapshot* acquire_snapshot(SecurityKey key) const; + void release_snapshot(MBOSnapshot**) const; }; class MBOProcessor { diff --git a/src/lib/utils/ConcurrentPool.h b/src/lib/utils/ConcurrentPool.h index 96802c5dc..91ad99095 100644 --- a/src/lib/utils/ConcurrentPool.h +++ b/src/lib/utils/ConcurrentPool.h @@ -41,7 +41,7 @@ class ConcurrentPool { _all.push_back(std::move(buffer)); } - void release(T**const item) { + void release(T** item) { if (*item) { (*item)->reset(); _free.push(*item); From 69f9f2791e0479577eae13bf7605f9a717c74501 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 29 Mar 2019 15:39:22 -0400 Subject: [PATCH 010/131] SPARK-2203 --- src/lib/md/MDConsumerHandle.cpp | 11 +++++++++++ src/lib/md/MDConsumerHandle.h | 3 +++ 2 files changed, 14 insertions(+) diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index 616480af4..af2729ac2 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -1,5 +1,6 @@ #ifndef NO_CPP0X #include "MDConsumerHandle.h" +#include "MBOProcessor.h" #include "lib/domain_model/Security.h" #include @@ -372,6 +373,16 @@ CoconutSecurityStatus MDConsumerHandle::status() const { never; } +std::unique_ptr> MDConsumerHandle::get_mbo() { + if (_type == HANDLETYPE_DEPTH) { + MBOSnapshot* snapshot = book.mbo->acquire_snapshot(SecurityKey(key())); + if (snapshot != nullptr) { + return {snapshot,[mbo=book.mbo](auto* s) { mbo->release_snapshot(&s); }}; + } + } + return nullptr; + } + }}//namespace diff --git a/src/lib/md/MDConsumerHandle.h b/src/lib/md/MDConsumerHandle.h index e85b6b93c..72c387fe8 100644 --- a/src/lib/md/MDConsumerHandle.h +++ b/src/lib/md/MDConsumerHandle.h @@ -14,6 +14,7 @@ namespace md { class MDConsumerHandle; class MBOReader; +struct MBOSnapshot; struct SyntheticStrategy { std::unique_ptr strategy; @@ -272,6 +273,8 @@ public: return *this; } + std::unique_ptr> get_mbo(); + private: friend class MDConsumerInst; friend class MDConsumerImpl; From f513f7d9f2561825f51dd44657a10829b331a4f3 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 1 Apr 2019 12:11:03 -0400 Subject: [PATCH 011/131] SPARK-2204 mbo api in QueryServer --- src/lib/api/services/MarketDataRelay.proto | 16 ++++- src/lib/md/MDConsumerHandle.cpp | 2 +- src/lib/md/MDConsumerHandle.h | 2 +- src/lib/md/relay/QueryServer.cpp | 73 ++++++++++++++++++++++ 4 files changed, 89 insertions(+), 4 deletions(-) diff --git a/src/lib/api/services/MarketDataRelay.proto b/src/lib/api/services/MarketDataRelay.proto index d83daf33b..d5b949673 100644 --- a/src/lib/api/services/MarketDataRelay.proto +++ b/src/lib/api/services/MarketDataRelay.proto @@ -17,17 +17,29 @@ message ExpirationNameList { message ExpirationNameListReply { repeated ExpirationNameList securities = 2; } - message SecurityKeyList { repeated uint32 keys = 1; } +message OrderQuery { + uint32 security_id = 1; + uint32 order_id = 2; +} +message PIQRequest { + repeated OrderQuery orders = 1; +} +message PIQResponse { + // order id => position + map position = 1; +} + service MarketDataRelay { option (bts.api.service_description)="Poll the market data service for updates"; rpc requestmd(bts.co2.MDRequest) returns (bts.co2.MDReply); rpc request_products(bts.api.VoidMessage) returns (ProductListReply); rpc request_names(bts.api.VoidMessage) returns (ExpirationNameListReply); - rpc get_securities(SecurityKeyList) returns (bts.api.SecurityList); + + rpc get_piq(PIQRequest) returns (PIQResponse); } diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index af2729ac2..40efeded5 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -373,7 +373,7 @@ CoconutSecurityStatus MDConsumerHandle::status() const { never; } -std::unique_ptr> MDConsumerHandle::get_mbo() { +std::unique_ptr> MDConsumerHandle::get_mbo() const { if (_type == HANDLETYPE_DEPTH) { MBOSnapshot* snapshot = book.mbo->acquire_snapshot(SecurityKey(key())); if (snapshot != nullptr) { diff --git a/src/lib/md/MDConsumerHandle.h b/src/lib/md/MDConsumerHandle.h index 72c387fe8..506e64f7b 100644 --- a/src/lib/md/MDConsumerHandle.h +++ b/src/lib/md/MDConsumerHandle.h @@ -273,7 +273,7 @@ public: return *this; } - std::unique_ptr> get_mbo(); + std::unique_ptr> get_mbo() const; private: friend class MDConsumerInst; diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index e8331308b..76771c20c 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -9,6 +9,7 @@ #include "../MDConsumer.h" #include "../MDClient.h" #include "../PlatformLoader.h" +#include "../MBOProcessor.h" #include "lib/logging/logging.h" @@ -129,6 +130,27 @@ struct MDConsumerGroup { }; +class HandleIndex { + mutable std::mutex _mutex; + std::unordered_map _index; +public: + template + auto lock(CB cb) { + std::lock_guard guard(_mutex); + return cb(_index); + } + template + auto lock(CB cb) const { + std::lock_guard guard(_mutex); + return cb(_index); + } + void clear() { + lock([](std::unordered_map& index) { + index.clear(); + }); + } +}; + void convert(PriceSize ps, PriceSizeMessage* pps) { pps->set_mantissa(ps.mantissa()); @@ -168,6 +190,7 @@ class QueryServerImpl : public IQueryServer { MDConsumer _consumer; int _sub; boost::container::flat_map> _groups; + HandleIndex _handle_index; public: QueryServerImpl(MDClient& client, metrics::MetricsContainer& stats) : @@ -198,6 +221,10 @@ public: const auto& pgroup = iter->second; bts_assert(pgroup); pgroup->add_security(handle); + // Also insert into handle index + _handle_index.lock([handle](auto& index) { + index[SecurityKey(handle.key())] = handle; + }); } }); } @@ -211,6 +238,7 @@ public: for (auto& pair : _groups) { pair.second->set_consumer(_consumer); } + _handle_index.clear(); subscribe_securities(); utils::Dispatcher::background().post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000); } @@ -392,6 +420,42 @@ public: } } + void get_piq(const PIQRequest& req, PIQResponse* result) const { + // for each pair in request + // look up the mbo + // search for the order in the mbo and add it to the result if found + for(const auto& entry : req.orders()) { + auto id = SecurityKey(entry.security_id()); + auto handle = _handle_index.lock( + [id](auto& index){ + auto iter = index.find(id); + if (iter != index.end()) { + return iter->second; + } + return MDConsumerHandle(); + }); + if (handle.is_initialized()) { + // get mbo + auto mbo = handle.get_mbo(); + // search for order + IntPrice price = {}; + size_t position = 0; + // guess at side + for (size_t x = 0; x < mbo->bids.size(); ++x) { + // append to result + auto& oo = (*mbo)[x]; + // note: needs price + + // if (oo. + } + for (size_t x = 0; x < mbo->asks.size(); ++x) { + // append to result + } + (void)result; + } + } + } + }; struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { @@ -487,6 +551,15 @@ struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { reply(*preply); }); } + + virtual void get_piq(const api::AuthToken&, const PIQRequest& request, const std::function& reply) override { + _threadpool.post([=]() { + google::protobuf::Arena arena; + PIQResponse* presponse = google::protobuf::Arena::CreateMessage(&arena); + _db.get_piq(request, presponse); + reply(*presponse); + }); + } }; struct QueryServer { From 086e90acbe8226a58afab9b068448952cd263d7d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 2 Apr 2019 11:13:28 -0400 Subject: [PATCH 012/131] Move queryserver to linux only --- src/lib/md/test/md2_tests.cpp | 34 ++++--------------------- src/lib/md/test/md_test_utils.cpp | 41 +++++++++++++++++++++++++++++++ src/lib/md/test/md_test_utils.h | 2 ++ 3 files changed, 48 insertions(+), 29 deletions(-) create mode 100644 src/lib/md/test/md_test_utils.cpp diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index d945fea8a..d9da74ddd 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -1,3 +1,4 @@ +#ifndef _WIN32 #include "catch/catch.hpp" #include "../SettlementRecorder.h" #include "../coconut2.h" @@ -15,6 +16,7 @@ #include "lib/config/protos/MarketData.pb.h" #include "lib/utils/Sqlite_utils.h" +#include "md_test_utils.h" #include #include @@ -55,29 +57,8 @@ Platform TEST_PLATFORM = PLATFORM_SIM; // // Helper functions for Producer/Consumer tests // -vector create_test_securities(MDProducer& producer, const char* postfix ="") { - FixedFutureSecurity ffs; - FixedOptionSecurity fos; - ffs.symbol("FOO"+string(postfix)); - fos.symbol("FOOX"+string(postfix)); - ffs.exchange = VENUE_BTS; - fos.exchange = VENUE_BTS; - ffs.key = invalid_security_key; - fos.key = invalid_security_key; - ffs.expiration_year = fos.expiration_year = 2015; - ffs.expiration_day = fos.expiration_day = 27; - fos.strike = 1000; - fos.call_put = PUTORCALL_CALL; - // Add 12 expirations to 2 different shared memory regions - vector result; - for (uint32_t x = 1; x <= 12; ++x) { - ffs.expiration_month = fos.expiration_month = x; - result.push_back(producer.add_security(ffs)); - result.push_back(producer.add_security(fos)); - // In a real app these securities would now be passed to the SecurityIDRecorder - } - return result; -} + +std::vector create_test_securities(MDProducer& producer, const char* postfix = ""); set get_test_symbols(const char* postfix="") { set symbols; @@ -146,12 +127,6 @@ Security mkspread(string symbol, security_key key, Security leg1, Security leg2, return mkspread(symbol, key, vlegs); } -void assign_ids(vector::iterator begin, vector::iterator end, security_key start_key = 1000) { - for (; begin != end; ++begin) { - (*begin).set_key(++start_key); - } -} - class Co2Tests { @@ -2417,3 +2392,4 @@ TEST_CASE("IntPrice", "[md]") { }}} +#endif \ No newline at end of file diff --git a/src/lib/md/test/md_test_utils.cpp b/src/lib/md/test/md_test_utils.cpp new file mode 100644 index 000000000..bc2848c16 --- /dev/null +++ b/src/lib/md/test/md_test_utils.cpp @@ -0,0 +1,41 @@ +#include "md_test_utils.h" +#include "../MDProducer.h" + +#include +#include + +namespace bts { +namespace md { +namespace test { + +std::vector create_test_securities(MDProducer& producer, const char* postfix ="") { + FixedFutureSecurity ffs; + FixedOptionSecurity fos; + ffs.symbol("FOO"+std::string(postfix)); + fos.symbol("FOOX"+std::string(postfix)); + ffs.exchange = api::VENUE_BTS; + fos.exchange = api::VENUE_BTS; + ffs.key = invalid_security_key; + fos.key = invalid_security_key; + ffs.expiration_year = fos.expiration_year = 2015; + ffs.expiration_day = fos.expiration_day = 27; + fos.strike = 1000; + fos.call_put = api::PUTORCALL_CALL; + // Add 12 expirations to 2 different shared memory regions + std::vector result; + for (uint32_t x = 1; x <= 12; ++x) { + ffs.expiration_month = fos.expiration_month = x; + result.push_back(producer.add_security(ffs)); + result.push_back(producer.add_security(fos)); + // In a real app these securities would now be passed to the SecurityIDRecorder + } + return result; +} + +void assign_ids(std::vector::iterator begin, std::vector::iterator end, security_key start_key) { + for (; begin != end; ++begin) { + (*begin).set_key(++start_key); + } +} + +}}}//namespace \ No newline at end of file diff --git a/src/lib/md/test/md_test_utils.h b/src/lib/md/test/md_test_utils.h index 23a82070f..7eccddc3c 100644 --- a/src/lib/md/test/md_test_utils.h +++ b/src/lib/md/test/md_test_utils.h @@ -2,6 +2,7 @@ #include "lib/md/IntPrice.h" #include "lib/md/PriceSize.h" +#include "../MDProducerHandle.h" #include namespace bts { @@ -28,5 +29,6 @@ Book compute_synthetic_book(const api::Security& security, const std::vector::iterator begin, std::vector::iterator end, security_key start_key = 1000); }}}//namespace \ No newline at end of file From df14522433c66bdbed5cde15b0fd33b9ea9f4c84 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 2 Apr 2019 13:07:42 -0400 Subject: [PATCH 013/131] SPARK-2204 QueryServer api --- src/lib/CMakeDeps_win.cmake | 2 +- src/lib/md/MBOProcessor.h | 1 + src/lib/md/relay/QueryServer.cpp | 47 +++++++++++++------ .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 4 +- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/lib/CMakeDeps_win.cmake b/src/lib/CMakeDeps_win.cmake index 879f19a67..18c633341 100644 --- a/src/lib/CMakeDeps_win.cmake +++ b/src/lib/CMakeDeps_win.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 04270e8bd..bbb8c7e21 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -15,6 +15,7 @@ using OrderKey = utils::ValueWrapper; struct SnapshotOrder { OrderKey id; + IntPrice price; int32_t size; }; struct MBOSnapshot { diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 76771c20c..816f55e78 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -1,4 +1,5 @@ #ifndef NO_CPP0X +#ifndef _WIN32 #include "QueryServer.h" #include "lib/config/config.h" #include "lib/utils/Dispatcher.h" @@ -31,6 +32,7 @@ #include #include #include +#include #include #include @@ -420,15 +422,36 @@ public: } } + static std::optional search_orders(const OrderKey oid, const std::vector& orders) { + IntPrice price = {}; + size_t position = 0; + for (size_t x = 0; x < orders.size(); ++x) { + const auto& oo = orders[x]; + if (oo.id == oid) { + return { position }; + } + else if (price == oo.price) { + position += 1; + } + else { + price = oo.price; + position = 0; + } + } + return std::nullopt; + } + void get_piq(const PIQRequest& req, PIQResponse* result) const { // for each pair in request // look up the mbo // search for the order in the mbo and add it to the result if found + auto& rmap = *result->mutable_position(); for(const auto& entry : req.orders()) { - auto id = SecurityKey(entry.security_id()); + auto order_key = OrderKey(entry.order_id()); + auto security = SecurityKey(entry.security_id()); auto handle = _handle_index.lock( - [id](auto& index){ - auto iter = index.find(id); + [security](auto& index){ + auto iter = index.find(security); if (iter != index.end()) { return iter->second; } @@ -438,20 +461,13 @@ public: // get mbo auto mbo = handle.get_mbo(); // search for order - IntPrice price = {}; - size_t position = 0; - // guess at side - for (size_t x = 0; x < mbo->bids.size(); ++x) { - // append to result - auto& oo = (*mbo)[x]; - // note: needs price - - // if (oo. + auto search_result = search_orders(order_key, mbo->bids); + if (!search_result) { + search_result = search_orders(order_key, mbo->asks); } - for (size_t x = 0; x < mbo->asks.size(); ++x) { - // append to result + if (search_result) { + rmap[order_key.value()] = search_result.value(); } - (void)result; } } } @@ -628,3 +644,4 @@ std::unique_ptr create_query_server(MDClient& mdc, metrics::Metric }}//namespace #endif +#endif diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index f25450a6d..bd943250f 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -129,10 +129,10 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con auto* snapshot = _mbo->allocate_snapshot(); // copy from onixs book for (const auto& bid : book.bids()) { - snapshot->bids.push_back({ OrderKey(bid.id()), bid.quantity() }); + snapshot->bids.push_back({ OrderKey(bid.id()), convert_price(bid.price(), 0), bid.quantity() }); } for (const auto& ask : book.offers()) { - snapshot->asks.push_back({ OrderKey(ask.id()), ask.quantity() }); + snapshot->asks.push_back({ OrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); } _mbo->publish_snapshot(SecurityKey(key), snapshot); } From 8793c6c527744f4aa4e4ea94a2c47b5357743ece Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 2 Apr 2019 15:00:02 -0400 Subject: [PATCH 014/131] Replace ManyConsole with CommandLineParser --- src/client/sparkcmd/Cmd/ConfigCheckCommand.cs | 21 +++--- src/client/sparkcmd/Cmd/ConsoleCommandEx.cs | 20 ------ .../sparkcmd/Cmd/ScriptClientCommand.cs | 33 +++++---- src/client/sparkcmd/Cmd/ServiceTail.cs | 34 ++++----- src/client/sparkcmd/Cmd/ServiceTestCommand.cs | 29 ++++---- .../sparkcmd/Cmd/SettingsManagerCommand.cs | 11 ++- src/client/sparkcmd/Cmd/TestCommand.cs | 71 +++++++++++-------- src/client/sparkcmd/Program.cs | 29 ++++++-- src/client/sparkcmd/sparkcmd.csproj | 7 +- 9 files changed, 139 insertions(+), 116 deletions(-) delete mode 100644 src/client/sparkcmd/Cmd/ConsoleCommandEx.cs diff --git a/src/client/sparkcmd/Cmd/ConfigCheckCommand.cs b/src/client/sparkcmd/Cmd/ConfigCheckCommand.cs index 432fce36b..7284e48db 100644 --- a/src/client/sparkcmd/Cmd/ConfigCheckCommand.cs +++ b/src/client/sparkcmd/Cmd/ConfigCheckCommand.cs @@ -1,14 +1,19 @@ -using ManyConsole; using System.Linq; +using System; +using System.Collections.Generic; +using CommandLine; namespace bts { - public class Configcheck : ConsoleCommand { - public Configcheck() { - IsCommand("configcheck", "Ye olde Configcheck"); - HasAdditionalArguments(null, " [Keys to check]"); - } - public override int Run(string[] args) { - mbts.config.Configuration.configcheck(args.Length, args.ToArray()); + + [Verb("configcheck", HelpText="Ye olde Configcheck")] + public class ConfigcheckOptions { + [Value(0, MetaName="keys", HelpText="Keys to print")] + public IEnumerable keys { get; set; } + } + public class Configcheck { + public static int Run(ConfigcheckOptions options) { + var keys = options.keys.ToArray(); + mbts.config.Configuration.configcheck(keys.Length, keys); return 0; } } diff --git a/src/client/sparkcmd/Cmd/ConsoleCommandEx.cs b/src/client/sparkcmd/Cmd/ConsoleCommandEx.cs deleted file mode 100644 index 18c73075d..000000000 --- a/src/client/sparkcmd/Cmd/ConsoleCommandEx.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ManyConsole { - public static class ConsoleCommandEx { - public static string get_argument(this ConsoleCommand _, string arg_name) { - var arg = Environment.GetCommandLineArgs().FirstOrDefault(str => str.StartsWith("--"+arg_name)); - if (arg != null) { - var parts = arg.Split(new[] { '=' }, 2); - if (parts.Length > 1) { - return parts[1]; - } - } - return ""; - } - } -} diff --git a/src/client/sparkcmd/Cmd/ScriptClientCommand.cs b/src/client/sparkcmd/Cmd/ScriptClientCommand.cs index 18f0c18c9..5f00cf392 100644 --- a/src/client/sparkcmd/Cmd/ScriptClientCommand.cs +++ b/src/client/sparkcmd/Cmd/ScriptClientCommand.cs @@ -6,32 +6,37 @@ using System.Reflection; using System.Threading.Tasks; using System.Reactive.Linq; using System.Reactive; -using ManyConsole; using System.Linq; +using CommandLine; namespace sparkcmd { - public class ScriptClientCommand : ConsoleCommand { + [Verb("script", HelpText="Remote control for shell diagnostic host")] + public class ScriptClientOptions { + [Option('s', "server", HelpText="Run a script server for testing.")] + public bool server { get; set; } + + [Option('h', "host", HelpText="Connect to {host}")] + public string host { get; set; } + + [Option('f', "fromfile", HelpText="Read commands from file instead of stdin")] + public string file { get; set; } + + } + public class ScriptClientCommand { bool _server = false; string _host; string _logfile; - public ScriptClientCommand() { - IsCommand("script", "Remote control for shell diagnostic host"); - HasOption("s|server", "Run a script server for testing.", x => _server = true); - HasOption("h|host", "Connect to {host}", h => _host = h); - HasOption("f|fromfile", "Read commands from file instead of stdin", f => _logfile = f); - } - public override int Run(string[] args) { - if (_server) { + public static int Run(ScriptClientOptions o) { + if (o.server) { ScriptClient.runserver(); } else { - var host = string.IsNullOrEmpty(_host) ? args.First() : _host; - if (string.IsNullOrEmpty(_logfile) && File.Exists(_logfile)) { - ScriptClient.start(host, File.OpenRead(_logfile)); + if (string.IsNullOrEmpty(o.file) && File.Exists(o.file)) { + ScriptClient.start(o.host, File.OpenRead(o.file)); } else { - ScriptClient.start(host, Console.OpenStandardInput()); + ScriptClient.start(o.host, Console.OpenStandardInput()); } } return 0; diff --git a/src/client/sparkcmd/Cmd/ServiceTail.cs b/src/client/sparkcmd/Cmd/ServiceTail.cs index 7ebbb0c92..926bc5a6c 100644 --- a/src/client/sparkcmd/Cmd/ServiceTail.cs +++ b/src/client/sparkcmd/Cmd/ServiceTail.cs @@ -1,6 +1,6 @@ using bts.api.core; using bts.test; -using ManyConsole; +using CommandLine; using System; using System.Collections.Generic; using System.Linq; @@ -8,24 +8,26 @@ using System.Text; using System.Threading.Tasks; namespace bts { - class TailService : ConsoleCommand { + class TailServiceOptions { + [Option('m', "method", Required=true, HelpText = "'[ServiceName].[Method]' to call")] + public string method { get;set; } + + [Option('t', "time", HelpText="Include timestamp")] + public bool time { get; set; } + + [Option('p', "path", HelpText="Path inside the row to use for display. Ex: 'Id' or 'Name' ")] + public string path { get; set; } + + [Option('a', "all", HelpText="Include 'all' rows on every print")] + public bool all { get; set; } + } + class TailService { static bool _all = false; static bool _timestamp = false; - public TailService() { - void ignore(string _) { } - IsCommand("tailservice", "Tail updates to named service api. Ex: tail --method=SyntheticSecurityService.metadata"); - HasRequiredOption("method", "Service.method to tail", ignore); - HasOption("time", "Include timestamp", _ => _timestamp = true); - // The following are not implemented. If you need them get to work. - HasOption("path", "Path inside the row to use for display. Ex: 'Id' or 'Name' ", ignore); - HasOption("all", "Invlude 'all' rows on every print", _ => _all = true); - HasAdditionalArguments(null, " [nothing]"); - } - public override int Run(string[] remainingArguments) { - var arg = this.get_argument("method"); - var split = arg.Split('.'); + public static int Run(TailServiceOptions o) { + var split = o.method.Split('.'); if (split.Length != 2) { - Console.WriteLine("Try --method=[ServiceName].[Method]"); + Console.WriteLine("Try --method [ServiceName].[Method]"); return -1; } tail(split[0], split[1]); diff --git a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs index 2b2d1acd0..331469b30 100644 --- a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs +++ b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs @@ -2,30 +2,29 @@ using bts.api; using bts.api.core; using bts.api.synthetic; using bts.test; -using ManyConsole; +using CommandLine; using System; using System.Threading; namespace bts { - public class ServiceTest : ConsoleCommand { - bool _synthetic; - bool _user; - bool _admin; - public ServiceTest() { - IsCommand("service", "Put code here for testing during service development"); - HasOption("synthetic", "Run synthetic test", _ => _synthetic = true); - HasOption("user", "user test", _ => _user = true); - HasOption("admin", "admin test", _ => _admin = true); - } - public override int Run(string[] remainingArguments) { - if (_synthetic) { + public class ServiceTestOptions { + [Option(nameof(synthetic))] + public bool synthetic { get; set; } + [Option(nameof(user))] + public bool user { get; set; } + [Option(nameof(admin))] + public bool admin { get; set; } + } + public class ServiceTest { + public static int Run(ServiceTestOptions o) { + if (o.synthetic) { synthetic_test(); } - if (_user) { + if (o.user) { user_test(); } - if (_admin) { + if (o.admin) { admin_test(); } return 0; diff --git a/src/client/sparkcmd/Cmd/SettingsManagerCommand.cs b/src/client/sparkcmd/Cmd/SettingsManagerCommand.cs index 3c74b9c87..7d01ad08d 100644 --- a/src/client/sparkcmd/Cmd/SettingsManagerCommand.cs +++ b/src/client/sparkcmd/Cmd/SettingsManagerCommand.cs @@ -1,14 +1,13 @@ -using ManyConsole; using System.Reflection; +using CommandLine; namespace bts { - public class SettinsManagerCommand : ConsoleCommand { - public SettinsManagerCommand() { - IsCommand("SettingsManager", "Copy workspaces around"); - } + [Verb("settingsmanager", HelpText="Copy workspaces around")] + public class SettingsManagerOptions { } - public override int Run(string[] remainingArguments) { + public class SettingsManagerCommand { + public static int Run(SettingsManagerOptions _) { var asm = Assembly.Load("spark"); var type = asm.GetType("bts.settings.SettingsManagerTool"); var run = type.GetMethod("run", BindingFlags.Static | BindingFlags.Public); diff --git a/src/client/sparkcmd/Cmd/TestCommand.cs b/src/client/sparkcmd/Cmd/TestCommand.cs index b8a0f790c..89dbf7498 100644 --- a/src/client/sparkcmd/Cmd/TestCommand.cs +++ b/src/client/sparkcmd/Cmd/TestCommand.cs @@ -1,6 +1,5 @@ using bts.test; -using ManyConsole; using System; using mbts; using System.Collections.Generic; @@ -8,47 +7,59 @@ using bts.api.test; using System.Threading.Tasks; using mbts.config; using System.Linq; +using CommandLine; namespace sparkcmd { - public class TestCommand : ConsoleCommand { - uint _timeout = 0; - uint _parallel = 1; - bool _list; - TestType test_type = TestType.Unit; + [Verb("test", HelpText="Run Tests")] + public class TestCommandOptions { - bool self_contained = false; + [Option('t', "timeout", HelpText="Timeout unit tets", Default=0)] + public uint timeout { get; set; } + [Option('l', "list", HelpText="List tests")] + public bool list { get; set; } + [Option('i', "insular", HelpText="Tests run sparkd and use their own config (if specified)")] + public bool self_contained { get; set; } - public TestCommand() { - IsCommand("test", "Run managed unit tests"); - // i comes thorugh as "t" which seems daft. - HasOption("t|timeout", "Timeout unit tests", i => _timeout = 10); - HasOption("n|parallel", "How many tests to run in parallel", i => _parallel = i); - HasOption("l|list", "List all tests", _ => _list = true); - HasOption("unit","Run unit tests", _ => test_type = TestType.Unit); - HasOption("system","Run system tests", _ => test_type = TestType.System); - HasOption("shell","Run shell tests", _ => test_type = TestType.Shell); - HasOption("i|insular|selfcontained", "Tests run sparkd and use their own config (if specified)", _ => self_contained = true); - HasAdditionalArguments(null, " [Test List]"); + // [Option("unit", HelpText="Run unit tests")] + // public bool unit { get; set; } + // [Option("system", HelpText="Run system tests")] + // public bool system { get; set; } + // [Option("shell", HelpText="Run shell tests")] + // public bool shell { get; set; } + + [Value(0, MetaName="TestType")] + public string type { get; set; } + + [Value(1, MetaName="Tests", HelpText="Test list")] + public IEnumerable tests { get; set; } + + } + public static class TestCommand { + static TestType test_type(string t) { + switch(t.ToLower()) { + case "unit": return TestType.Unit; + case "system": return TestType.System; + case "shell": return TestType.Shell; + } + return TestType.Unit; } - public override int Run(string[] tests) { - if (_list) { - list_tests(); + public static int Run(TestCommandOptions o) { + if (o.list) { + list_tests(test_type(o.type)); return 0; } - //Configuration.initialize_for_test(); - - if (_timeout == 0 && tests.Length == 1 && !self_contained) { + if (o.timeout == 0 && o.tests.Count() == 1 && !o.self_contained) { //System.Threading.Thread.Sleep(5000); using (var trr = new TestResultReceiver(true)) { - InProcessTestRunner.run_test_synchronously(tests[0], Enumerator, trr.get_reporter()); + InProcessTestRunner.run_test_synchronously(o.tests.First(), Enumerator, trr.get_reporter()); trr.print_summary(); return trr.ErrorCount; } } - else if (tests.Length > 0) { + else if (o.tests.Count() > 0) { return run_tests(Enumerator.match(tests)); } else { @@ -56,16 +67,16 @@ namespace sparkcmd { } } - Lazy _enumerator = new Lazy(() => new TestEnumerator("spark_test")); - TestEnumerator Enumerator => _enumerator.Value; + static Lazy _enumerator = new Lazy(() => new TestEnumerator("spark_test")); + static TestEnumerator Enumerator => _enumerator.Value; - int run_tests(IEnumerable tests) { + static int run_tests(IEnumerable tests) { var runner = new TestRunner(); // waits for all child processes to exit or time out return runner.run_tests(tests, _timeout, self_contained); } - void list_tests() { + static void list_tests(TestType test_type) { Console.WriteLine("All tests: "); foreach (var t in Enumerator.get_groups_of_type(test_type)) { ConsoleEx.WriteLine(t.name, ConsoleColor.Green); diff --git a/src/client/sparkcmd/Program.cs b/src/client/sparkcmd/Program.cs index 790308e27..14b3df9bb 100644 --- a/src/client/sparkcmd/Program.cs +++ b/src/client/sparkcmd/Program.cs @@ -2,16 +2,37 @@ using bts.api.core; using bts.api.synthetic; using bts.api.test; -using ManyConsole; using System; using System.Linq; +using CommandLine; +using bts; +using System.Collections.Generic; +using System.Linq.Expressions; +using System.Reflection; namespace sparkcmd { class Program { + + // static object option(Func run) {return run;} + // static Type argType(object arg) { + // return arg.GetType().GetGenericArguments().First(); + // } + // static int Run(Func,int> on_error, params object[] args) { + // var types = args.Select(argType).ToArray(); + // var parse = CommandLine.Parser.Default.GetType().GetMethod("ParseArguments", BindingFlags.Public).MakeGenericMethod(types); + // return 3; + // } static int Main(string[] args) { - mbts.config.Configuration.initialize(); - var commands = ConsoleCommandDispatcher.FindCommandsInSameAssemblyAs(typeof(Program)); - return ConsoleCommandDispatcher.DispatchCommand(commands, args, Console.Out); + // note: this can be made to infer the options type list from the list of delegates + return CommandLine.Parser.Default + .ParseArguments(args) + .MapResult( + Configcheck.Run, + ScriptClientCommand.Run, + TailService.Run, + ServiceTest.Run, + SettingsManagerCommand.Run, + errors => 1); } } } \ No newline at end of file diff --git a/src/client/sparkcmd/sparkcmd.csproj b/src/client/sparkcmd/sparkcmd.csproj index 9046c4610..1a8c7f607 100644 --- a/src/client/sparkcmd/sparkcmd.csproj +++ b/src/client/sparkcmd/sparkcmd.csproj @@ -21,7 +21,8 @@ - - - + + + + From c923be6ced4a9ba98c4c4255c9ac434b7c53781f Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 3 Apr 2019 16:11:07 -0400 Subject: [PATCH 015/131] SPARK 2206 add test command to print out PIQ data for all orders --- src/client/spark_core/utils/OrderUtilities.cs | 19 ++++++++++ src/client/sparkcmd/Cmd/ServiceTestCommand.cs | 38 +++++++++++++++++++ src/lib/api/services/MarketDataRelay.proto | 2 +- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 9 ++++- 4 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 src/client/spark_core/utils/OrderUtilities.cs diff --git a/src/client/spark_core/utils/OrderUtilities.cs b/src/client/spark_core/utils/OrderUtilities.cs new file mode 100644 index 000000000..b2042ad14 --- /dev/null +++ b/src/client/spark_core/utils/OrderUtilities.cs @@ -0,0 +1,19 @@ +namespace bts.api.orderman { + + public static class OrderUtilities { + public static NativeOrderStateImmutable? native_order_state(this OrderStateImmutable order) { + switch(order.CustomCase) { + case OrderState.CustomOneofCase.Market: + return order.Market.NativeState; + case OrderState.CustomOneofCase.Limit: + return order.Limit.NativeState; + case OrderState.CustomOneofCase.Stop: + return order.Stop.NativeState; + case OrderState.CustomOneofCase.StopLimit: + return order.StopLimit.NativeState; + default: + return null; + } + } + } +} \ No newline at end of file diff --git a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs index 2b2d1acd0..5450df0d7 100644 --- a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs +++ b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs @@ -1,10 +1,15 @@ using bts.api; using bts.api.core; +using bts.api.orderman; using bts.api.synthetic; +using bts.co2; using bts.test; using ManyConsole; using System; +using System.Reactive.Linq; using System.Threading; +using System.Threading.Tasks; +using System.Linq; namespace bts { public class ServiceTest : ConsoleCommand { @@ -28,10 +33,43 @@ namespace bts { if (_admin) { admin_test(); } + else { + mbo_test(); + } return 0; } + private static void mbo_test() { + using (var ctx = new ServiceContext()) { + var relay_svc = ctx.locate(); + var order_sv = ctx.locate(); + + var upd = order_sv.subscribei_orders().FirstAsync().Wait(); /* don't use Wait() */ + var orders = + upd.all + .Where(o => + o.State.Status == OrderStatus.Working && + o.State.native_order_state().HasValue) + .ToList(); + + var request = new PIQRequest(); + request.Orders.AddRange(orders.Select(o => + new OrderQuery { + SecurityId = o.Def.SecurityKey, + OrderId = UInt64.Parse(o.State.native_order_state().Value.ExchangeOrderId) })); + + Console.WriteLine($"Requesting positions for {request.Orders.Count} orders..."); + var result = relay_svc.get_piq(request); + + Console.WriteLine($"got response - {result.Position.Count} entries"); + foreach (var kvp in result.Position) { + Console.Write($"> {kvp.Key} => "); + ConsoleEx.WriteLine($"{kvp.Value}", ConsoleColor.Yellow); + } + } + } + private static void synthetic_test() { using (var ctx = new ServiceContext()) { var security = new Security { diff --git a/src/lib/api/services/MarketDataRelay.proto b/src/lib/api/services/MarketDataRelay.proto index d5b949673..b2284a2e3 100644 --- a/src/lib/api/services/MarketDataRelay.proto +++ b/src/lib/api/services/MarketDataRelay.proto @@ -23,7 +23,7 @@ message SecurityKeyList { message OrderQuery { uint32 security_id = 1; - uint32 order_id = 2; + uint64 order_id = 2; } message PIQRequest { repeated OrderQuery orders = 1; diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index c711993ac..cd4f49518 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -120,7 +120,14 @@ map> create_feed_engines(const vector Date: Thu, 4 Apr 2019 10:30:12 -0400 Subject: [PATCH 016/131] SPARK-2197 Add some MBO stats logged every 5 minutes --- src/lib/config/protos/toggles.proto | 4 +++ src/lib/md/MBOProcessor.cpp | 45 ++++++++++++++++++++++++++++- src/lib/utils/ConcurrentPool.h | 18 ++++++++++-- 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/lib/config/protos/toggles.proto b/src/lib/config/protos/toggles.proto index 5e1fbd736..3aeedd91b 100644 --- a/src/lib/config/protos/toggles.proto +++ b/src/lib/config/protos/toggles.proto @@ -34,4 +34,8 @@ message SoundNotifications { message VolumeAtPrice{ optional bool enabled = 1 [default = true]; +} + +message MBOStats { + optional bool enabled = 1 [default = true]; } \ No newline at end of file diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 4f89023a6..4d5f2a33d 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -1,9 +1,13 @@ #include "MBOProcessor.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/ConcurrentPool.h" +#include "lib/config/config.h" +#include "lib/config/protos/toggles.pb.h" #include #include +#include +#include #include @@ -20,11 +24,50 @@ struct SecurityKeyHashCompare { } }; +void print_pool_stats(const utils::ConcurrentPool& pool) { + size_t count = 0; + std::vector bid_sz,ask_sz; + pool.unsafe_iterate([&](const MBOSnapshot& s) { + if (s.bids.size() > 0 || s.asks.size() > 0) { + count++; + bid_sz.push_back(s.bids.size()); + ask_sz.push_back(s.asks.size()); + } + }); + + std::sort(bid_sz.begin(), bid_sz.end()); + std::sort(ask_sz.begin(), ask_sz.end()); + size_t bid_tot =0, ask_tot = 0; + for(size_t i : bid_sz) { bid_tot += i; } + for(size_t i : ask_sz) { ask_tot += i; } + + std::stringstream bid_percentiles, ask_percentiles; + for (double x : { 0.5, 0.75, 0.9, 0.95, 0.99 }) { + size_t index = count * x; + bid_percentiles << bid_sz[index] << " "; + ask_percentiles << ask_sz[index] << " "; + } + bid_percentiles << bid_sz[count-1] << " "; + ask_percentiles << ask_sz[count-1] << " "; + LOG(md,Info, + "MBO statistics: count: {} - %[50,75,90,95,99,100] bid: [{}] - ask: [{}]", + count, bid_percentiles.str(), ask_percentiles.str()); +} + typedef tbb::concurrent_hash_map MapType; struct MBOProcessorImpl { utils::ConcurrentPool pool; MapType snapshots; - MBOProcessorImpl() : pool(2048,512), snapshots() {} + utils::DispatcherTimer timer; + MBOProcessorImpl() : + pool(2048,512), + snapshots(), + timer(utils::Dispatcher::background(), 600*1000, [&]() { print_pool_stats(pool); }) + { + if (config::get().enabled()) { + timer.start(); + } + } }; // diff --git a/src/lib/utils/ConcurrentPool.h b/src/lib/utils/ConcurrentPool.h index 91ad99095..92d86e7ea 100644 --- a/src/lib/utils/ConcurrentPool.h +++ b/src/lib/utils/ConcurrentPool.h @@ -10,11 +10,12 @@ template class ConcurrentPool { tbb::concurrent_vector> _all; tbb::concurrent_queue _free; + size_t _initial_size; size_t _grow_size; - std::atomic _outstanding; + size_t _outstanding; public: ConcurrentPool(size_t initial_size=256, size_t grow_size=32): - _all(), _free(), _grow_size(grow_size), _outstanding(0) { + _all(), _free(), _initial_size(initial_size), _grow_size(grow_size), _outstanding(0) { reserve(initial_size); } @@ -34,6 +35,7 @@ class ConcurrentPool { void reserve(size_t count) { std::unique_ptr buffer {new T[count]}; + _outstanding += count; for (size_t x = 0; x < count; ++x) { T* temp = &buffer[x]; release(&temp); @@ -50,6 +52,18 @@ class ConcurrentPool { } } + template + void unsafe_iterate(CB cb) const { + for (auto iter = _all.begin(); iter != _all.end(); ++iter) { + size_t size = (iter == _all.begin()) ? _initial_size : _grow_size; + for (size_t x = 0; x < size; ++x) { + const std::unique_ptr& bucket = *iter; + const T* start = bucket.get(); + cb(start[x]); + } + } + } + }; }} //namespace From b664c42caee7cde642219ae7d507be7d5ce6a3ad Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 8 Apr 2019 09:52:45 -0400 Subject: [PATCH 017/131] SPARK-2179 streaming piq 1/2 --- src/lib/api/services/MBOService.proto | 15 +++ src/lib/api/services/MarketDataRelay.proto | 14 --- src/lib/config/protos/toggles.proto | 1 - src/lib/md/MBOServiceServer.cpp | 102 +++++++++++++++++++++ src/lib/md/MBOServiceServer.h | 25 +++++ src/lib/md/relay/QueryServer.cpp | 73 +-------------- src/lib/utils/LockBox.h | 24 +++++ 7 files changed, 168 insertions(+), 86 deletions(-) create mode 100644 src/lib/api/services/MBOService.proto create mode 100644 src/lib/md/MBOServiceServer.cpp create mode 100644 src/lib/md/MBOServiceServer.h create mode 100644 src/lib/utils/LockBox.h diff --git a/src/lib/api/services/MBOService.proto b/src/lib/api/services/MBOService.proto new file mode 100644 index 000000000..0a00fc22e --- /dev/null +++ b/src/lib/api/services/MBOService.proto @@ -0,0 +1,15 @@ +syntax="proto3"; +package bts.md; +option csharp_namespace="bts.md"; +option cc_enable_arenas=true; +import "lib/api/apioptions.proto"; +import "lib/api/apicore.proto"; + +message OrderPiq { + uint32 order_id = 1 [(bts.api.key)=true]; + uint32 position = 2; +} + +service MBOService { + rpc piq (bts.api.VoidMessage) returns (stream OrderPiq); +} \ No newline at end of file diff --git a/src/lib/api/services/MarketDataRelay.proto b/src/lib/api/services/MarketDataRelay.proto index b2284a2e3..197eecae0 100644 --- a/src/lib/api/services/MarketDataRelay.proto +++ b/src/lib/api/services/MarketDataRelay.proto @@ -21,18 +21,6 @@ message SecurityKeyList { repeated uint32 keys = 1; } -message OrderQuery { - uint32 security_id = 1; - uint64 order_id = 2; -} -message PIQRequest { - repeated OrderQuery orders = 1; -} -message PIQResponse { - // order id => position - map position = 1; -} - service MarketDataRelay { option (bts.api.service_description)="Poll the market data service for updates"; rpc requestmd(bts.co2.MDRequest) returns (bts.co2.MDReply); @@ -40,6 +28,4 @@ service MarketDataRelay { rpc request_products(bts.api.VoidMessage) returns (ProductListReply); rpc request_names(bts.api.VoidMessage) returns (ExpirationNameListReply); rpc get_securities(SecurityKeyList) returns (bts.api.SecurityList); - - rpc get_piq(PIQRequest) returns (PIQResponse); } diff --git a/src/lib/config/protos/toggles.proto b/src/lib/config/protos/toggles.proto index 3aeedd91b..6923cfbe9 100644 --- a/src/lib/config/protos/toggles.proto +++ b/src/lib/config/protos/toggles.proto @@ -35,7 +35,6 @@ message SoundNotifications { message VolumeAtPrice{ optional bool enabled = 1 [default = true]; } - message MBOStats { optional bool enabled = 1 [default = true]; } \ No newline at end of file diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp new file mode 100644 index 000000000..0603d3951 --- /dev/null +++ b/src/lib/md/MBOServiceServer.cpp @@ -0,0 +1,102 @@ +#ifndef _WIN32 +#include "MBOServiceServer.h" + +#include "lib/api/services/OrderService.client.pb.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/TimeStamp.h" +#include "MDClient.h" + +#include "lib/utils/LockBox.h" + +namespace bts { +namespace md { + +// Simple class to handle MD recovery and maintain an index of SecurityKey=>MDConsumerHandle +class HandleIndex { + MDClient& _client; + int _sub; + std::optional _consumer; + utils::LockBox> _index; + +public: + HandleIndex(md::MDClient& client): + _client(client), + _sub(0), + _consumer(std::nullopt), + _index() { + _sub = _client.on_consumer().subscribe([=](const MDConsumer& c) { on_consumer(c); }); + on_consumer(_client.current_consumer()); + } + + ~HandleIndex() { _client.on_consumer().unsubscribe(_sub); } + +private: + void on_consumer(const MDConsumer& c) { + MDConsumer temp(c); + if (temp.all_heartbeats_ok()) { + auto existing = _consumer ? _consumer->id() : 0; + LOG(md,Info, "Updateing MBOService MDConsumer. Old {} new {}", existing, temp.id()); + // clear the index + _index.lock([](auto& index) { index.clear(); }); + // Swap to new consumer + if (_consumer) { + _consumer->swap(temp); + } + else { _consumer = temp; } + // Subscribe to on_security + auto id = _consumer->id(); + _consumer->on_security([this,id](const SecurityPtr&, MDConsumerHandle handle) { + if (_consumer->id() == id) { + _index.lock([handle](auto& index) { + index[SecurityKey(handle.key())] = handle; + }); + } + }); + } + } +}; + + + +struct MBOServiceImpl { + MBOServiceImpl( + api::ServiceContext& ctx, + utils::Dispatcher& d, + md::MDClient& c): + dispatcher(d), + client(c), + orders(ctx.locate()), + handle_index(c), + timer(d, 100, [&]() {}) + {} + + utils::Dispatcher& dispatcher; + md::MDClient& client; + api::orderman::OrderService orders; + + HandleIndex handle_index; + utils::DispatcherTimer timer; +}; + +/////////////////////// + +MBOServiceServer::MBOServiceServer( + api::ServiceContext& ctx, + utils::Dispatcher& d, + md::MDClient& c + ) : + MBOServiceBase(ctx), + _me(new MBOServiceImpl(ctx,d,c)) + { + // subscribe to mdclient + // make publishers + // subscribe to orders + + // _me = utils::make_unique(); + } +MBOServiceServer::~MBOServiceServer() { } + + + +}} //namespace +#endif \ No newline at end of file diff --git a/src/lib/md/MBOServiceServer.h b/src/lib/md/MBOServiceServer.h new file mode 100644 index 000000000..9e47aadfa --- /dev/null +++ b/src/lib/md/MBOServiceServer.h @@ -0,0 +1,25 @@ +#pragma once +#include "lib/api/public/ServiceContext.h" +#include "lib/api/services/MBOService.server.pb.h" +#include + +namespace bts { +namespace utils { class Dispatcher; } +namespace api { namespace orderman { class OrderService; } } +namespace md { + +class MDClient; + +struct MBOServiceImpl; +class MBOServiceServer : public MBOServiceBase { + std::unique_ptr _me; + public: + MBOServiceServer( + api::ServiceContext& ctx, + utils::Dispatcher&, + md::MDClient&); + ~MBOServiceServer(); + +}; + +}} //namespace diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 816f55e78..f7f8c9b84 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -1,4 +1,3 @@ -#ifndef NO_CPP0X #ifndef _WIN32 #include "QueryServer.h" #include "lib/config/config.h" @@ -132,28 +131,6 @@ struct MDConsumerGroup { }; -class HandleIndex { - mutable std::mutex _mutex; - std::unordered_map _index; -public: - template - auto lock(CB cb) { - std::lock_guard guard(_mutex); - return cb(_index); - } - template - auto lock(CB cb) const { - std::lock_guard guard(_mutex); - return cb(_index); - } - void clear() { - lock([](std::unordered_map& index) { - index.clear(); - }); - } -}; - - void convert(PriceSize ps, PriceSizeMessage* pps) { pps->set_mantissa(ps.mantissa()); pps->set_exponent(ps.exponent()); @@ -192,7 +169,6 @@ class QueryServerImpl : public IQueryServer { MDConsumer _consumer; int _sub; boost::container::flat_map> _groups; - HandleIndex _handle_index; public: QueryServerImpl(MDClient& client, metrics::MetricsContainer& stats) : @@ -223,10 +199,6 @@ public: const auto& pgroup = iter->second; bts_assert(pgroup); pgroup->add_security(handle); - // Also insert into handle index - _handle_index.lock([handle](auto& index) { - index[SecurityKey(handle.key())] = handle; - }); } }); } @@ -240,7 +212,7 @@ public: for (auto& pair : _groups) { pair.second->set_consumer(_consumer); } - _handle_index.clear(); + // _handle_index.clear(); subscribe_securities(); utils::Dispatcher::background().post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000); } @@ -440,38 +412,6 @@ public: } return std::nullopt; } - - void get_piq(const PIQRequest& req, PIQResponse* result) const { - // for each pair in request - // look up the mbo - // search for the order in the mbo and add it to the result if found - auto& rmap = *result->mutable_position(); - for(const auto& entry : req.orders()) { - auto order_key = OrderKey(entry.order_id()); - auto security = SecurityKey(entry.security_id()); - auto handle = _handle_index.lock( - [security](auto& index){ - auto iter = index.find(security); - if (iter != index.end()) { - return iter->second; - } - return MDConsumerHandle(); - }); - if (handle.is_initialized()) { - // get mbo - auto mbo = handle.get_mbo(); - // search for order - auto search_result = search_orders(order_key, mbo->bids); - if (!search_result) { - search_result = search_orders(order_key, mbo->asks); - } - if (search_result) { - rmap[order_key.value()] = search_result.value(); - } - } - } - } - }; struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { @@ -567,17 +507,9 @@ struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { reply(*preply); }); } - - virtual void get_piq(const api::AuthToken&, const PIQRequest& request, const std::function& reply) override { - _threadpool.post([=]() { - google::protobuf::Arena arena; - PIQResponse* presponse = google::protobuf::Arena::CreateMessage(&arena); - _db.get_piq(request, presponse); - reply(*presponse); - }); - } }; + struct QueryServer { QueryServer(utils::Dispatcher& threadpool, const bts::api::ServiceContext& ctx, @@ -644,4 +576,3 @@ std::unique_ptr create_query_server(MDClient& mdc, metrics::Metric }}//namespace #endif -#endif diff --git a/src/lib/utils/LockBox.h b/src/lib/utils/LockBox.h new file mode 100644 index 000000000..4e37b8024 --- /dev/null +++ b/src/lib/utils/LockBox.h @@ -0,0 +1,24 @@ +#pragma once +#include + +namespace bts { +namespace utils { + +template +class LockBox { + mutable std::mutex _mutex; + T _data; +public: + template + auto lock(CB cb) { + std::lock_guard guard(_mutex); + return cb(_data); + } + template + auto lock(CB cb) const { + std::lock_guard guard(_mutex); + return cb(_data); + } +}; + +}} //namespace From 9fd7db55ca7a0d6f66fbf142cd9a6fa29b47b8e9 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 9 Apr 2019 09:54:42 -0400 Subject: [PATCH 018/131] SPARK-2248 Update.h should have add/remove/change instead of just add/remove --- src/lib/api/public/ClientChannel.h | 47 +++++++++++++++++++----------- src/lib/api/public/RawUpdate.h | 14 +++++++-- src/lib/api/public/Update.h | 32 ++++++++++++++++++-- src/lib/md/MBOServiceServer.cpp | 21 +++++++++---- 4 files changed, 86 insertions(+), 28 deletions(-) diff --git a/src/lib/api/public/ClientChannel.h b/src/lib/api/public/ClientChannel.h index 3918f3f05..25cbd019f 100644 --- a/src/lib/api/public/ClientChannel.h +++ b/src/lib/api/public/ClientChannel.h @@ -5,6 +5,8 @@ #include "ApiException.h" #include "RawUpdate.h" #include "ServiceContext.h" +#include "lib/logging/logging.h" + #include #include #include @@ -12,6 +14,17 @@ namespace bts { namespace api { +template +std::shared_ptr deserialize(const char* buffer, size_t length) { + google::protobuf::io::ArrayInputStream stream(buffer, length); + auto item = std::make_shared(); + if (item->ParseFromZeroCopyStream(&stream)) { + return item; + } + LOG(Api,Error,"Failed to deserialize proto from api stream. This is likely fatal."); + return nullptr; +} + class ClientChannel { int _id; @@ -80,29 +93,29 @@ public: } else { bool first_update = true; - auto cb = [observer, first_update](EventType event_type, const StreamUpdate* changes, int change_count) mutable { + auto cb = [observer, first_update, method, qualifier](EventType event_type, const StreamUpdate* changes, int change_count) mutable { if (event_type == EventType::TABLE_UPDATED) { - std::vector> added_items; + std::vector> added_items, changed_items; std::vector added_keys, - removed_keys; + changed_keys, + removed_keys; for (size_t x = 0; x < change_count; ++x) { const StreamUpdate& change = changes[x]; - if (change.type != StreamUpdate::REMOVE) { - added_keys.reserve(change_count); - added_items.reserve(change_count); - added_keys.push_back(change.key); - google::protobuf::io::ArrayInputStream stream(change.buffer, change.buffer_len); - auto item = std::make_shared(); - if (item->ParseFromZeroCopyStream(&stream)) { - added_items.push_back(move(item)); - } - } - if (change.type != StreamUpdate::CREATE) { - removed_keys.reserve(change_count); - removed_keys.push_back(change.key); + switch(change.type) { + case StreamUpdate::REMOVE: + removed_keys.push_back(change.key); + break; + case StreamUpdate::REPLACE: + changed_items.push_back(deserialize(change.buffer, change.buffer_len)); + changed_keys.push_back(change.key); + break; + case StreamUpdate::CREATE: + added_items.push_back(deserialize(change.buffer, change.buffer_len)); + added_keys.push_back(change.key); + break; } } - observer.on_next(RawUpdate>(added_items, added_keys, removed_keys)); + observer.on_next(RawUpdate>(added_items, added_keys, changed_items, changed_keys, removed_keys)); } else if (event_type == EventType::TABLE_CREATED) { if (!first_update) { diff --git a/src/lib/api/public/RawUpdate.h b/src/lib/api/public/RawUpdate.h index be79fa432..93e7cabe3 100644 --- a/src/lib/api/public/RawUpdate.h +++ b/src/lib/api/public/RawUpdate.h @@ -6,10 +6,16 @@ namespace api { template struct RawUpdate { - RawUpdate(std::vector& added_items, std::vector& added_keys, std::vector& removed_keys) : + RawUpdate(std::vector& added_items, + std::vector& added_keys, + std::vector& changed_items, + std::vector& changed_keys, + std::vector& removed_keys) : is_reset(false), added_items(std::move(added_items)), added_keys(std::move(added_keys)), + changed_items(std::move(changed_items)), + changed_keys(std::move(changed_keys)), removed_keys(std::move(removed_keys)) {} @@ -20,12 +26,16 @@ namespace api { bool is_reset; std::vector added_items; std::vector added_keys; + + std::vector changed_items; + std::vector changed_keys; + std::vector removed_keys; private: RawUpdate() : is_reset(true), - added_items(), added_keys(), removed_keys() + added_items(), added_keys(), changed_items(), changed_keys(), removed_keys() {} }; diff --git a/src/lib/api/public/Update.h b/src/lib/api/public/Update.h index d37345679..4901bded1 100644 --- a/src/lib/api/public/Update.h +++ b/src/lib/api/public/Update.h @@ -9,12 +9,21 @@ namespace bts { namespace api { + template + class Change { + T _old, _new; + public: + Change(const T& old_value, const T& new_value) : _old(old_value), _new(new_value) { } + const T& old() { return _old; } + const T& current() { return _new; } + }; template struct Update { - Update(std::vector& all, std::vector& added, std::vector& removed) : + Update(std::vector& all, std::vector& added, std::vector> changed, std::vector& removed) : _all_items(std::move(all)), _added_items(std::move(added)), + _changed_items(std::move(changed)), _removed_items(std::move(removed)) {} @@ -22,17 +31,20 @@ namespace api { Update(Update&& other) : _all_items(std::move(other._all_items)), _added_items(std::move(other._added_items)), + _changed_items(std::move(other._changed_items)), _removed_items(std::move(other._removed_items)) {} TKey key(const T& t) const { return ServiceType::key(t); } const std::vector& all() const { return _all_items; } const std::vector& added() const { return _added_items; } + const std::vector>& changed() const { return _changed_items; } const std::vector& removed() const { return _removed_items; } private: std::vector _all_items; std::vector _added_items; + std::vector> _changed_items; std::vector _removed_items; }; @@ -49,9 +61,12 @@ namespace api { } state->clear(); std::vector all, added; - return Update(all, added, removed); + std::vector> changed; + return Update(all, added, changed, removed); } else { + // + // Remove std::vector removed; removed.reserve(raw.removed_keys.size()); for (int key : raw.removed_keys) { @@ -61,6 +76,8 @@ namespace api { state->erase(iter); } } + // + // Add for (size_t x = 0; x < raw.added_keys.size(); ++x) { (*state)[raw.added_keys[x]] = raw.added_items[x]; } @@ -69,7 +86,16 @@ namespace api { for (const auto& item : *state) { all.push_back(item.second); } - return Update(all, raw.added_items, removed); + // + // Change + std::vector> changed; + changed.reserve(raw.changed_keys.size()); + for (size_t x = 0; x < raw.changed_keys.size(); ++x) { + changed.push_back(Change{(*state)[raw.changed_keys[x]], raw.changed_items[x]}); + (*state)[raw.changed_keys[x]] = raw.changed_items[x]; + } + + return Update(all, raw.added_items, changed, removed); } }); }; diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index 0603d3951..0c4c2ce49 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -86,14 +86,23 @@ MBOServiceServer::MBOServiceServer( md::MDClient& c ) : MBOServiceBase(ctx), - _me(new MBOServiceImpl(ctx,d,c)) + _me(new MBOServiceImpl(ctx,d,c,make_piq_publisher())) { - // subscribe to mdclient - // make publishers - // subscribe to orders - - // _me = utils::make_unique(); + // subscribe to mdclient ✔ + // make publishers ✔ + // subscribe to orders => stash active orders dict + _me->orders_sub = + _me->orders.subscribe_orders().subscribe([=](const auto& update) { + const auto& added = update.added(); + const auto& removed = update.removed(); + // api thread + _me->dispatcher.post([this,added,removed]() { + // dispatcher thread + // handle add/remove/change + }); + }); } + MBOServiceServer::~MBOServiceServer() { } From 06aa540110044355bae1aa8866db62570f978810 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 9 Apr 2019 09:57:44 -0400 Subject: [PATCH 019/131] SPARK-2248 Update api subscribe clients for this api change --- src/lib/api/public/Update.h | 4 ++-- src/server/sparkd/position/PositionServiceServer.cpp | 4 ++++ src/server/sparkd/user/AdminServiceServer.cpp | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/lib/api/public/Update.h b/src/lib/api/public/Update.h index 4901bded1..7de659ded 100644 --- a/src/lib/api/public/Update.h +++ b/src/lib/api/public/Update.h @@ -14,8 +14,8 @@ namespace api { T _old, _new; public: Change(const T& old_value, const T& new_value) : _old(old_value), _new(new_value) { } - const T& old() { return _old; } - const T& current() { return _new; } + const T& old() const { return _old; } + const T& current() const { return _new; } }; template diff --git a/src/server/sparkd/position/PositionServiceServer.cpp b/src/server/sparkd/position/PositionServiceServer.cpp index 274ae22b9..560710a10 100644 --- a/src/server/sparkd/position/PositionServiceServer.cpp +++ b/src/server/sparkd/position/PositionServiceServer.cpp @@ -48,6 +48,10 @@ me(new PImpl(context, me->publisher.add_or_update(_on_fill(*f,false)); for (const auto& f : update.added()) me->publisher.add_or_update(_on_fill(*f,true)); + for (const auto& f : update.changed()) { + me->publisher.add_or_update(_on_fill(*f.old(),false)); + me->publisher.add_or_update(_on_fill(*f.current(),true)); + } me->publisher.flush(); }); } diff --git a/src/server/sparkd/user/AdminServiceServer.cpp b/src/server/sparkd/user/AdminServiceServer.cpp index 09b33068d..9e317e7fc 100644 --- a/src/server/sparkd/user/AdminServiceServer.cpp +++ b/src/server/sparkd/user/AdminServiceServer.cpp @@ -66,6 +66,10 @@ AdminServiceServer::AdminServiceServer(bts::api::ServiceContext& context) if(it != _order_map.end()) _order_map.erase(it); } + for (const auto& ch : update.changed()) { + const auto& o = ch.current(); + _order_map[o->key()] = o; + } for (const auto& o : update.added()){ _order_map[o->key()]=o; } From d76b098c5171f8a64e0de4c773e2511a274cab02 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 8 Apr 2019 16:42:47 -0400 Subject: [PATCH 020/131] SPARK-2179 streaming piq 2/3 --- src/client/sparkcmd/Cmd/ServiceTestCommand.cs | 52 +++++++++---------- src/lib/md/MBOServiceServer.cpp | 32 +++++++++++- 2 files changed, 56 insertions(+), 28 deletions(-) diff --git a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs index 5450df0d7..1f449cd4d 100644 --- a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs +++ b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs @@ -34,41 +34,41 @@ namespace bts { admin_test(); } else { - mbo_test(); + // mbo_test(); } return 0; } - private static void mbo_test() { - using (var ctx = new ServiceContext()) { - var relay_svc = ctx.locate(); - var order_sv = ctx.locate(); + // private static void mbo_test() { + // using (var ctx = new ServiceContext()) { + // var relay_svc = ctx.locate(); + // var order_sv = ctx.locate(); - var upd = order_sv.subscribei_orders().FirstAsync().Wait(); /* don't use Wait() */ - var orders = - upd.all - .Where(o => - o.State.Status == OrderStatus.Working && - o.State.native_order_state().HasValue) - .ToList(); + // var upd = order_sv.subscribei_orders().FirstAsync().Wait(); /* don't use Wait() */ + // var orders = + // upd.all + // .Where(o => + // o.State.Status == OrderStatus.Working && + // o.State.native_order_state().HasValue) + // .ToList(); - var request = new PIQRequest(); - request.Orders.AddRange(orders.Select(o => - new OrderQuery { - SecurityId = o.Def.SecurityKey, - OrderId = UInt64.Parse(o.State.native_order_state().Value.ExchangeOrderId) })); + // var request = new PIQRequest(); + // request.Orders.AddRange(orders.Select(o => + // new OrderQuery { + // SecurityId = o.Def.SecurityKey, + // OrderId = UInt64.Parse(o.State.native_order_state().Value.ExchangeOrderId) })); - Console.WriteLine($"Requesting positions for {request.Orders.Count} orders..."); - var result = relay_svc.get_piq(request); + // Console.WriteLine($"Requesting positions for {request.Orders.Count} orders..."); + // var result = relay_svc.get_piq(request); - Console.WriteLine($"got response - {result.Position.Count} entries"); - foreach (var kvp in result.Position) { - Console.Write($"> {kvp.Key} => "); - ConsoleEx.WriteLine($"{kvp.Value}", ConsoleColor.Yellow); - } - } - } + // Console.WriteLine($"got response - {result.Position.Count} entries"); + // foreach (var kvp in result.Position) { + // Console.Write($"> {kvp.Key} => "); + // ConsoleEx.WriteLine($"{kvp.Value}", ConsoleColor.Yellow); + // } + // } + // } private static void synthetic_test() { using (var ctx = new ServiceContext()) { diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index 0c4c2ce49..2841633d3 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -8,10 +8,15 @@ #include "lib/utils/LockBox.h" +#include +#include + namespace bts { namespace md { +// // Simple class to handle MD recovery and maintain an index of SecurityKey=>MDConsumerHandle + class HandleIndex { MDClient& _client; int _sub; @@ -56,18 +61,29 @@ private: } }; +struct ActiveOrder { + uint32_t id; + int64_t ex_id; + int32_t position; +}; +// +// MBOService struct MBOServiceImpl { MBOServiceImpl( api::ServiceContext& ctx, utils::Dispatcher& d, - md::MDClient& c): + md::MDClient& c, + MBOServiceBase::PiqPublisher&& pub + ): dispatcher(d), client(c), orders(ctx.locate()), handle_index(c), - timer(d, 100, [&]() {}) + publisher(pub), + timer(d, 200, [&]() {timer_tick();}), + orders_sub() {} utils::Dispatcher& dispatcher; @@ -75,9 +91,21 @@ struct MBOServiceImpl { api::orderman::OrderService orders; HandleIndex handle_index; + MBOServiceBase::PiqPublisher publisher; utils::DispatcherTimer timer; + + rxcpp::composite_subscription orders_sub; + + void timer_tick(); + void flush(); }; +void MBOServiceImpl::timer_tick() { + // for each active + + // publisher.flush(); +} + /////////////////////// MBOServiceServer::MBOServiceServer( From a101a267ed859cbd55a005cbaa051ee27108199c Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 9 Apr 2019 14:58:15 -0400 Subject: [PATCH 021/131] More consistent to call old previous for Change --- src/lib/api/public/Update.h | 2 +- src/server/sparkd/position/PositionServiceServer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/api/public/Update.h b/src/lib/api/public/Update.h index 7de659ded..c4b615950 100644 --- a/src/lib/api/public/Update.h +++ b/src/lib/api/public/Update.h @@ -14,7 +14,7 @@ namespace api { T _old, _new; public: Change(const T& old_value, const T& new_value) : _old(old_value), _new(new_value) { } - const T& old() const { return _old; } + const T& previous() const { return _old; } const T& current() const { return _new; } }; diff --git a/src/server/sparkd/position/PositionServiceServer.cpp b/src/server/sparkd/position/PositionServiceServer.cpp index 560710a10..49706a39c 100644 --- a/src/server/sparkd/position/PositionServiceServer.cpp +++ b/src/server/sparkd/position/PositionServiceServer.cpp @@ -49,7 +49,7 @@ me(new PImpl(context, for (const auto& f : update.added()) me->publisher.add_or_update(_on_fill(*f,true)); for (const auto& f : update.changed()) { - me->publisher.add_or_update(_on_fill(*f.old(),false)); + me->publisher.add_or_update(_on_fill(*f.previous(),false)); me->publisher.add_or_update(_on_fill(*f.current(),true)); } me->publisher.flush(); From 0cf4b5c2c8597f7bac1e9ea970c8cf20535fa592 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 9 Apr 2019 15:07:10 -0400 Subject: [PATCH 022/131] SPARK-2224 3/3 basic piq stream (unblocks jake) --- src/lib/api/services/MBOService.proto | 4 +- src/lib/md/ExchangeOrderKey.h | 7 + src/lib/md/MBOProcessor.h | 5 +- src/lib/md/MBOServiceServer.cpp | 156 ++++++++++++++++-- src/lib/md/relay/QueryServer.cpp | 19 --- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 4 +- 6 files changed, 156 insertions(+), 39 deletions(-) create mode 100644 src/lib/md/ExchangeOrderKey.h diff --git a/src/lib/api/services/MBOService.proto b/src/lib/api/services/MBOService.proto index 0a00fc22e..4a8383f8b 100644 --- a/src/lib/api/services/MBOService.proto +++ b/src/lib/api/services/MBOService.proto @@ -5,11 +5,11 @@ option cc_enable_arenas=true; import "lib/api/apioptions.proto"; import "lib/api/apicore.proto"; -message OrderPiq { +message OrderPIQ { uint32 order_id = 1 [(bts.api.key)=true]; uint32 position = 2; } service MBOService { - rpc piq (bts.api.VoidMessage) returns (stream OrderPiq); + rpc piq (bts.api.VoidMessage) returns (stream OrderPIQ); } \ No newline at end of file diff --git a/src/lib/md/ExchangeOrderKey.h b/src/lib/md/ExchangeOrderKey.h new file mode 100644 index 000000000..b3d59e459 --- /dev/null +++ b/src/lib/md/ExchangeOrderKey.h @@ -0,0 +1,7 @@ +#pragma once +#include "lib/utils/ValueWrapper.h" +namespace bts { +namespace md { +using ExchangeOrderKey = utils::ValueWrapper; +using SparkOrderKey = utils::ValueWrapper; +} } \ No newline at end of file diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index bbb8c7e21..17246fd21 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -4,17 +4,16 @@ #include "lib/utils/TimeStamp.h" #include "lib/utils/ValueWrapper.h" #include "IntPrice.h" +#include "ExchangeOrderKey.h" #include #include namespace bts { namespace md { - -using OrderKey = utils::ValueWrapper; struct SnapshotOrder { - OrderKey id; + ExchangeOrderKey id; IntPrice price; int32_t size; }; diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index 2841633d3..6a9218a80 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -5,11 +5,17 @@ #include "lib/utils/Dispatcher.h" #include "lib/utils/TimeStamp.h" #include "MDClient.h" +#include "ExchangeOrderKey.h" +#include "MBOProcessor.h" #include "lib/utils/LockBox.h" #include #include +#include + + +using namespace bts::api::orderman; namespace bts { namespace md { @@ -61,10 +67,36 @@ private: } }; -struct ActiveOrder { - uint32_t id; - int64_t ex_id; - int32_t position; +class ActiveOrder { + SecurityKey _security; + SparkOrderKey _id; + ExchangeOrderKey _ex_id; + int32_t _position; + +public: + ActiveOrder(): _security(), _id(), _ex_id(), _position(-1) { bts_assert(false); } + ActiveOrder(SecurityKey sec, SparkOrderKey s, ExchangeOrderKey ex): _security(sec), _id(s), _ex_id(ex), _position(-1) {} + ActiveOrder(SecurityKey sec, SparkOrderKey s, const std::string& ex): _security(sec), _id(s), _ex_id(), _position(-1) { + _ex_id = ExchangeOrderKey(std::stoul(ex)); + } + + SparkOrderKey id() const { return _id; } + ExchangeOrderKey exchange_id() const { return _ex_id; } + int32_t& position() { return _position; }; + int32_t position() const { return _position; }; + bool set_position(int32_t pos) { + if (pos != _position) { + _position = pos; + return true; + } + return false; + } + OrderPIQ to_piq() const { + OrderPIQ result; + result.set_order_id(_id.value()); + result.set_position(_position); + return result; + } }; // @@ -81,33 +113,102 @@ struct MBOServiceImpl { client(c), orders(ctx.locate()), handle_index(c), - publisher(pub), + publish(pub), timer(d, 200, [&]() {timer_tick();}), orders_sub() {} utils::Dispatcher& dispatcher; md::MDClient& client; - api::orderman::OrderService orders; + OrderService orders; HandleIndex handle_index; - MBOServiceBase::PiqPublisher publisher; + MBOServiceBase::PiqPublisher publish; utils::DispatcherTimer timer; rxcpp::composite_subscription orders_sub; + std::unordered_map active; void timer_tick(); void flush(); }; void MBOServiceImpl::timer_tick() { - // for each active + //Dispatcher thread + std::vector rows; - // publisher.flush(); + publish(nullptr, &rows, nullptr); } + /////////////////////// +std::optional search_orders(const ExchangeOrderKey oid, const std::vector& orders) { + IntPrice price = {}; + size_t position = 0; + for (size_t x = 0; x < orders.size(); ++x) { + const auto& oo = orders[x]; + if (oo.id == oid) { + return { position }; + } + else if (price == oo.price) { + position += 1; + } + else { + price = oo.price; + position = 0; + } + } + return std::nullopt; +} + +bool update_position(const HandleIndex& , ActiveOrder& order) { + order.set_position(-2); + return true; +} + +const NativeOrderState* native_order_state(const OrderRecord& record) { + switch(record.state().custom_case()) { + case OrderState::kMarket: + return &record.state().market().native_state(); + case OrderState::kLimit: + return &record.state().limit().native_state(); + case OrderState::kStop: + return &record.state().stop().native_state(); + case OrderState::kStopLimit: + return &record.state().stop_limit().native_state(); + default: + return nullptr; + } +} + +bool is_active_exchange_order(const OrderRecord& record) { + return record.state().status() == ORDER_STATUS_WORKING && + native_order_state(record) != nullptr; +} + +void try_remove(MBOServiceImpl& me, std::set& to_remove, const OrderRecord& record) { + auto key = SparkOrderKey(record.key()); + auto iter = me.active.find(key); + if (iter != me.active.end()) { + me.active.erase(key); + to_remove.insert(key.value()); + } +} +void try_add(MBOServiceImpl& me, std::vector& to_add, const OrderRecord& record) { + if (is_active_exchange_order(record)) { + auto native = native_order_state(record); + bts_assert(native); + ActiveOrder order( + SecurityKey(record.def().security_key()), + SparkOrderKey(record.key()), + native->exchange_order_id()); + update_position(me.handle_index, order); + to_add.push_back(order.to_piq()); + me.active[order.id()] = order; + } +} + MBOServiceServer::MBOServiceServer( api::ServiceContext& ctx, utils::Dispatcher& d, @@ -121,12 +222,41 @@ MBOServiceServer::MBOServiceServer( // subscribe to orders => stash active orders dict _me->orders_sub = _me->orders.subscribe_orders().subscribe([=](const auto& update) { - const auto& added = update.added(); - const auto& removed = update.removed(); // api thread - _me->dispatcher.post([this,added,removed]() { + const auto& added = update.added(); + const auto& changed = update.changed(); + const auto& removed = update.removed(); + _me->dispatcher.post([this,added,changed,removed]() { // dispatcher thread - // handle add/remove/change + std::vector to_add, to_change; + std::set to_remove; + + // Remove + for(const auto& r : removed) { + // If it was active remove it and add to to_remove + try_remove(*_me, to_remove, *r); + } + // Added + for(const auto& add : added) { + // If active add look up the position and add to the state + try_add(*_me, to_add, *add); + } + // Changed + for(const auto& ch : changed) { + // new order is active and old is not look up piq and add to state + bool is_active = is_active_exchange_order(*ch.current()); + bool was_active = is_active_exchange_order(*ch.previous()); + if (is_active && !was_active) { + try_add(*_me, to_add, *ch.current()); + } + else if (was_active && !is_active) { + try_remove(*_me, to_remove, *ch.current()); + } + // is_active && was_active => don't care + // !is_active && !was_active => don't care + } + + _me->publish(&to_add, &to_change, &to_remove); }); }); } diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index f7f8c9b84..4dc971d81 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -393,25 +393,6 @@ public: } } } - - static std::optional search_orders(const OrderKey oid, const std::vector& orders) { - IntPrice price = {}; - size_t position = 0; - for (size_t x = 0; x < orders.size(); ++x) { - const auto& oo = orders[x]; - if (oo.id == oid) { - return { position }; - } - else if (price == oo.price) { - position += 1; - } - else { - price = oo.price; - position = 0; - } - } - return std::nullopt; - } }; struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index bd943250f..85867b973 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -129,10 +129,10 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con auto* snapshot = _mbo->allocate_snapshot(); // copy from onixs book for (const auto& bid : book.bids()) { - snapshot->bids.push_back({ OrderKey(bid.id()), convert_price(bid.price(), 0), bid.quantity() }); + snapshot->bids.push_back({ ExchangeOrderKey(bid.id()), convert_price(bid.price(), 0), bid.quantity() }); } for (const auto& ask : book.offers()) { - snapshot->asks.push_back({ OrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); + snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); } _mbo->publish_snapshot(SecurityKey(key), snapshot); } From c1789bb4a196936b66c6b33533a90c001496eb8c Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 9 Apr 2019 15:14:35 -0400 Subject: [PATCH 023/131] SPARK-2244 Helps to enable the service --- src/server/sparkd/cmds/Run.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 3dd41a86c..5f255fc56 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -15,6 +15,7 @@ #include "lib/md/SyntheticSecurityService.h" #include "lib/md/relay/QueryServer.h" #include "lib/md/MBOProcessor.h" +#include "lib/md/MBOServiceServer.h" #include "lib/metrics/metrics.h" #include "lib/metrics/system_metrics.h" #include "lib/orderman/order_factory/DynamicOrderRegistry.h" @@ -65,10 +66,11 @@ int Run::execute(std::vector const &) { api::synthetic::SyntheticSecurityService synthetic_service(sc); md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service, &mbo.reader()); - md::QueryServerPtr md_query_server( md::run_query_server(sc, mdclient, stats)); + md::MBOServiceServer mboserver(sc, utils::Dispatcher::background(), mdclient); + // initialize order plugins orderman::DynamicOrderRegistry dyn_order_registry; SparkServices plugin_services(dyn_order_registry); From 5ff9f13e7ede1ba07cb31056170dcb6000e8ccd4 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Apr 2019 10:11:17 -0400 Subject: [PATCH 024/131] Update cmakedeps --- src/lib/CMakeDeps_win.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/CMakeDeps_win.cmake b/src/lib/CMakeDeps_win.cmake index 879f19a67..e6cf6b118 100644 --- a/src/lib/CMakeDeps_win.cmake +++ b/src/lib/CMakeDeps_win.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file From 5ba7cd561cb4168f27adc27ae08a026940763f09 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Apr 2019 10:16:18 -0400 Subject: [PATCH 025/131] SPARK-2244 remove parameterless ActiveOrder constructor --- src/lib/md/MBOServiceServer.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index 6a9218a80..9f1b57a4b 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -74,7 +74,8 @@ class ActiveOrder { int32_t _position; public: - ActiveOrder(): _security(), _id(), _ex_id(), _position(-1) { bts_assert(false); } + // empty constructor needed for unordered map operator[] + // ActiveOrder(): _security(), _id(), _ex_id(), _position(-1) { bts_assert(false); } ActiveOrder(SecurityKey sec, SparkOrderKey s, ExchangeOrderKey ex): _security(sec), _id(s), _ex_id(ex), _position(-1) {} ActiveOrder(SecurityKey sec, SparkOrderKey s, const std::string& ex): _security(sec), _id(s), _ex_id(), _position(-1) { _ex_id = ExchangeOrderKey(std::stoul(ex)); @@ -205,7 +206,7 @@ void try_add(MBOServiceImpl& me, std::vector& to_add, const OrderRecor native->exchange_order_id()); update_position(me.handle_index, order); to_add.push_back(order.to_piq()); - me.active[order.id()] = order; + me.active.insert(std::make_pair(order.id(), order)); } } @@ -257,6 +258,7 @@ MBOServiceServer::MBOServiceServer( } _me->publish(&to_add, &to_change, &to_remove); + if (!_me->timer.running()) _me->timer.start(); }); }); } From a3de83dd04721d755863bef7223787ca07690c66 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Apr 2019 10:24:33 -0400 Subject: [PATCH 026/131] SPARK-2244 Update position on timer tick --- src/lib/md/MBOServiceServer.cpp | 53 +++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index 9f1b57a4b..d4c7e3261 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -27,20 +27,22 @@ class HandleIndex { MDClient& _client; int _sub; std::optional _consumer; - utils::LockBox> _index; public: + utils::LockBox> index; + HandleIndex(md::MDClient& client): _client(client), _sub(0), _consumer(std::nullopt), - _index() { + index() { _sub = _client.on_consumer().subscribe([=](const MDConsumer& c) { on_consumer(c); }); on_consumer(_client.current_consumer()); } ~HandleIndex() { _client.on_consumer().unsubscribe(_sub); } + private: void on_consumer(const MDConsumer& c) { MDConsumer temp(c); @@ -48,7 +50,7 @@ private: auto existing = _consumer ? _consumer->id() : 0; LOG(md,Info, "Updateing MBOService MDConsumer. Old {} new {}", existing, temp.id()); // clear the index - _index.lock([](auto& index) { index.clear(); }); + index.lock([](auto& index) { index.clear(); }); // Swap to new consumer if (_consumer) { _consumer->swap(temp); @@ -58,7 +60,7 @@ private: auto id = _consumer->id(); _consumer->on_security([this,id](const SecurityPtr&, MDConsumerHandle handle) { if (_consumer->id() == id) { - _index.lock([handle](auto& index) { + index.lock([handle](auto& index) { index[SecurityKey(handle.key())] = handle; }); } @@ -81,6 +83,7 @@ public: _ex_id = ExchangeOrderKey(std::stoul(ex)); } + SecurityKey security_id() const { return _security; } SparkOrderKey id() const { return _id; } ExchangeOrderKey exchange_id() const { return _ex_id; } int32_t& position() { return _position; }; @@ -134,12 +137,6 @@ struct MBOServiceImpl { void flush(); }; -void MBOServiceImpl::timer_tick() { - //Dispatcher thread - std::vector rows; - - publish(nullptr, &rows, nullptr); -} /////////////////////// @@ -163,9 +160,39 @@ std::optional search_orders(const ExchangeOrderKey oid, const std::vec return std::nullopt; } -bool update_position(const HandleIndex& , ActiveOrder& order) { - order.set_position(-2); - return true; +bool update_position(const HandleIndex& handles, ActiveOrder& order) { + const auto& index = handles.index; + auto handle = index.lock([id=order.security_id()](const auto& index) { + auto iter = index.find(id); + return iter != index.end() ? iter->second : MDConsumerHandle {}; + }); + + if (handle.is_initialized()) { + auto mbo = handle.get_mbo(); + if (mbo) { + auto search_result = search_orders(order.exchange_id(), mbo->bids); + if (!search_result) { + search_result = search_orders(order.exchange_id(), mbo->asks); + } + if (search_result) { + order.set_position(search_result.value()); + return true; + } + } + } + return false; +} + +void MBOServiceImpl::timer_tick() { + //Dispatcher thread + std::vector rows; + for (auto& pair : active) { + if (update_position(handle_index, pair.second)) { + rows.push_back(pair.second.to_piq()); + } + } + + publish(nullptr, &rows, nullptr); } const NativeOrderState* native_order_state(const OrderRecord& record) { From adec1bdfd06cee6c02334246fe969eb8d0b49018 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Apr 2019 10:57:44 -0400 Subject: [PATCH 027/131] SPARK-678 Dont' crash on null array --- .../api/Channels/QuickBusClientChannel.cs | 28 ++++++++++--------- src/lib/api/core/ApiClientInterop.cpp | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/client/spark_api/api/Channels/QuickBusClientChannel.cs b/src/client/spark_api/api/Channels/QuickBusClientChannel.cs index ac054c2c8..593edf9f7 100644 --- a/src/client/spark_api/api/Channels/QuickBusClientChannel.cs +++ b/src/client/spark_api/api/Channels/QuickBusClientChannel.cs @@ -110,19 +110,21 @@ namespace bts.api.core { the_update.Clear(); try { if (event_type == ApiNativeMethods.EventType.TABLE_UPDATED) { - foreach (var change in changes) { - switch (change.type) { - case ApiNativeMethods.ChangeType.REMOVE: - the_update.removed_keys.Add(change.key); - break; - case ApiNativeMethods.ChangeType.REPLACE: - the_update.changed_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); - the_update.changed_keys.Add(change.key); - break; - case ApiNativeMethods.ChangeType.CREATE: - the_update.added_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); - the_update.added_keys.Add(change.key); - break; + if (changes != null) { + foreach (var change in changes) { + switch (change.type) { + case ApiNativeMethods.ChangeType.REMOVE: + the_update.removed_keys.Add(change.key); + break; + case ApiNativeMethods.ChangeType.REPLACE: + the_update.changed_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); + the_update.changed_keys.Add(change.key); + break; + case ApiNativeMethods.ChangeType.CREATE: + the_update.added_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); + the_update.added_keys.Add(change.key); + break; + } } } observer.OnNext(the_update); diff --git a/src/lib/api/core/ApiClientInterop.cpp b/src/lib/api/core/ApiClientInterop.cpp index 3927bbc3e..487b2eabb 100644 --- a/src/lib/api/core/ApiClientInterop.cpp +++ b/src/lib/api/core/ApiClientInterop.cpp @@ -176,7 +176,7 @@ void safe_call_subscription_callback(raw_subscription_callback cb, EventType typ if (updates.size() > 0) cb(type, &updates[0], updates.size(), extra); else - cb(type, nullptr, updates.size(), extra); + cb(type, nullptr, 0, extra); }); } From f6c0f393bdee701fe83d14ccefc6a9a086701b91 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Apr 2019 11:04:28 -0400 Subject: [PATCH 028/131] SPARK-2244 Suppress empty updates --- src/lib/md/MBOServiceServer.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index d4c7e3261..acc9aa035 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -175,8 +175,7 @@ bool update_position(const HandleIndex& handles, ActiveOrder& order) { search_result = search_orders(order.exchange_id(), mbo->asks); } if (search_result) { - order.set_position(search_result.value()); - return true; + return order.set_position(search_result.value()); } } } @@ -191,8 +190,9 @@ void MBOServiceImpl::timer_tick() { rows.push_back(pair.second.to_piq()); } } - - publish(nullptr, &rows, nullptr); + if (!rows.empty()) { + publish(nullptr, &rows, nullptr); + } } const NativeOrderState* native_order_state(const OrderRecord& record) { From f7fd81f3af8c3f6bb8d66a03f1946a90df5a7f88 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Apr 2019 14:47:50 -0400 Subject: [PATCH 029/131] Improve Servicetail --- src/client/sparkcmd/Cmd/ServiceTail.cs | 20 +++++++++++++++++++- src/client/sparkcmd/sparkcmd.csproj | 2 ++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/client/sparkcmd/Cmd/ServiceTail.cs b/src/client/sparkcmd/Cmd/ServiceTail.cs index 7ebbb0c92..2820bbef6 100644 --- a/src/client/sparkcmd/Cmd/ServiceTail.cs +++ b/src/client/sparkcmd/Cmd/ServiceTail.cs @@ -8,8 +8,15 @@ using System.Text; using System.Threading.Tasks; namespace bts { + + enum PrintFormat { + ToString, + Json + } class TailService : ConsoleCommand { static bool _all = false; + static PrintFormat _format = PrintFormat.ToString; + static bool _timestamp = false; public TailService() { void ignore(string _) { } @@ -19,6 +26,7 @@ namespace bts { // The following are not implemented. If you need them get to work. HasOption("path", "Path inside the row to use for display. Ex: 'Id' or 'Name' ", ignore); HasOption("all", "Invlude 'all' rows on every print", _ => _all = true); + HasOption("json", "print as json", _ => _format = PrintFormat.Json); HasAdditionalArguments(null, " [nothing]"); } public override int Run(string[] remainingArguments) { @@ -46,7 +54,17 @@ namespace bts { Console.Write(" "); } if (_all) { } - Console.WriteLine(value); + + switch(_format) { + case PrintFormat.Json: + dynamic dv = value; + var json = Newtonsoft.Json.JsonConvert.SerializeObject(new { dv.added, dv.removed, dv.changed }); + Console.WriteLine(json); + break; + case PrintFormat.ToString: + Console.WriteLine(value); + break; + } } } private static void tail(string service_name, string method_name) { diff --git a/src/client/sparkcmd/sparkcmd.csproj b/src/client/sparkcmd/sparkcmd.csproj index 9046c4610..b0132eaed 100644 --- a/src/client/sparkcmd/sparkcmd.csproj +++ b/src/client/sparkcmd/sparkcmd.csproj @@ -23,5 +23,7 @@ + + From a376fe9e22b0b15c77f83203d2a310b496d3f09e Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 12 Apr 2019 09:56:28 -0400 Subject: [PATCH 030/131] Fix pool_stats crash for the pool paupers --- src/lib/md/MBOProcessor.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 4d5f2a33d..4fec20880 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -35,6 +35,8 @@ void print_pool_stats(const utils::ConcurrentPool& pool) { } }); + if (count == 0) return; + std::sort(bid_sz.begin(), bid_sz.end()); std::sort(ask_sz.begin(), ask_sz.end()); size_t bid_tot =0, ask_tot = 0; From d89f93c323cc9d72e76b186a135ec6863bd0df4f Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Mon, 15 Apr 2019 11:05:40 -0500 Subject: [PATCH 031/131] Adding MBOService as an export in gui --- src/client/spark/Daemons/ShellServices.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/spark/Daemons/ShellServices.cs b/src/client/spark/Daemons/ShellServices.cs index 8fa232571..bcfd91e6e 100644 --- a/src/client/spark/Daemons/ShellServices.cs +++ b/src/client/spark/Daemons/ShellServices.cs @@ -6,6 +6,7 @@ using bts.api.persistence; using bts.api.position; using bts.api.synthetic; using bts.co2; +using bts.md; using bts.metrics; using bts.support; using System.ComponentModel.Composition; @@ -50,6 +51,9 @@ namespace Spark { AlgoServiceWrapper _algo_service_wrapper; [Export] public AlgoServiceWrapper algo_service_wrapper => _algo_service_wrapper ?? (_algo_service_wrapper = new AlgoServiceWrapper(context)); + + [Export] + public MBOService mbo_service => context.locate(); } class AlgoServiceWrapper { From 8130d5a0f86832117972c9d703396e14bd5e6b73 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:06:32 -0500 Subject: [PATCH 032/131] adding render_piq method --- src/client/spark/Orders/OrderSecPiq.cs | 17 +++++++++++++ .../spark/view/Spark/Ladder/LaderRender.cs | 24 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 src/client/spark/Orders/OrderSecPiq.cs diff --git a/src/client/spark/Orders/OrderSecPiq.cs b/src/client/spark/Orders/OrderSecPiq.cs new file mode 100644 index 000000000..77f7177d6 --- /dev/null +++ b/src/client/spark/Orders/OrderSecPiq.cs @@ -0,0 +1,17 @@ +using bts.api; +using bts.api.orderman; + + +namespace mbts.spark { + public struct OrderSecPiq { + public SecurityImmutable? sec; + public OrderRecordImmutable order; + public uint? piq; + + public OrderSecPiq(SecurityImmutable? sec_imm, OrderRecordImmutable ori, uint? placement_in_queue) { + sec = sec_imm; + order = ori; + piq = placement_in_queue; + } + } +} diff --git a/src/client/spark/view/Spark/Ladder/LaderRender.cs b/src/client/spark/view/Spark/Ladder/LaderRender.cs index 014176202..dfbc80d07 100644 --- a/src/client/spark/view/Spark/Ladder/LaderRender.cs +++ b/src/client/spark/view/Spark/Ladder/LaderRender.cs @@ -277,6 +277,30 @@ namespace mbts.spark { } } + // this is basically a copy of redner_sizes up above + internal static void render_piqs(DrawingVisual dv, GlyphContext glyph_context, int origin_index, double[] prices, (double price, uint piq)[] piqs, double row_height, ContractSpecificationImmutable spec, LadderColumnSettings settings) { + using (var dc = dv.RenderOpen()) { + using (_draw_with_clip(dc, settings, row_height, prices.Length)) { + if (!prices.within_range(origin_index) || piqs.is_null_or_empty()) + return; + var foreground = settings.foreground.rgb_brush(); + int curr_index = origin_index; + double curr_price = prices[curr_index]; + for (int i = 0; i < piqs.Length; i++) { + double price = piqs[i].price; + // prices are in descending order, so get diff from curr_price - price + int tick_diff = spec.tickdiff_unchecked(curr_price, price); + curr_index += tick_diff; + curr_price = price; + if (curr_index >= 0 && curr_index < prices.Length) { + var gr = format_text(piqs[i].piq.ToString(), curr_index, row_height, settings, glyph_context); + dc.DrawGlyphRun(foreground, gr); + } else break; + } + } + } + } + public static void render_mkt_break(DrawingVisual dv, int index, double row_height, uint color) => render_single_grid_line(dv, index, row_height, color, 0, SystemParameters.FullPrimaryScreenWidth); From 391397692c12430452edf2b6631244e1a03c3df6 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:08:28 -0500 Subject: [PATCH 033/131] adding PIQ case option --- src/client/spark/view/Spark/Ladder/LadderView.xaml.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs b/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs index 273bb4d9a..03e811d34 100644 --- a/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs +++ b/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs @@ -141,6 +141,10 @@ namespace mbts.spark { header = "Last"; width_prop = nameof(LadderVM.last_column_width); break; + case LadderGridHost.ColumnType.PIQ: + header = "PIQ"; + width_prop = nameof(LadderVM.piq_column_width); + break; default: throw ExceptionBuilder.Argument.unexpected_case(type, nameof(type)); } return _create(host, header, $"DataContext.{width_prop}"); From a646f288316787c3f9b5e647180319b1c67b0aae Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:14:05 -0500 Subject: [PATCH 034/131] added min_piq property; now stores (OrderRecordImmutable, uint?) instead of just OrderRecordImmutable --- .../spark/view/Spark/Ladder/LadderOrderVM.cs | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs b/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs index 792b59a84..2768b017b 100644 --- a/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs @@ -46,9 +46,9 @@ namespace mbts.spark { int working_quantity; LadderModel model => _parent.model; - Dictionary _orders = new Dictionary(); + Dictionary _orders = new Dictionary(); - public LadderOrderVM(OrderRecordImmutable o, double real_price, LadderVM parent, IBus bus, IObservable bs) { + public LadderOrderVM((OrderRecordImmutable order, uint? p) op, double real_price, LadderVM parent, IBus bus, IObservable bs) { _parent = parent; _bus = bus; _bs = bs; @@ -73,7 +73,7 @@ namespace mbts.spark { double get_font_size(LadderSettings s) => s.orders_column.font_size; Brush get_foreground(LadderSettings s) => s.orders_column.foreground.rgb_brush(); // - add_order(o); + add_order(op); _dispose = Disposable.Create( _bs.Subscribe(badge => { @@ -109,19 +109,19 @@ namespace mbts.spark { // if this badge only represents a single order, then is_exchange_order and the border (= algo color) // should be set appropriately if (_orders.Count == 1) { - var single_order = _orders.Values.First(); + var single_order = _orders.Values.First().order; var single_order_metadata = _parent.model.order_gateway.factory.get_metadata(single_order); border = single_order_metadata.Color.rgb_brush(); - is_exchange_order = OrderUtils.is_exchange_order(_orders.Values.First()); + is_exchange_order = OrderUtils.is_exchange_order(_orders.Values.First().order); } else { border = new SolidColorBrush(color); is_exchange_order = false; } has_multiple_orders_at_same_level = _orders.Count > 1; - is_hedge_order = _orders.Any(o => o.Value.sub_type() == OrderSubType.HedgingLeg); + is_hedge_order = _orders.Any(op => op.Value.order.sub_type() == OrderSubType.HedgingLeg); if (_orders.Any()) { - summary = _orders.Values.Select(order => order.short_description(_parent.security.ContractSpec)).Aggregate((s1, s2) => $"{s1}{Environment.NewLine}{Environment.NewLine}{s2}"); + summary = _orders.Values.Select(op => op.order.short_description(_parent.security.ContractSpec)).Aggregate((s1, s2) => $"{s1}{Environment.NewLine}{Environment.NewLine}{s2}"); } _set_badge_text(); @@ -132,8 +132,8 @@ namespace mbts.spark { RaisePropertyChanged(nameof(text_lines)); } - public void add_order(OrderRecordImmutable new_order) { - _orders[new_order.Key] = new_order; + public void add_order((OrderRecordImmutable order, uint? p) new_op) { + _orders[new_op.order.Key] = new_op; } public void remove_order(uint order_key) { @@ -149,10 +149,11 @@ namespace mbts.spark { public uint security_key => _parent.security.Id; public double price { get => double.IsNaN(_phantom_price) ? _real_price : _phantom_price; } + public uint? min_piq => _orders.Select(op => op.Value.p).Where(piq => piq.HasValue).Min(); public double real_price => _real_price; public Side side { get { - var distinct_sides = _orders.Values.Select(o => o.side()).Distinct(); + var distinct_sides = _orders.Values.Select(op => op.order.side()).Distinct(); if (distinct_sides.Count() > 1) { return Side.Undefined; } @@ -160,11 +161,11 @@ namespace mbts.spark { } } - IEnumerable _parent_and_child_not_sharing_slot() => _orders.Values.Where(o => !_orders.ContainsKey(o.ParentKey)); + IEnumerable _parent_and_child_not_sharing_slot() => _orders.Values.Select(op => op.order).Where(o => !_orders.ContainsKey(o.ParentKey)); public IReadOnlyList text_lines { get; private set; } - public bool has_parent => _orders.Values.Any( order => order.has_parent()); + public bool has_parent => _orders.Values.Any( op => op.order.has_parent()); public bool has_phantom_price => !double.IsNaN(_phantom_price); public ICommand detach_command => new DelegateCommand(() => _execute_command(_crack_order), () => !has_parent); @@ -191,16 +192,16 @@ namespace mbts.spark { // the setter is public so that the LadderVM can set this field public CustomOrderDefinition.CustomOneofCase parent_algo_type { get => _parent_algo_type.Value; set => _parent_algo_type.Value = value; } - public IEnumerable orders => _orders.Values; + public IEnumerable orders => _orders.Values.Select(o => o.order); void _cancel_order() { - foreach (var order in _orders.Values) { - model.order_gateway.cancel_order(order).ignore(); + foreach (var op in _orders.Values) { + model.order_gateway.cancel_order(op.order).ignore(); } } void _crack_order() { - var distinct_parent_keys = _orders.Values.Where(o => o.has_parent()).Select(x => x.ParentKey).Distinct(); + var distinct_parent_keys = _orders.Values.Where(op => op.order.has_parent()).Select(x => x.order.ParentKey).Distinct(); model.order_gateway.crack_orders(distinct_parent_keys).ignore(); } @@ -279,7 +280,7 @@ namespace mbts.spark { } public override string ToString() { - return $"Price: {this.price} Order Count: {this.orders.Count()}"; + return $"Price: {this.price} Order Count: {this.orders.Count()} PIQ: {(this.min_piq.HasValue ? this.min_piq.Value.ToString() : "")}"; } } } From 096bf3338c49a9a7c59ef1d5d3d3ef6837ac2af0 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:15:06 -0500 Subject: [PATCH 035/131] adding PIQ --- src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs b/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs index 3c4d7869d..43999f483 100644 --- a/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs +++ b/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs @@ -97,13 +97,13 @@ namespace mbts.ui.controls { TotalPnL = "Total PnL", PnLPrice = "PnL Price", Cancel = "Cancel", - MarketDescription = "Market Description", ClearedAlias = "Cleared Alias", ProductDescription = "Product Description", HubAlias = "Hub Alias", StripName = "Strip Name", - MicCode = "Mic Code" + MicCode = "Mic Code", + PositionInQueue = "PIQ" ; } From fa287b14ef266b562efbe2c7b72a631b0787dd1f Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:16:40 -0500 Subject: [PATCH 036/131] OrderSecurity now also has a piq; maybe we change the class name? --- src/client/spark/Orders/OrderSecurity.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/client/spark/Orders/OrderSecurity.cs b/src/client/spark/Orders/OrderSecurity.cs index 174d83344..eb46dc351 100644 --- a/src/client/spark/Orders/OrderSecurity.cs +++ b/src/client/spark/Orders/OrderSecurity.cs @@ -4,21 +4,24 @@ using System; namespace bts.api.orderman { public readonly struct OrderSecurity: IEquatable { - public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order) { + public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order, uint? piq) { this.security = sec; this.order = order; + this.piq = piq; } - public OrderSecurity((SecurityImmutable? sec, OrderRecordImmutable order) t) { + public OrderSecurity(OrderSecPiq t) { security = t.sec; order = t.order; + piq = t.piq; } - public static implicit operator OrderSecurity(ValueTuple x) => new OrderSecurity(x); + public static implicit operator OrderSecurity(OrderSecPiq x) => new OrderSecurity(x); public SecurityImmutable? security { get; } public OrderRecordImmutable order { get; } + public uint? piq { get; } public override bool Equals(object obj) => (obj is OrderSecurity os) && Equals(os); - public bool Equals(OrderSecurity other) => order.Equals(other.order) && security == other.security; - public override int GetHashCode() => (security, order).GetHashCode(); + public bool Equals(OrderSecurity other) => order.Equals(other.order) && security == other.security && piq == other.piq; + public override int GetHashCode() => (security, order, piq).GetHashCode(); public static bool operator == (OrderSecurity left, OrderSecurity right) => left.Equals(right); public static bool operator != (OrderSecurity left, OrderSecurity right) => !left.Equals(right); From d9b4e21c6156d2cdcaa3ed7f0e45a43b20e2a8c0 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:20:09 -0500 Subject: [PATCH 037/131] changing from tuple to simple OrderSecPiq for sanity --- src/client/spark/Daemons/OrderTicketDaemon.cs | 2 +- src/client/spark/Notifications/INotificationService.cs | 2 +- src/client/spark/Notifications/INotificationServiceDataCache.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/spark/Daemons/OrderTicketDaemon.cs b/src/client/spark/Daemons/OrderTicketDaemon.cs index 84e473c46..fa6091f92 100644 --- a/src/client/spark/Daemons/OrderTicketDaemon.cs +++ b/src/client/spark/Daemons/OrderTicketDaemon.cs @@ -19,7 +19,7 @@ namespace bts.shell { readonly OrdersModel _model; readonly ErrorReporter _error_reporter; IReadOnlyDictionary _md_map; - TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> _orders; + TableUpdate _orders; IDisposable _dispose; [ImportingConstructor] diff --git a/src/client/spark/Notifications/INotificationService.cs b/src/client/spark/Notifications/INotificationService.cs index a95ac2048..27086e52c 100644 --- a/src/client/spark/Notifications/INotificationService.cs +++ b/src/client/spark/Notifications/INotificationService.cs @@ -55,7 +55,7 @@ namespace bts.shell { void _on_activity(Activity activity) => _activity_notifications.process_item(activity); - void _on_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable),uint> update) { + void _on_orders(TableUpdate update) { // must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history _data_cache.update(update.as_iincremental()); foreach (var o in update.added.Concat(update.changed)) diff --git a/src/client/spark/Notifications/INotificationServiceDataCache.cs b/src/client/spark/Notifications/INotificationServiceDataCache.cs index fac9deeed..00fb48eb0 100644 --- a/src/client/spark/Notifications/INotificationServiceDataCache.cs +++ b/src/client/spark/Notifications/INotificationServiceDataCache.cs @@ -50,7 +50,7 @@ namespace bts.shell { public DateTime last_order_reset_time { get; private set; } public bool is_first_order_update { get; private set; } - internal void update(IIncrementalUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> data) { + internal void update(IIncrementalUpdate data) { DateTime now = DateTime.Now; if (data.removed.Count == _order_cache.Count) { last_order_reset_time = now; From 9a39d84739e256b73be9bfecc5857866432d6f6e Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:20:42 -0500 Subject: [PATCH 038/131] tuple to OrderSecPiq in here too --- src/client/spark/Orders/OrderActivitySummarizer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/spark/Orders/OrderActivitySummarizer.cs b/src/client/spark/Orders/OrderActivitySummarizer.cs index 3bcb6de13..42e934664 100644 --- a/src/client/spark/Orders/OrderActivitySummarizer.cs +++ b/src/client/spark/Orders/OrderActivitySummarizer.cs @@ -27,7 +27,7 @@ namespace bts.spark { static readonly IEqualityComparer _id_comparer = bts.utils.EqualityComparer.Create(a => a.Key); - public OrderActivitySummarizer(IBus bus, IObservable> orders) { + public OrderActivitySummarizer(IBus bus, IObservable> orders) { _bus = bus; _dispose = Disposable.Create( bus.subscribe(SparkTags.SecuritiesTag).Subscribe(sec => _security = sec), @@ -124,7 +124,7 @@ namespace bts.spark { #endregion - void orders_updated(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) { + void orders_updated(TableUpdate update) { foreach (var o in update.added) { if (!string.IsNullOrEmpty(o.order.State.LastMsg) && o.order.State.Created.ToDateTime().Date == DateTime.Today) { From 567edafe187b271e397d371bffa77c5f73ccf479 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:23:29 -0500 Subject: [PATCH 039/131] including the piq here on this new --- .../spark/Predicates/Data/Events/OrderFilledEventPredicate.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs index f8ca8f2ab..8f542fd5a 100644 --- a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs @@ -82,7 +82,7 @@ namespace mbts { // old order will be a copy of new order, except with filledQuantity set to 0 var old_state = ort.order.State.CloneToMutable(); old_state.FilledQuantity = 0; - return (new OrderSecurity(ort.security, ort.order.with(state: old_state)), ort); + return (new OrderSecurity(ort.security, ort.order.with(state: old_state), ort.piq), ort); } } return null; From b2ed88fb4a200dcfe9f63c8a9a9b013353c21861 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:23:57 -0500 Subject: [PATCH 040/131] adding binding expression for new PIQColumnWidth property --- src/client/spark/view/Spark/Ladder/LadderView.xaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/spark/view/Spark/Ladder/LadderView.xaml b/src/client/spark/view/Spark/Ladder/LadderView.xaml index 889e8e8bd..dc8b3afc2 100644 --- a/src/client/spark/view/Spark/Ladder/LadderView.xaml +++ b/src/client/spark/view/Spark/Ladder/LadderView.xaml @@ -541,6 +541,7 @@ price_column_width="{Binding price_column_width, Mode=TwoWay}" ask_column_width="{Binding ask_column_width, Mode=TwoWay}" LastColumnWidth="{Binding last_column_width, Mode=TwoWay}" + PIQColumnWidth="{Binding piq_column_width, Mode=TwoWay}" VolumeColumnWidth="{Binding volume_column_width, Mode=TwoWay}" > From c312a880a7b992ff38b1cc6689c7592ed7342c4e Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:29:29 -0500 Subject: [PATCH 041/131] adding MBOService for piq, left joining piq on orders --- src/client/spark/Orders/OrdersModel.cs | 32 +++++++++++++++++++++----- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index f8a4cca8c..977eb721f 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -3,6 +3,7 @@ using bts.api; using bts.api.core; using bts.api.orderman; using bts.api.position; +using bts.md; using bts.spark; using bts.utils; using mbts.collections; @@ -16,8 +17,9 @@ using System.ComponentModel.Composition; using System.Linq; using System.Reactive.Concurrency; using System.Reactive.Linq; +using System.Reactive.Subjects; using System.Threading.Tasks; -using OrderUpdate = bts.api.core.TableUpdate<(bts.api.SecurityImmutable? sec, bts.api.orderman.OrderRecordImmutable order), uint>; +using OrderUpdate = bts.api.core.TableUpdate; namespace mbts.spark { [Export] @@ -26,6 +28,7 @@ namespace mbts.spark { readonly OrderFactory _factory; readonly OrderService _service; readonly PositionService _position_service; + readonly MBOService _mbo_service; readonly SpreaderModel _spreader; readonly ErrorReporter _error; readonly DynamicOrderHelper _dynamic_helper; @@ -35,12 +38,13 @@ namespace mbts.spark { IDisposable _dispose; [ImportingConstructor] - OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, + OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, MBOService mbo_service, SpreaderModel spreader, DynamicOrderHelper dynamic_helper, AdminModel admin_model, IAlgoStudioModel algo_model, SecurityStore sstore, ErrorReporter error) { _bus = bus; _factory = factory; _service = service; _position_service = position_service; + _mbo_service = mbo_service; _spreader = spreader; _error = error; _dynamic_helper = dynamic_helper; @@ -48,13 +52,26 @@ namespace mbts.spark { this.algo_model = algo_model; var scheduler = ThreadPoolScheduler.Instance; var interval = TimeSpan.FromMilliseconds(100); - on_orders = + on_orders = _service .subscribei_orders() .Naggle(interval, scheduler) .PermissionFilter(admin_model.on_admin_data) .join(sstore, order => order.security_key()) - .Table().Cache(); + .Table() + .Cache() + .LeftJoin< + (SecurityImmutable?, OrderRecordImmutable), + OrderPIQImmutable, + OrderSecPiq, + uint + >(mbo_service.subscribei_piq().Table(), + si_ort => si_ort.Item2.Key, + p => p.OrderId, + (si_ort, r, p) => new OrderSecPiq(si_ort.Item1, si_ort.Item2, p.Position), + (si_ort, r) => new OrderSecPiq(si_ort.Item1, si_ort.Item2, null), + osp => osp.order.Key); + on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); working_orders = on_orders.Scan( @@ -103,7 +120,9 @@ namespace mbts.spark { f => f.fill.FillKey); on_fill_tuples = fill_order_tuples_join; _dispose = Disposable.Create( - on_orders.Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), + // on_orders changed from an IObservable to an IConnectableObservable, so we call + // RefCount() here to start the connection + on_orders.RefCount().Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), on_positions.Subscribe(_security_positions_demuxer), order_secs_algo_meta_join.Connect(), algo_meta_join.Connect(), @@ -120,12 +139,13 @@ namespace mbts.spark { public OrderFactory factory => _factory; public SpreaderModel spreader => _spreader; public DynamicOrderHelper dynamic_helper => _dynamic_helper; - public IObservable on_orders { get; } + public IConnectableObservable on_orders { get; } public IObservable> on_order_tuples { get; } public IObservable> on_order_securities { get; } public IObservable> on_fill_tuples { get; } public IObservable> working_orders { get; } public IObservable> on_positions { get; } + public AdminModel admin_model { get; } public IAlgoStudioModel algo_model { get; } From d76c2a30ceb4a9d1cf53227929eee176a862fc50 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:32:54 -0500 Subject: [PATCH 042/131] price_vs_piq property on LadderVM; appropriate changes for LadderOrderVM constructor --- src/client/spark/view/Spark/Ladder/LadderVM.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index 5be07f911..0bf612489 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -26,7 +26,7 @@ namespace mbts.spark { readonly LadderModel _model; readonly Property _high, _low, _change, _open, _settlement, _position, _left_panel_width, _zoom_factor; //readonly Property _window_width, _window_height; - readonly Property _orders_column_width, _bid_column_width, _price_column_width, _ask_column_width, _last_column_width, _volume_column_width; + readonly Property _orders_column_width, _bid_column_width, _price_column_width, _ask_column_width, _last_column_width, _volume_column_width, _piq_column_width; readonly Property _volume; readonly Property _auto_clear, _show_day, _show_high, _show_low, _show_open, _show_settle, _show_volume, _show_account, _show_tif, _show_order_type, _show_account_position, _show_size_preset, _show_auto_clear, _show_clear, _show_cancel_buys, @@ -58,6 +58,7 @@ namespace mbts.spark { _price_column_width = backing(nameof(price_column_width), ws.price_column_width); _ask_column_width = backing(nameof(ask_column_width), ws.ask_column_width); _last_column_width = backing(nameof(last_column_width), ws.last_column_width); + _piq_column_width = backing(nameof(piq_column_width), ws.piq_column_width); _volume_column_width = backing(nameof(volume_column_width), ws.volume_column_width); _left_panel_width = backing(nameof(left_panel_width), ws.order_entry_panel_width); _zoom_factor = backing(nameof(zoom_factor), ws.zoom_factor); @@ -157,9 +158,9 @@ namespace mbts.spark { } if (_slot_price_vs_orders.TryGetValue(slot_price_target, out LadderOrderVM slot_vm)) { - slot_vm.add_order(wo); + slot_vm.add_order((wo, osec.piq)); } else { - slot_vm = new LadderOrderVM(wo, slot_price_target, this, _bus, _settings_manager.subscribe_current().Select(s => s.badge_style)); + slot_vm = new LadderOrderVM((wo, osec.piq), slot_price_target, this, _bus, _settings_manager.subscribe_current().Select(s => s.badge_style)); _slot_price_vs_orders[slot_price_target] = slot_vm; } @@ -232,6 +233,7 @@ namespace mbts.spark { public event Action on_tick; public MDConsumerHandle md => _model.md; + public IEnumerable<(double, uint)> price_vs_piq => _slot_price_vs_orders.Values.Where(result => result.min_piq.HasValue).Select(o => (o.price, o.min_piq.Value)); public bool is_synthetic => _model.security.ContractCase == Security.ContractOneofCase.Synthetic; Property _status; @@ -306,13 +308,14 @@ namespace mbts.spark { public uint price_column_width { get => _price_column_width.Value; set => _price_column_width.set(value); } public uint ask_column_width { get => _ask_column_width.Value; set => _ask_column_width.set(value); } public uint last_column_width { get => _last_column_width.Value; set => _last_column_width.set(value); } + public uint piq_column_width { get => _piq_column_width.Value; set => _piq_column_width.set(value); } public uint volume_column_width { get => _volume_column_width.Value; set => _volume_column_width.set(value); } internal LadderModel model => _model; internal IObservable settings => _settings_manager.subscribe_current(); public LadderSettings get_save_state() { var state = _settings_manager.current_settings(); - state = state.with_window_specifics(new LadderSettings.WindowSpecifics(left_panel_width, orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width, zoom_factor)); + state = state.with_window_specifics(new LadderSettings.WindowSpecifics(left_panel_width, orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width, zoom_factor, piq_column_width)); state.use_defaults = _settings_manager.instance_settings == null; return state; } From 42feedfd36e1ebd95aa7ee0b41961c25d4202d5b Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:33:48 -0500 Subject: [PATCH 043/131] adding piq column to settings --- .../spark/view/Spark/Ladder/LadderSettings.cs | 19 +++++++++++++++---- .../view/Spark/Ladder/LadderSettingsVM.cs | 8 +++++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderSettings.cs b/src/client/spark/view/Spark/Ladder/LadderSettings.cs index 6d1784488..e950d5587 100644 --- a/src/client/spark/view/Spark/Ladder/LadderSettings.cs +++ b/src/client/spark/view/Spark/Ladder/LadderSettings.cs @@ -249,7 +249,8 @@ namespace mbts.spark { bid_column = LadderColumnSettings.Default, ask_column = LadderColumnSettings.Default, price_column = LadderColumnSettings.Default, - last_column = LadderColumnSettings.Default; + last_column = LadderColumnSettings.Default, + piq_column = LadderColumnSettings.Default; orders_column.background = BTSColors.LadderSpecial.default_column_background; bid_column.foreground = 0xFFFFFF; bid_column.background = BTSColors.MarketData.bid; @@ -257,6 +258,7 @@ namespace mbts.spark { ask_column.background = BTSColors.MarketData.ask; price_column.background = BTSColors.LadderSpecial.default_column_background; last_column.background = 0xFFFFFF; + piq_column.background = 0xFFFFFF; return new LadderSettings { row_height = 20, grid_line_color = 0x535353, @@ -265,6 +267,7 @@ namespace mbts.spark { ask_column = ask_column, price_column = price_column, last_column = last_column, + piq_column = piq_column, volume_column = VolumeColumnSettings.Default, calculator_buttons = new[] { 1, 5, 10, 25, 50, 100 }, scroll_bar_visible = false, @@ -417,6 +420,9 @@ namespace mbts.spark { [ProtoMember(48)] public NetChangeTerms net_change { get; set; } + [ProtoMember(49)] + public LadderColumnSettings piq_column { get; set; } + public bool Equals(LadderSettings other) => other != null && row_height == other.row_height && @@ -426,6 +432,7 @@ namespace mbts.spark { ask_column.safe_equals(other.ask_column) && price_column.safe_equals(other.price_column) && last_column.safe_equals(other.last_column) && + piq_column.safe_equals(other.piq_column) && calculator_buttons.safe_equals(other.calculator_buttons, (a, b) => a.SequenceEqual(b)) && scroll_bar_visible == other.scroll_bar_visible && default_order_type == other.default_order_type && @@ -476,6 +483,7 @@ namespace mbts.spark { EqualityComparer.safe_get_hash_code(ask_column), EqualityComparer.safe_get_hash_code(price_column), EqualityComparer.safe_get_hash_code(last_column), + EqualityComparer.safe_get_hash_code(piq_column), EqualityComparer.safe_get_hash_code(calculator_buttons), scroll_bar_visible.GetHashCode(), default_order_type.GetHashCode(), @@ -517,12 +525,12 @@ namespace mbts.spark { public struct WindowSpecifics { public readonly double order_entry_panel_width, zoom_factor; - public readonly uint orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width; + public readonly uint orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width, piq_column_width; public WindowSpecifics(LadderSettings s) - : this(s.order_entry_panel_initial_width, s.orders_column.width, s.bid_column.width, s.price_column.width, s.ask_column.width, s.last_column.width, s.volume_column.general.width, s.zoom_factor) { } + : this(s.order_entry_panel_initial_width, s.orders_column.width, s.bid_column.width, s.price_column.width, s.ask_column.width, s.last_column.width, s.volume_column.general.width, s.zoom_factor, s.piq_column.width) { } - public WindowSpecifics(double order_entry_panel_width, uint orders_column_width, uint bid_column_width, uint price_column_width, uint ask_column_width, uint last_column_width, uint volume_column_width, double zoom_factor) { + public WindowSpecifics(double order_entry_panel_width, uint orders_column_width, uint bid_column_width, uint price_column_width, uint ask_column_width, uint last_column_width, uint volume_column_width, double zoom_factor, uint piq_column_width) { this.order_entry_panel_width = order_entry_panel_width; this.orders_column_width = orders_column_width; this.bid_column_width = bid_column_width; @@ -531,6 +539,7 @@ namespace mbts.spark { this.last_column_width = last_column_width; this.volume_column_width = volume_column_width; this.zoom_factor = zoom_factor; + this.piq_column_width = piq_column_width; } } } @@ -543,6 +552,7 @@ namespace mbts.spark { yield return settings.ask_column; yield return settings.volume_column.general; yield return settings.last_column; + yield return settings.piq_column; } public static IEnumerable visible_columns(this LadderSettings settings) => @@ -560,6 +570,7 @@ namespace mbts.spark { source.price_column.width = specifics.price_column_width; source.ask_column.width = specifics.ask_column_width; source.last_column.width = specifics.last_column_width; + source.piq_column.width = specifics.piq_column_width; source.volume_column.general.width = specifics.volume_column_width; source.zoom_factor = specifics.zoom_factor; return source; diff --git a/src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs b/src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs index b3f9bed8b..295aa6739 100644 --- a/src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs @@ -29,7 +29,8 @@ namespace mbts.spark { _bid = new LadderColumnSettingsVM("Bid"), _price = new LadderColumnSettingsVM("Price"), _ask = new LadderColumnSettingsVM("Ask"), - _last = new LadderColumnSettingsVM("Last", true); + _last = new LadderColumnSettingsVM("Last", true), + _piq = new LadderColumnSettingsVM("PIQ", true); VolumeColumnVM _volume = new VolumeColumnVM(); LadderMarketBreakLineVM _bid_mkt_line = new LadderMarketBreakLineVM("Bid"), @@ -82,6 +83,7 @@ namespace mbts.spark { _price.font_size_changed += on_font_size_changed; _ask.font_size_changed += on_font_size_changed; _last.font_size_changed += on_font_size_changed; + _piq.font_size_changed += on_font_size_changed; void on_font_size_changed(double oldval, double newval) => _row_height.refresh(); } @@ -120,6 +122,7 @@ namespace mbts.spark { if (!is_synthetic) { yield return _volume.general; yield return _last; + yield return _piq; } } } @@ -130,6 +133,7 @@ namespace mbts.spark { public LadderColumnSettingsVM ask_column => _ask; public LadderColumnSettingsVM volume_column => _volume.general; public LadderColumnSettingsVM last_column => _last; + public LadderColumnSettingsVM piq_column => _piq; public VolumeColumnVM volume => _volume; public LadderMarketBreakLineVM session_range_fill => _session_range_fill; public ObservableEditableCollection> size_presets { get; } @@ -209,6 +213,7 @@ namespace mbts.spark { _price.assign_from(_current.price_column); _volume.assign_from(_current.volume_column); _last.assign_from(_current.last_column); + _piq.assign_from(_current.piq_column); _bid_mkt_line.assign_from(_current.bid_break_line); _ask_mkt_line.assign_from(_current.ask_break_line); _mid_mkt_line.assign_from(_current.mid_break_line); @@ -254,6 +259,7 @@ namespace mbts.spark { clone.ask_column = _ask.get_settings(); clone.volume_column = _volume.get_settings(); clone.last_column = _last.get_settings(); + clone.piq_column = _piq.get_settings(); clone.bid_break_line = _bid_mkt_line.get_settings(); clone.ask_break_line = _ask_mkt_line.get_settings(); clone.mid_break_line = _mid_mkt_line.get_settings(); From c32eada3aca17bfe39a8293cc39399a86f022750 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:49:03 -0500 Subject: [PATCH 044/131] the big enchilada - adding PIQ column to LadderGridHost --- .../spark/view/Spark/Ladder/LadderGridHost.cs | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderGridHost.cs b/src/client/spark/view/Spark/Ladder/LadderGridHost.cs index 2f91999d2..516ccf0f8 100644 --- a/src/client/spark/view/Spark/Ladder/LadderGridHost.cs +++ b/src/client/spark/view/Spark/Ladder/LadderGridHost.cs @@ -42,7 +42,8 @@ namespace mbts.spark { _ask_vis_tup = new VisualGlyphTuple(), _price_vis_tup = new VisualGlyphTuple(), _last_vis_tup = new VisualGlyphTuple(), - _volume_vis_tup = new VisualGlyphTuple(); + _volume_vis_tup = new VisualGlyphTuple(), + _piq_vis_tup = new VisualGlyphTuple(); readonly RefValueContainer> _volume_container = new RefValueContainer>(); @@ -92,7 +93,7 @@ namespace mbts.spark { }; } - public enum ColumnType { Order, Bid, Price, Ask, Last, Volume, None } + public enum ColumnType { Order, Bid, Price, Ask, Last, Volume, None, PIQ } public event Action visible_columns_changed; @@ -192,6 +193,18 @@ namespace mbts.spark { set => SetValue(LastColumnWidthProperty, value); } + public static readonly DependencyProperty PIQColumnWidthProperty = + WPFHelper.create_dp( + nameof(PIQColumnWidth), + FrameworkPropertyMetadataOptions.BindsTwoWayByDefault, + (o, oldval, newval) => o._adjust_column_width(o.columns.FirstOrDefault(c => c.column_type == ColumnType.PIQ), (double)newval - (double)oldval), + 50u); + + public uint PIQColumnWidth { + get => (uint)GetValue(PIQColumnWidthProperty); + set => SetValue(PIQColumnWidthProperty, value); + } + #region Command public ICommand Command { get => (ICommand)GetValue(CommandProperty); @@ -404,7 +417,8 @@ namespace mbts.spark { double x = 0; var col_list = new List { new Column(_settings.orders_column, ColumnType.Order, null, x), - new Column(_settings.bid_column, ColumnType.Bid, _bid_vis_tup, (x += _settings.orders_column.show ? _settings.orders_column.width : 0)), + new Column(_settings.piq_column, ColumnType.PIQ, _piq_vis_tup, (x += _settings.orders_column.show ? _settings.orders_column.width : 0)), + new Column(_settings.bid_column, ColumnType.Bid, _bid_vis_tup, (x += _settings.piq_column.show ? _settings.piq_column.width : 0)), (_price_column = new Column(_settings.price_column, ColumnType.Price, _price_vis_tup, (x += _settings.bid_column.show ? _settings.bid_column.width : 0))), new Column(_settings.ask_column, ColumnType.Ask, _ask_vis_tup, (x += _settings.price_column.show ? _settings.price_column.width : 0)) }; @@ -509,6 +523,8 @@ namespace mbts.spark { if (depth_changed) _refresh_sizes(); } + // TODO - detect changes in PIQ to selectively redraw + _refresh_piq(); _refresh_volume_at_price(); _last_origin_price = _mkt_break_lines.origin; _price_dirty = false; @@ -545,6 +561,14 @@ namespace mbts.spark { LadderRender.render_last(_last_vis_tup.visual, _last_vis_tup.glyph_context, _last_trade, last_index, _settings.row_height, _settings.last_column); } + void _refresh_piq() { + if (!_settings.piq_column.show) + return; + int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer); + var piq_list = _vm.price_vs_piq; + LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, origin_index, _price_list, piq_list.ToArray(), _settings.row_height, security.ContractSpec, _settings.piq_column); + } + void _refresh_sizes() { Log.assert_d(_price_list != null); int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer); @@ -693,6 +717,7 @@ namespace mbts.spark { _ask_vis_tup.glyph_context = glyph_context(s.ask_column); _price_vis_tup.glyph_context = glyph_context(s.price_column); _last_vis_tup.glyph_context = glyph_context(s.last_column); + _piq_vis_tup.glyph_context = glyph_context(s.piq_column); _volume_vis_tup.glyph_context = glyph_context(s.volume_column.general); GlyphContext glyph_context(LadderColumnSettings cs) => @@ -720,6 +745,7 @@ namespace mbts.spark { case ColumnType.Price: SetCurrentValue(price_column_widthProperty, col.settings.width); break; case ColumnType.Ask: SetCurrentValue(ask_column_widthProperty, col.settings.width); break; case ColumnType.Last: SetCurrentValue(LastColumnWidthProperty, col.settings.width); break; + case ColumnType.PIQ: SetCurrentValue(PIQColumnWidthProperty, col.settings.width); break; case ColumnType.Volume: SetCurrentValue(VolumeColumnWidthProperty, col.settings.width); _volume_container.mark_dirty(); From 6181548902e40239814b607db626eaf8578fbf9c Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 10:56:31 -0500 Subject: [PATCH 045/131] piq background color now matches orders background color --- src/client/spark/view/Spark/Ladder/LadderSettings.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderSettings.cs b/src/client/spark/view/Spark/Ladder/LadderSettings.cs index e950d5587..329082867 100644 --- a/src/client/spark/view/Spark/Ladder/LadderSettings.cs +++ b/src/client/spark/view/Spark/Ladder/LadderSettings.cs @@ -258,7 +258,7 @@ namespace mbts.spark { ask_column.background = BTSColors.MarketData.ask; price_column.background = BTSColors.LadderSpecial.default_column_background; last_column.background = 0xFFFFFF; - piq_column.background = 0xFFFFFF; + piq_column.background = BTSColors.LadderSpecial.default_column_background; return new LadderSettings { row_height = 20, grid_line_color = 0x535353, From 905c1cab2ce40b09214b60240834702e58b22fe7 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 16 Apr 2019 10:08:50 -0400 Subject: [PATCH 046/131] SPARK-2206 tests --- src/lib/md/test/mbo_tests.cpp | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/lib/md/test/mbo_tests.cpp diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp new file mode 100644 index 000000000..70c4492e9 --- /dev/null +++ b/src/lib/md/test/mbo_tests.cpp @@ -0,0 +1,27 @@ +#ifndef _WIN32 +#include "catch/catch.hpp" +#include "../MBOProcessor.h" +namespace bts { +namespace md { +namespace test { +int mbo_tests = 0; + +TEST_CASE("mbo basic", "[md][mbo]") { + MBOProcessor p; + // publish a snapshot + auto sec = SecurityKey(99); + { + auto s = p.writer().allocate_snapshot(); + s->bids.push_back({ ExchangeOrderKey(1), { 100, 0 }, 42 }); + p.writer().publish_snapshot(sec, s); + } + + auto s = p.reader().acquire_snapshot(sec); + REQUIRE(s); + REQUIRE(s->bids.size() == 1); + REQUIRE(s->bids[0].id.value() == 1); + REQUIRE(s->asks.size() == 0); +} + +}}} +#endif \ No newline at end of file From d7850d6b7040627f1600d5a30d3aa9caa311df27 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 16 Apr 2019 12:14:15 -0400 Subject: [PATCH 047/131] MBO diagnostic api --- src/client/sparkcmd/Cmd/ServiceTestCommand.cs | 56 +++++++++---------- src/lib/api/services/MBOService.proto | 20 ++++++- src/lib/md/MBOServiceServer.cpp | 28 ++++++++++ src/lib/md/MBOServiceServer.h | 2 + 4 files changed, 75 insertions(+), 31 deletions(-) diff --git a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs index 1f449cd4d..daf94bc80 100644 --- a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs +++ b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs @@ -10,6 +10,7 @@ using System.Reactive.Linq; using System.Threading; using System.Threading.Tasks; using System.Linq; +using bts.md; namespace bts { public class ServiceTest : ConsoleCommand { @@ -22,6 +23,7 @@ namespace bts { HasOption("synthetic", "Run synthetic test", _ => _synthetic = true); HasOption("user", "user test", _ => _user = true); HasOption("admin", "admin test", _ => _admin = true); + HasAdditionalArguments(null, " [Keys to check]"); } public override int Run(string[] remainingArguments) { if (_synthetic) { @@ -34,41 +36,37 @@ namespace bts { admin_test(); } else { - // mbo_test(); + mbo_test(remainingArguments); } return 0; } + static string try_get(string[] args, int index) { + if (args.Length < index) { + return args[index]; + } + return ""; + } - // private static void mbo_test() { - // using (var ctx = new ServiceContext()) { - // var relay_svc = ctx.locate(); - // var order_sv = ctx.locate(); + private static void mbo_test(string[] args) { + var key = UInt32.Parse(args.FirstOrDefault()); + var side = try_get(args, 1) == "bid" ? api.Side.Buy : api.Side.Sell; + using (var ctx = new ServiceContext()) { + var mbo = ctx.locate(); + var data = mbo.get_orders_for_security(new MBOSecurityQuery { SecurityId = key }); - // var upd = order_sv.subscribei_orders().FirstAsync().Wait(); /* don't use Wait() */ - // var orders = - // upd.all - // .Where(o => - // o.State.Status == OrderStatus.Working && - // o.State.native_order_state().HasValue) - // .ToList(); - - // var request = new PIQRequest(); - // request.Orders.AddRange(orders.Select(o => - // new OrderQuery { - // SecurityId = o.Def.SecurityKey, - // OrderId = UInt64.Parse(o.State.native_order_state().Value.ExchangeOrderId) })); - - // Console.WriteLine($"Requesting positions for {request.Orders.Count} orders..."); - // var result = relay_svc.get_piq(request); - - // Console.WriteLine($"got response - {result.Position.Count} entries"); - // foreach (var kvp in result.Position) { - // Console.Write($"> {kvp.Key} => "); - // ConsoleEx.WriteLine($"{kvp.Value}", ConsoleColor.Yellow); - // } - // } - // } + Console.WriteLine($"Order data for {data.SecurityId}"); + var vector = side == api.Side.Buy ? data.Bid : data.Ask; + var price = 0.0; + foreach(var e in vector) { + if (e.Price != price) { + price = e.Price; + Console.WriteLine("================"); + } + Console.WriteLine(e); + } + } + } private static void synthetic_test() { using (var ctx = new ServiceContext()) { diff --git a/src/lib/api/services/MBOService.proto b/src/lib/api/services/MBOService.proto index 4a8383f8b..8250375cc 100644 --- a/src/lib/api/services/MBOService.proto +++ b/src/lib/api/services/MBOService.proto @@ -6,10 +6,26 @@ import "lib/api/apioptions.proto"; import "lib/api/apicore.proto"; message OrderPIQ { - uint32 order_id = 1 [(bts.api.key)=true]; - uint32 position = 2; + uint32 order_id = 1 [(bts.api.key)=true]; + uint32 position = 2; +} + +message OrderPosition { + uint64 exchange_id = 1; + float price = 2; + int32 size = 3; +} +message PositionData { + uint32 security_id = 1; + repeated OrderPosition bid = 10; + repeated OrderPosition ask = 20; +} +message MBOSecurityQuery { + uint32 security_id = 1; } service MBOService { rpc piq (bts.api.VoidMessage) returns (stream OrderPIQ); + rpc get_orders_for_security (MBOSecurityQuery) returns (PositionData); + } \ No newline at end of file diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index acc9aa035..a7f8c4bb7 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -293,6 +293,34 @@ MBOServiceServer::MBOServiceServer( MBOServiceServer::~MBOServiceServer() { } +void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request, const api::Response& reply) { + PositionData response; + response.set_security_id(request.security_id()); + auto handle = _me->handle_index.index.lock([id=request.security_id()](const auto& index) { + auto iter = index.find(SecurityKey(id)); + return iter != index.end() ? iter->second : MDConsumerHandle {}; + }); + + if(handle.is_initialized()) { + auto mbo = handle.get_mbo(); + if (mbo) { + for (const auto& o : mbo->bids) { + auto* entry = response.add_bid(); + entry->set_exchange_id(o.id.value()); + entry->set_price(o.price.price()); + entry->set_size(o.size); + } + for (const auto& o : mbo->asks) { + auto* entry = response.add_ask(); + entry->set_exchange_id(o.id.value()); + entry->set_price(o.price.price()); + entry->set_size(o.size); + } + } + } + + reply(response); +} }} //namespace #endif \ No newline at end of file diff --git a/src/lib/md/MBOServiceServer.h b/src/lib/md/MBOServiceServer.h index 9e47aadfa..d0f590a18 100644 --- a/src/lib/md/MBOServiceServer.h +++ b/src/lib/md/MBOServiceServer.h @@ -20,6 +20,8 @@ class MBOServiceServer : public MBOServiceBase { md::MDClient&); ~MBOServiceServer(); + void get_orders_for_security(const MBOSecurityQuery&, const api::Response&) override; + }; }} //namespace From 8716099ac6a94a5ba39eb1a8eeaabf1e90b6df42 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 11:14:58 -0500 Subject: [PATCH 048/131] Removing OrderSecPiq struct since OrderSecurity does everything that did; we still may want to rename it down the line --- src/client/spark/Daemons/OrderTicketDaemon.cs | 6 +++--- .../spark/Notifications/INotificationService.cs | 2 +- .../INotificationServiceDataCache.cs | 2 +- .../spark/Orders/OrderActivitySummarizer.cs | 10 +++++----- src/client/spark/Orders/OrderSecPiq.cs | 17 ----------------- src/client/spark/Orders/OrderSecurity.cs | 5 ++--- src/client/spark/Orders/OrdersModel.cs | 16 ++++++++-------- 7 files changed, 20 insertions(+), 38 deletions(-) delete mode 100644 src/client/spark/Orders/OrderSecPiq.cs diff --git a/src/client/spark/Daemons/OrderTicketDaemon.cs b/src/client/spark/Daemons/OrderTicketDaemon.cs index fa6091f92..568a6b1ac 100644 --- a/src/client/spark/Daemons/OrderTicketDaemon.cs +++ b/src/client/spark/Daemons/OrderTicketDaemon.cs @@ -19,7 +19,7 @@ namespace bts.shell { readonly OrdersModel _model; readonly ErrorReporter _error_reporter; IReadOnlyDictionary _md_map; - TableUpdate _orders; + TableUpdate _orders; IDisposable _dispose; [ImportingConstructor] @@ -60,13 +60,13 @@ namespace bts.shell { } void _show_modify_order(uint order_key) { - if (_orders.state.TryGetValue(order_key, out var ori) && ori.sec.HasValue + if (_orders.state.TryGetValue(order_key, out var ori) && ori.security.HasValue && (ori.order.HasDef && ori.order.Def.HasCustomDef && !(ori.order.Def.CustomDef.HasStop || ori.order.Def.CustomDef.HasStopLimit))) { var md = _md_map.TryGetValue(ori.order.security_key()); _show_order_window(new OrderModificationDialog { DataContext = new ExistingOrderTicketVM(_model, Observable.Return((md,ori.order)), - Observable.Return(_model.factory.default_ladder_settings(ori.sec.Value).calculator_buttons) + Observable.Return(_model.factory.default_ladder_settings(ori.security.Value).calculator_buttons) ) }); } diff --git a/src/client/spark/Notifications/INotificationService.cs b/src/client/spark/Notifications/INotificationService.cs index 27086e52c..820e2ed72 100644 --- a/src/client/spark/Notifications/INotificationService.cs +++ b/src/client/spark/Notifications/INotificationService.cs @@ -55,7 +55,7 @@ namespace bts.shell { void _on_activity(Activity activity) => _activity_notifications.process_item(activity); - void _on_orders(TableUpdate update) { + void _on_orders(TableUpdate update) { // must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history _data_cache.update(update.as_iincremental()); foreach (var o in update.added.Concat(update.changed)) diff --git a/src/client/spark/Notifications/INotificationServiceDataCache.cs b/src/client/spark/Notifications/INotificationServiceDataCache.cs index 00fb48eb0..cf51f6e14 100644 --- a/src/client/spark/Notifications/INotificationServiceDataCache.cs +++ b/src/client/spark/Notifications/INotificationServiceDataCache.cs @@ -50,7 +50,7 @@ namespace bts.shell { public DateTime last_order_reset_time { get; private set; } public bool is_first_order_update { get; private set; } - internal void update(IIncrementalUpdate data) { + internal void update(IIncrementalUpdate data) { DateTime now = DateTime.Now; if (data.removed.Count == _order_cache.Count) { last_order_reset_time = now; diff --git a/src/client/spark/Orders/OrderActivitySummarizer.cs b/src/client/spark/Orders/OrderActivitySummarizer.cs index 42e934664..b085dddc9 100644 --- a/src/client/spark/Orders/OrderActivitySummarizer.cs +++ b/src/client/spark/Orders/OrderActivitySummarizer.cs @@ -27,7 +27,7 @@ namespace bts.spark { static readonly IEqualityComparer _id_comparer = bts.utils.EqualityComparer.Create(a => a.Key); - public OrderActivitySummarizer(IBus bus, IObservable> orders) { + public OrderActivitySummarizer(IBus bus, IObservable> orders) { _bus = bus; _dispose = Disposable.Create( bus.subscribe(SparkTags.SecuritiesTag).Subscribe(sec => _security = sec), @@ -124,12 +124,12 @@ namespace bts.spark { #endregion - void orders_updated(TableUpdate update) { + void orders_updated(TableUpdate update) { foreach (var o in update.added) { if (!string.IsNullOrEmpty(o.order.State.LastMsg) && o.order.State.Created.ToDateTime().Date == DateTime.Today) { - if (o.sec.HasValue) { - log_status(ActivityLogCategory.warn, o.sec.Value, o.order.State.LastMsg); + if (o.security.HasValue) { + log_status(ActivityLogCategory.warn, o.security.Value, o.order.State.LastMsg); } } } @@ -137,7 +137,7 @@ namespace bts.spark { foreach (var order in update.changes) { var neworder = order.New.order; var oldorder = order.Old.order; - var security = order.New.sec ?? order.Old.sec; + var security = order.New.security ?? order.Old.security; // // description var mod_desc = modified_description(oldorder, neworder, security?.ContractSpec); diff --git a/src/client/spark/Orders/OrderSecPiq.cs b/src/client/spark/Orders/OrderSecPiq.cs deleted file mode 100644 index 77f7177d6..000000000 --- a/src/client/spark/Orders/OrderSecPiq.cs +++ /dev/null @@ -1,17 +0,0 @@ -using bts.api; -using bts.api.orderman; - - -namespace mbts.spark { - public struct OrderSecPiq { - public SecurityImmutable? sec; - public OrderRecordImmutable order; - public uint? piq; - - public OrderSecPiq(SecurityImmutable? sec_imm, OrderRecordImmutable ori, uint? placement_in_queue) { - sec = sec_imm; - order = ori; - piq = placement_in_queue; - } - } -} diff --git a/src/client/spark/Orders/OrderSecurity.cs b/src/client/spark/Orders/OrderSecurity.cs index eb46dc351..71889e5a6 100644 --- a/src/client/spark/Orders/OrderSecurity.cs +++ b/src/client/spark/Orders/OrderSecurity.cs @@ -9,12 +9,11 @@ namespace bts.api.orderman { this.order = order; this.piq = piq; } - public OrderSecurity(OrderSecPiq t) { - security = t.sec; + public OrderSecurity(OrderSecurity t) { + security = t.security; order = t.order; piq = t.piq; } - public static implicit operator OrderSecurity(OrderSecPiq x) => new OrderSecurity(x); public SecurityImmutable? security { get; } public OrderRecordImmutable order { get; } public uint? piq { get; } diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 977eb721f..37a7d2d2b 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -19,7 +19,7 @@ using System.Reactive.Concurrency; using System.Reactive.Linq; using System.Reactive.Subjects; using System.Threading.Tasks; -using OrderUpdate = bts.api.core.TableUpdate; +using OrderUpdate = bts.api.core.TableUpdate; namespace mbts.spark { [Export] @@ -63,13 +63,13 @@ namespace mbts.spark { .LeftJoin< (SecurityImmutable?, OrderRecordImmutable), OrderPIQImmutable, - OrderSecPiq, + OrderSecurity, uint >(mbo_service.subscribei_piq().Table(), si_ort => si_ort.Item2.Key, p => p.OrderId, - (si_ort, r, p) => new OrderSecPiq(si_ort.Item1, si_ort.Item2, p.Position), - (si_ort, r) => new OrderSecPiq(si_ort.Item1, si_ort.Item2, null), + (si_ort, r, p) => new OrderSecurity(si_ort.Item1, si_ort.Item2, p.Position), + (si_ort, r) => new OrderSecurity(si_ort.Item1, si_ort.Item2, null), osp => osp.order.Key); on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); @@ -82,8 +82,8 @@ namespace mbts.spark { on_order_securities = on_orders .as_iincremental() - .filter(t => t.sec.HasValue) - .Select(t => t.sec.Value); + .filter(t => t.security.HasValue) + .Select(t => t.security.Value); _activity = new OrderActivitySummarizer(bus, on_orders); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); @@ -100,8 +100,8 @@ namespace mbts.spark { t => t.order.Key, am => am.algo.key, t => new AlgoKey(new bts.persist.algo.AlgoPersistedKey(t.order.Def.AlgoTemplateKey)), - (os, okey, am, mkey) => new OrderSecurityAlgoMeta(os.order, os.sec, am.algo, am.meta), - (os, okey, mkey) => new OrderSecurityAlgoMeta(os.order, os.sec, null, null), + (os, okey, am, mkey) => new OrderSecurityAlgoMeta(os.order, os.security, am.algo, am.meta), + (os, okey, mkey) => new OrderSecurityAlgoMeta(os.order, os.security, null, null), ot => ot.order.Key); on_order_tuples = order_secs_algo_meta_join; var on_fills = From 95500187f0156064e7abe6546262bd1c882dfa1d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 16 Apr 2019 12:43:24 -0400 Subject: [PATCH 049/131] Indicate our orders in diagnostic --- src/lib/api/services/MBOService.proto | 5 +++-- src/lib/md/MBOServiceServer.cpp | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/lib/api/services/MBOService.proto b/src/lib/api/services/MBOService.proto index 8250375cc..7ceabb12c 100644 --- a/src/lib/api/services/MBOService.proto +++ b/src/lib/api/services/MBOService.proto @@ -12,8 +12,9 @@ message OrderPIQ { message OrderPosition { uint64 exchange_id = 1; - float price = 2; - int32 size = 3; + uint32 order_id = 2; + float price = 10; + int32 size = 11; } message PositionData { uint32 security_id = 1; diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index a7f8c4bb7..39c668c95 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -154,7 +154,7 @@ std::optional search_orders(const ExchangeOrderKey oid, const std::vec } else { price = oo.price; - position = 0; + position = 1; } } return std::nullopt; @@ -301,6 +301,11 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request, return iter != index.end() ? iter->second : MDConsumerHandle {}; }); + std::unordered_map reverse; + for (const auto& pair : _me->active) { + reverse.insert(std::make_pair(pair.second.exchange_id(), pair.first)); + } + if(handle.is_initialized()) { auto mbo = handle.get_mbo(); if (mbo) { @@ -309,12 +314,20 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request, entry->set_exchange_id(o.id.value()); entry->set_price(o.price.price()); entry->set_size(o.size); + auto iter = reverse.find(o.id); + if (iter != reverse.end()) { + entry->set_order_id(iter->second.value()); + } } for (const auto& o : mbo->asks) { auto* entry = response.add_ask(); entry->set_exchange_id(o.id.value()); entry->set_price(o.price.price()); entry->set_size(o.size); + auto iter = reverse.find(o.id); + if (iter != reverse.end()) { + entry->set_order_id(iter->second.value()); + } } } } From 05abbc4a503872319b3fb2f57c99de63d196f116 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 12:03:33 -0500 Subject: [PATCH 050/131] don't show uint.Max piqs during server restart --- src/client/spark/view/Spark/Ladder/LadderVM.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index 0bf612489..fadbf56b0 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -233,7 +233,10 @@ namespace mbts.spark { public event Action on_tick; public MDConsumerHandle md => _model.md; - public IEnumerable<(double, uint)> price_vs_piq => _slot_price_vs_orders.Values.Where(result => result.min_piq.HasValue).Select(o => (o.price, o.min_piq.Value)); + // we return only entries who have piqs and whose piqs are not uint.Max + // uint.Max happens when the server is restarted but the client is still up + // maybe long term we don't show this data while the exchange connection is down + public IEnumerable<(double, uint)> price_vs_piq => _slot_price_vs_orders.Values.Where(result => result.min_piq.HasValue && result.min_piq.Value != uint.MaxValue).Select(o => (o.price, o.min_piq.Value)); public bool is_synthetic => _model.security.ContractCase == Security.ContractOneofCase.Synthetic; Property _status; From 4c8d0a4217f035c6782d5cab1c2a10fb3dc5d8ee Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 14:13:06 -0500 Subject: [PATCH 051/131] add null check on r.newval --- src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index 552d080aa..4262c4735 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -64,7 +64,7 @@ namespace mbts.spark { }), _columns, _current_row.SubscribeHookLatest( - r => r.newval.subscribe_data().Subscribe( + r => r.newval?.subscribe_data().Subscribe( data => selected_order_changed?.Invoke(data.order) ) ) From bc35e773cead49f14ae176f0b969c6bc9fd92e40 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 16 Apr 2019 14:19:43 -0500 Subject: [PATCH 052/131] piq_column settings should never return null, which allevites doing the same in sanitize_ladder_settings and another time when dealing with WindowSpecifics --- src/client/spark/view/Spark/Ladder/LadderSettings.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderSettings.cs b/src/client/spark/view/Spark/Ladder/LadderSettings.cs index 329082867..e5a8cf4a7 100644 --- a/src/client/spark/view/Spark/Ladder/LadderSettings.cs +++ b/src/client/spark/view/Spark/Ladder/LadderSettings.cs @@ -401,7 +401,7 @@ namespace mbts.spark { public bool clear_visible { get; set; } // [ProtoMember(43), DefaultValue(_default_order_entry_panel_width)] - public double order_entry_panel_initial_width { get; set; } + public double order_entry_panel_initial_width { get; set; } [ProtoMember(44), DefaultValue(true)] public bool use_defaults { get; set; } @@ -421,7 +421,11 @@ namespace mbts.spark { public NetChangeTerms net_change { get; set; } [ProtoMember(49)] - public LadderColumnSettings piq_column { get; set; } + LadderColumnSettings _piq_column; + public LadderColumnSettings piq_column { + get => _piq_column ?? (_piq_column = LadderSettings.Default.piq_column); + set => _piq_column = value; + } public bool Equals(LadderSettings other) => other != null && From 73c9be3bec68c496ba088554d9b1e23cf3b37088 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 16 Apr 2019 15:45:44 -0400 Subject: [PATCH 053/131] SPARK-2197 bugfix --- src/lib/md/MBOServiceServer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index 39c668c95..e25715ee4 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -147,7 +147,8 @@ std::optional search_orders(const ExchangeOrderKey oid, const std::vec for (size_t x = 0; x < orders.size(); ++x) { const auto& oo = orders[x]; if (oo.id == oid) { - return { position }; + if (oo.price == price) return { position }; + else return 0 ; } else if (price == oo.price) { position += 1; From b059aedbceba6f27c40a0e7376021bf6b25eb0a2 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 17 Apr 2019 08:53:26 -0500 Subject: [PATCH 054/131] adding PlacementInQueue key to toggles so that we have a config option to enable/disable once deployed; defaults to enabled --- src/client/spark/Orders/OrdersModel.cs | 13 +++++++++++-- src/lib/config/protos/toggles.proto | 3 +++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 37a7d2d2b..3835a80d7 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -7,6 +7,7 @@ using bts.md; using bts.spark; using bts.utils; using mbts.collections; +using mbts.config; using mbts.spark.utilities; using mbts.utils; using System; @@ -36,6 +37,7 @@ namespace mbts.spark { readonly TableUpdateDemuxer _security_orders_demuxer; readonly TableUpdateDemuxer _security_positions_demuxer; IDisposable _dispose; + static readonly bool _piq_enabled = Configuration.get().enabled; [ImportingConstructor] OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, MBOService mbo_service, @@ -52,14 +54,16 @@ namespace mbts.spark { this.algo_model = algo_model; var scheduler = ThreadPoolScheduler.Instance; var interval = TimeSpan.FromMilliseconds(100); - on_orders = + var pre_on_orders = _service .subscribei_orders() .Naggle(interval, scheduler) .PermissionFilter(admin_model.on_admin_data) .join(sstore, order => order.security_key()) .Table() - .Cache() + .Cache(); + if (_piq_enabled) { + on_orders = pre_on_orders .LeftJoin< (SecurityImmutable?, OrderRecordImmutable), OrderPIQImmutable, @@ -72,6 +76,11 @@ namespace mbts.spark { (si_ort, r) => new OrderSecurity(si_ort.Item1, si_ort.Item2, null), osp => osp.order.Key); + } else { + on_orders = pre_on_orders.Select((ft) => new OrderSecurity(ft.Item1, ft.Item2, null), os => os.order.Key) + .Publish(); + } + on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); working_orders = on_orders.Scan( diff --git a/src/lib/config/protos/toggles.proto b/src/lib/config/protos/toggles.proto index 6923cfbe9..2925a8721 100644 --- a/src/lib/config/protos/toggles.proto +++ b/src/lib/config/protos/toggles.proto @@ -37,4 +37,7 @@ message VolumeAtPrice{ } message MBOStats { optional bool enabled = 1 [default = true]; +} +message PlacementInQueue { + optional bool enabled = 1 [default = true]; } \ No newline at end of file From 6692797c3f55b71dfce940ce73a97569dae62428 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 17 Apr 2019 13:03:43 -0500 Subject: [PATCH 055/131] changing from .Publish to .Replay b/c apparently Publish doesn't send old values i.e. previously working orders won't show up, but new ones will --- src/client/spark/Orders/OrdersModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 3835a80d7..694260926 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -78,7 +78,7 @@ namespace mbts.spark { } else { on_orders = pre_on_orders.Select((ft) => new OrderSecurity(ft.Item1, ft.Item2, null), os => os.order.Key) - .Publish(); + .Replay(1); } on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); From ff2d150dc463f6c2b3151a169042616c678ef175 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 17 Apr 2019 17:33:46 -0500 Subject: [PATCH 056/131] restricting orders with piqs to just ladders for now --- src/client/spark/Daemons/OrderTicketDaemon.cs | 6 +- .../Notifications/INotificationService.cs | 2 +- .../INotificationServiceDataCache.cs | 2 +- .../spark/Orders/OrderActivitySummarizer.cs | 10 +-- src/client/spark/Orders/OrderSecurity.cs | 21 +++++-- src/client/spark/Orders/OrdersModel.cs | 61 +++++++++++++++---- .../spark/view/Spark/Ladder/LadderVM.cs | 2 +- 7 files changed, 75 insertions(+), 29 deletions(-) diff --git a/src/client/spark/Daemons/OrderTicketDaemon.cs b/src/client/spark/Daemons/OrderTicketDaemon.cs index 568a6b1ac..84e473c46 100644 --- a/src/client/spark/Daemons/OrderTicketDaemon.cs +++ b/src/client/spark/Daemons/OrderTicketDaemon.cs @@ -19,7 +19,7 @@ namespace bts.shell { readonly OrdersModel _model; readonly ErrorReporter _error_reporter; IReadOnlyDictionary _md_map; - TableUpdate _orders; + TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> _orders; IDisposable _dispose; [ImportingConstructor] @@ -60,13 +60,13 @@ namespace bts.shell { } void _show_modify_order(uint order_key) { - if (_orders.state.TryGetValue(order_key, out var ori) && ori.security.HasValue + if (_orders.state.TryGetValue(order_key, out var ori) && ori.sec.HasValue && (ori.order.HasDef && ori.order.Def.HasCustomDef && !(ori.order.Def.CustomDef.HasStop || ori.order.Def.CustomDef.HasStopLimit))) { var md = _md_map.TryGetValue(ori.order.security_key()); _show_order_window(new OrderModificationDialog { DataContext = new ExistingOrderTicketVM(_model, Observable.Return((md,ori.order)), - Observable.Return(_model.factory.default_ladder_settings(ori.security.Value).calculator_buttons) + Observable.Return(_model.factory.default_ladder_settings(ori.sec.Value).calculator_buttons) ) }); } diff --git a/src/client/spark/Notifications/INotificationService.cs b/src/client/spark/Notifications/INotificationService.cs index 820e2ed72..8e294a368 100644 --- a/src/client/spark/Notifications/INotificationService.cs +++ b/src/client/spark/Notifications/INotificationService.cs @@ -55,7 +55,7 @@ namespace bts.shell { void _on_activity(Activity activity) => _activity_notifications.process_item(activity); - void _on_orders(TableUpdate update) { + void _on_orders(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) { // must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history _data_cache.update(update.as_iincremental()); foreach (var o in update.added.Concat(update.changed)) diff --git a/src/client/spark/Notifications/INotificationServiceDataCache.cs b/src/client/spark/Notifications/INotificationServiceDataCache.cs index cf51f6e14..fac9deeed 100644 --- a/src/client/spark/Notifications/INotificationServiceDataCache.cs +++ b/src/client/spark/Notifications/INotificationServiceDataCache.cs @@ -50,7 +50,7 @@ namespace bts.shell { public DateTime last_order_reset_time { get; private set; } public bool is_first_order_update { get; private set; } - internal void update(IIncrementalUpdate data) { + internal void update(IIncrementalUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> data) { DateTime now = DateTime.Now; if (data.removed.Count == _order_cache.Count) { last_order_reset_time = now; diff --git a/src/client/spark/Orders/OrderActivitySummarizer.cs b/src/client/spark/Orders/OrderActivitySummarizer.cs index b085dddc9..b4b8e8b3d 100644 --- a/src/client/spark/Orders/OrderActivitySummarizer.cs +++ b/src/client/spark/Orders/OrderActivitySummarizer.cs @@ -27,7 +27,7 @@ namespace bts.spark { static readonly IEqualityComparer _id_comparer = bts.utils.EqualityComparer.Create(a => a.Key); - public OrderActivitySummarizer(IBus bus, IObservable> orders) { + public OrderActivitySummarizer(IBus bus, IObservable> orders) { _bus = bus; _dispose = Disposable.Create( bus.subscribe(SparkTags.SecuritiesTag).Subscribe(sec => _security = sec), @@ -124,12 +124,12 @@ namespace bts.spark { #endregion - void orders_updated(TableUpdate update) { + void orders_updated(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) { foreach (var o in update.added) { if (!string.IsNullOrEmpty(o.order.State.LastMsg) && o.order.State.Created.ToDateTime().Date == DateTime.Today) { - if (o.security.HasValue) { - log_status(ActivityLogCategory.warn, o.security.Value, o.order.State.LastMsg); + if (o.sec.HasValue) { + log_status(ActivityLogCategory.warn, o.sec.Value, o.order.State.LastMsg); } } } @@ -137,7 +137,7 @@ namespace bts.spark { foreach (var order in update.changes) { var neworder = order.New.order; var oldorder = order.Old.order; - var security = order.New.security ?? order.Old.security; + var security = order.New.sec ?? order.Old.sec; // // description var mod_desc = modified_description(oldorder, neworder, security?.ContractSpec); diff --git a/src/client/spark/Orders/OrderSecurity.cs b/src/client/spark/Orders/OrderSecurity.cs index 71889e5a6..7045f2fe4 100644 --- a/src/client/spark/Orders/OrderSecurity.cs +++ b/src/client/spark/Orders/OrderSecurity.cs @@ -4,16 +4,27 @@ using System; namespace bts.api.orderman { public readonly struct OrderSecurity: IEquatable { + public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order) { + this.security = sec; + this.order = order; + piq = null; + } + public OrderSecurity((SecurityImmutable? sec, OrderRecordImmutable order) t) { + security = t.sec; + order = t.order; + piq = null; + } public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order, uint? piq) { this.security = sec; this.order = order; this.piq = piq; } - public OrderSecurity(OrderSecurity t) { - security = t.security; - order = t.order; - piq = t.piq; - } + //public OrderSecurity(OrderSecurity t) { + // security = t.security; + // order = t.order; + // piq = t.piq; + //} + public static implicit operator OrderSecurity(ValueTuple x) => new OrderSecurity(x); public SecurityImmutable? security { get; } public OrderRecordImmutable order { get; } public uint? piq { get; } diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 694260926..7172a93cf 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -20,7 +20,8 @@ using System.Reactive.Concurrency; using System.Reactive.Linq; using System.Reactive.Subjects; using System.Threading.Tasks; -using OrderUpdate = bts.api.core.TableUpdate; +using OrderUpdate = bts.api.core.TableUpdate<(bts.api.SecurityImmutable? sec, bts.api.orderman.OrderRecordImmutable order), uint>; +using OrderUpdatePiq = bts.api.core.TableUpdate; namespace mbts.spark { [Export] @@ -54,7 +55,7 @@ namespace mbts.spark { this.algo_model = algo_model; var scheduler = ThreadPoolScheduler.Instance; var interval = TimeSpan.FromMilliseconds(100); - var pre_on_orders = + on_orders = _service .subscribei_orders() .Naggle(interval, scheduler) @@ -62,8 +63,9 @@ namespace mbts.spark { .join(sstore, order => order.security_key()) .Table() .Cache(); + if (_piq_enabled) { - on_orders = pre_on_orders + on_orders_with_piq = on_orders .LeftJoin< (SecurityImmutable?, OrderRecordImmutable), OrderPIQImmutable, @@ -75,9 +77,9 @@ namespace mbts.spark { (si_ort, r, p) => new OrderSecurity(si_ort.Item1, si_ort.Item2, p.Position), (si_ort, r) => new OrderSecurity(si_ort.Item1, si_ort.Item2, null), osp => osp.order.Key); - } else { - on_orders = pre_on_orders.Select((ft) => new OrderSecurity(ft.Item1, ft.Item2, null), os => os.order.Key) + on_orders_with_piq = on_orders + .Select((ft) => new OrderSecurity(ft.Item1, ft.Item2, null), os => os.order.Key) .Replay(1); } @@ -88,11 +90,18 @@ namespace mbts.spark { (workers, table) => _update_working_orders(workers, table)) .Replay(1).RefCount(); + working_orders_with_piq = + on_orders_with_piq.Scan( + ImmutableDictionary.Empty, + (workers, table) => _update_working_orders(workers, table)) + .Replay(1).RefCount(); + + on_order_securities = on_orders .as_iincremental() - .filter(t => t.security.HasValue) - .Select(t => t.security.Value); + .filter(t => t.sec.HasValue) + .Select(t => t.sec.Value); _activity = new OrderActivitySummarizer(bus, on_orders); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); @@ -109,8 +118,8 @@ namespace mbts.spark { t => t.order.Key, am => am.algo.key, t => new AlgoKey(new bts.persist.algo.AlgoPersistedKey(t.order.Def.AlgoTemplateKey)), - (os, okey, am, mkey) => new OrderSecurityAlgoMeta(os.order, os.security, am.algo, am.meta), - (os, okey, mkey) => new OrderSecurityAlgoMeta(os.order, os.security, null, null), + (os, okey, am, mkey) => new OrderSecurityAlgoMeta(os.order, os.sec, am.algo, am.meta), + (os, okey, mkey) => new OrderSecurityAlgoMeta(os.order, os.sec, null, null), ot => ot.order.Key); on_order_tuples = order_secs_algo_meta_join; var on_fills = @@ -129,9 +138,8 @@ namespace mbts.spark { f => f.fill.FillKey); on_fill_tuples = fill_order_tuples_join; _dispose = Disposable.Create( - // on_orders changed from an IObservable to an IConnectableObservable, so we call - // RefCount() here to start the connection - on_orders.RefCount().Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), + on_orders.Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), + on_orders_with_piq.RefCount().Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), on_positions.Subscribe(_security_positions_demuxer), order_secs_algo_meta_join.Connect(), algo_meta_join.Connect(), @@ -148,11 +156,13 @@ namespace mbts.spark { public OrderFactory factory => _factory; public SpreaderModel spreader => _spreader; public DynamicOrderHelper dynamic_helper => _dynamic_helper; - public IConnectableObservable on_orders { get; } + public IObservable on_orders { get; } + public IConnectableObservable on_orders_with_piq { get; } public IObservable> on_order_tuples { get; } public IObservable> on_order_securities { get; } public IObservable> on_fill_tuples { get; } public IObservable> working_orders { get; } + public IObservable> working_orders_with_piq { get; } public IObservable> on_positions { get; } public AdminModel admin_model { get; } @@ -249,8 +259,33 @@ namespace mbts.spark { } return clone.ToImmutableDictionary(); } + + static ImmutableDictionary + _update_working_orders(ImmutableDictionary target, OrderUpdatePiq table) { + var clone = target.ToBuilder(); + foreach (var o in table.removed) { + if (o.order.is_active()) { + clone.Remove(o.order.Key); + } + } + foreach (var o in table.changes) { + if (o.Old.order.is_active()) { + clone.Remove(o.Old.order.Key); + } + if (o.New.order.is_active()) { + clone.Add(o.New.order.Key, o.New); + } + } + foreach (var o in table.added) { + if (o.order.is_active()) { + clone.Add(o.order.Key, o); + } + } + return clone.ToImmutableDictionary(); + } } + // usinig new OrdersModel extensions to reduce the amount of code change needed to switch uis over to the new OrdersModel, // without changing any code in the actual new OrdersModel class. static class OrdersModelExtensions { diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index fadbf56b0..b32e25417 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -133,7 +133,7 @@ namespace mbts.spark { account_positions, size_calculator = new CalculatorLogic(0, _settings_manager.subscribe_current().Select(s => s.calculator_buttons)), bus.subscribe(SparkTags.LogActivity).OnDispatcher().Subscribe(status_update), - model.order_gateway.working_orders.OnDispatcher().Subscribe(working => { + model.order_gateway.working_orders_with_piq.OnDispatcher().Subscribe(working => { // Here we manage N orders to 1 price slot in the ladder Dictionary> orders_used_in_this_update = new Dictionary>(_equality); From dcfbb6e62ffa48eb5ffe3a782f4ef529458ad5cb Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Thu, 18 Apr 2019 08:58:56 -0500 Subject: [PATCH 057/131] need a new security order demuxer for the new piq observable --- src/client/spark/Orders/OrdersModel.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 7172a93cf..5be617a10 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -36,6 +36,8 @@ namespace mbts.spark { readonly DynamicOrderHelper _dynamic_helper; readonly OrderActivitySummarizer _activity; readonly TableUpdateDemuxer _security_orders_demuxer; + // we need a new one of these for the time being since the ladder is subscribing to orders with piq + readonly TableUpdateDemuxer _security_orders_piq_demuxer; readonly TableUpdateDemuxer _security_positions_demuxer; IDisposable _dispose; static readonly bool _piq_enabled = Configuration.get().enabled; @@ -105,6 +107,7 @@ namespace mbts.spark { _activity = new OrderActivitySummarizer(bus, on_orders); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); + _security_orders_piq_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); _security_positions_demuxer = new TableUpdateDemuxer(o => o.RecordId, p => p.SecurityId); var algo_meta_join = algo_model.definitions.LeftJoin( algo_model.metadata_service.subscribe_metadata().Table(m => m.id), @@ -139,7 +142,7 @@ namespace mbts.spark { on_fill_tuples = fill_order_tuples_join; _dispose = Disposable.Create( on_orders.Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), - on_orders_with_piq.RefCount().Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), + on_orders_with_piq.RefCount().Select(t => t.order, o => o.Key).Subscribe(_security_orders_piq_demuxer), on_positions.Subscribe(_security_positions_demuxer), order_secs_algo_meta_join.Connect(), algo_meta_join.Connect(), From 36722e27ad3609c9cdc71832d6ba29622084f1e9 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Thu, 18 Apr 2019 11:08:32 -0400 Subject: [PATCH 058/131] Remove SDK docs from the manual --- doc/user/content/bts_spark/Algos/_index.md | 46 +++++++++++++++++++ .../content/bts_spark_sdk/Client/_index.md | 6 --- .../content/bts_spark_sdk/Server/_index.md | 6 --- doc/user/content/bts_spark_sdk/_index.md | 6 --- .../client/spark_sdk/doc}/Overview.md | 0 .../client/spark_sdk/doc}/Threading.md | 0 6 files changed, 46 insertions(+), 18 deletions(-) create mode 100644 doc/user/content/bts_spark/Algos/_index.md delete mode 100644 doc/user/content/bts_spark_sdk/Client/_index.md delete mode 100644 doc/user/content/bts_spark_sdk/Server/_index.md delete mode 100644 doc/user/content/bts_spark_sdk/_index.md rename {doc/user/content/bts_spark_sdk/Client => src/client/spark_sdk/doc}/Overview.md (100%) rename {doc/user/content/bts_spark_sdk/Client => src/client/spark_sdk/doc}/Threading.md (100%) diff --git a/doc/user/content/bts_spark/Algos/_index.md b/doc/user/content/bts_spark/Algos/_index.md new file mode 100644 index 000000000..f3955e979 --- /dev/null +++ b/doc/user/content/bts_spark/Algos/_index.md @@ -0,0 +1,46 @@ ++++ +title = "BTS Basic Algos" +weight = 10 +sort_by = "weight" +insert_anchor_links = "right" ++++ + + +## Lean + +The goal of the lean order is to watch the opposite-inside market size and cross when that size goes below the threshold (this also implies a cross if that size goes to zero and the market trades away from you) If the market trades toward you, you can opt into several different behaviors + +##### Parameters + * Cross Thresh - size thresh in absolute or ratio terms + * Max Chase Ticks - max levels to pay up, in ticks + + +## Join +The goal of the join order is to watch the opposite-inside market and join when the market flips (that is, when the best offer becomes the best bid) and has at least threshold size. + +##### Parameters + * Join Thresh - size thresh in absolute or ratio terms + + +## Top + +The goal of the top order is to not trade by yourself, but the intent at the time of order entry is to get filled.If the order is on the market (say current bid price) AND the quantity on that bid price is below top thresh then cancel the order else remain working. + +##### Parameters + * Top Thresh - size thresh in absolute or ratio terms + + +## Cover +A Cover order’s purpose is to fire a hedge order (of any type) in response to fills from it’s child order on the opposite side of the market N ticks away + +##### Parameters + * Profit Ticks - ticks above or below fill price to place profit / cover order + + +## OCO +An OCO order creates a relationship between a child order and a 'stop' or 'other' order such that when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill. + +Composing this order with Cover simulates scalping functionality + +##### Parameters + * Stop Ticks - ticks below or above main order to place stop / other order \ No newline at end of file diff --git a/doc/user/content/bts_spark_sdk/Client/_index.md b/doc/user/content/bts_spark_sdk/Client/_index.md deleted file mode 100644 index dd7a5550e..000000000 --- a/doc/user/content/bts_spark_sdk/Client/_index.md +++ /dev/null @@ -1,6 +0,0 @@ -+++ -title = "Client" -weight = 1 -sort_by = "weight" -redirect_to = "bts_spark_sdk/Client/overview/" -+++ \ No newline at end of file diff --git a/doc/user/content/bts_spark_sdk/Server/_index.md b/doc/user/content/bts_spark_sdk/Server/_index.md deleted file mode 100644 index c6b0c27ff..000000000 --- a/doc/user/content/bts_spark_sdk/Server/_index.md +++ /dev/null @@ -1,6 +0,0 @@ -+++ -title = "Server" -weight = 2 -sort_by = "weight" -insert_anchor_links = "right" -+++ diff --git a/doc/user/content/bts_spark_sdk/_index.md b/doc/user/content/bts_spark_sdk/_index.md deleted file mode 100644 index 2a698fcda..000000000 --- a/doc/user/content/bts_spark_sdk/_index.md +++ /dev/null @@ -1,6 +0,0 @@ -+++ -title = "BTS Spark SDK" -weight = 2 -sort_by = "weight" -insert_anchor_links = "right" -+++ \ No newline at end of file diff --git a/doc/user/content/bts_spark_sdk/Client/Overview.md b/src/client/spark_sdk/doc/Overview.md similarity index 100% rename from doc/user/content/bts_spark_sdk/Client/Overview.md rename to src/client/spark_sdk/doc/Overview.md diff --git a/doc/user/content/bts_spark_sdk/Client/Threading.md b/src/client/spark_sdk/doc/Threading.md similarity index 100% rename from doc/user/content/bts_spark_sdk/Client/Threading.md rename to src/client/spark_sdk/doc/Threading.md From 02d0f6191ce573fa2bdd2f2a9a715979ca43672e Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 18 Apr 2019 14:06:44 -0400 Subject: [PATCH 059/131] -added a manifest to our csproj that we'd meant to add anyway for win7 subprocess creation -added a fallback for stunnel if we can't create a windows job object --- src/client/spark/App.xaml.cs | 53 +++++++++++++++++++++++------------ src/client/spark/Spark.csproj | 3 +- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/src/client/spark/App.xaml.cs b/src/client/spark/App.xaml.cs index 25aecff08..4fd7024dc 100644 --- a/src/client/spark/App.xaml.cs +++ b/src/client/spark/App.xaml.cs @@ -75,6 +75,37 @@ namespace mbts.ui.shell Environment.Exit(0); } + void start_stunnel(string path, uint job_error) + { + string config_error; + if (!mbts.config.Configuration.try_initialize(out config_error)) + { + _stunnel_process = Process.Start(path + "bin\\stunnel.exe"); + if (_stunnel_process != null) + { + for (int i = 0; i < 10; i++) + { + if (mbts.config.Configuration.try_initialize(out config_error)) + break; + System.Threading.Thread.Sleep(500); + } + } + else + stunnel_job_error = "Stunnel Job Error code: " + job_error + + " could not start stunnel " + (config_error ?? string.Empty); + } + } + + void kill_stunnel() + { + try + { + Process[] ps = Process.GetProcessesByName("stunnel"); + ps.ForEach(p => p.Kill()); + } + catch { } + } + Process _stunnel_process; static internal string stunnel_job_error { get; private set;} void init_stunnel(string[] argv) @@ -85,24 +116,10 @@ namespace mbts.ui.shell if (File.Exists(path + "config\\stunnel.conf")) { uint job_error = BTSUINativeMethods.setup_process_job_object(); - if(job_error == 0) - { - _stunnel_process = Process.Start(path + "bin\\stunnel.exe"); - string config_error; bool success = false; - if(_stunnel_process != null) - { - for(int i = 0; i < 10; i++) - { - if(success = mbts.config.Configuration.try_initialize(out config_error)) - break; - System.Threading.Thread.Sleep(500); - } - } - else - stunnel_job_error = "Stunnel Job Error code: " + job_error + " could not start stunnel"; - } - else - stunnel_job_error = "Stunnel Job Error code: " + job_error; + //if we can't put this process and stunnel into a job, try to stop stunnel then restart + if(job_error != 0) + kill_stunnel(); + start_stunnel(path, job_error); } } catch(Exception ex) diff --git a/src/client/spark/Spark.csproj b/src/client/spark/Spark.csproj index c6eb7e982..dbf62a939 100644 --- a/src/client/spark/Spark.csproj +++ b/src/client/spark/Spark.csproj @@ -4,7 +4,8 @@ Blue Trading Systems Blue Trading Systems Spark - The game has changed. Make your Move. + Advanced Derivatives Trading + spark.manifest true Debug;Release x64 From 546553406ba208b839c6ee855237cc7942a92e67 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 18 Apr 2019 14:13:09 -0400 Subject: [PATCH 060/131] PR comments --- src/client/spark/App.xaml.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/client/spark/App.xaml.cs b/src/client/spark/App.xaml.cs index 4fd7024dc..36810c745 100644 --- a/src/client/spark/App.xaml.cs +++ b/src/client/spark/App.xaml.cs @@ -101,7 +101,14 @@ namespace mbts.ui.shell try { Process[] ps = Process.GetProcessesByName("stunnel"); - ps.ForEach(p => p.Kill()); + ps.ForEach(p => + { + try + { + p.Kill(); + } + catch {} + }); } catch { } } From 7838bdf04f7119c0d65f26a68dcbd5ccbeb43ca0 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 18 Apr 2019 14:16:08 -0400 Subject: [PATCH 061/131] too much whitespace for even me --- src/client/spark/App.xaml.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/client/spark/App.xaml.cs b/src/client/spark/App.xaml.cs index 36810c745..ea2b1de78 100644 --- a/src/client/spark/App.xaml.cs +++ b/src/client/spark/App.xaml.cs @@ -103,10 +103,7 @@ namespace mbts.ui.shell Process[] ps = Process.GetProcessesByName("stunnel"); ps.ForEach(p => { - try - { - p.Kill(); - } + try { p.Kill(); } catch {} }); } From 5392d05ec8fbb840d33aa0071c244ba505b82a67 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 17 Apr 2019 20:22:12 -0400 Subject: [PATCH 062/131] MBO perf work --- .vscode/launch.json | 2 +- src/lib/md/IntPrice.h | 6 + src/lib/md/MBOProcessor.cpp | 91 ++++++++++-- src/lib/md/MBOProcessor.h | 78 ++++++++++- src/lib/md/test/mbo_tests.cpp | 132 +++++++++++++++++- src/server/sparkd/cmds/Run.cpp | 2 +- .../md/platforms/mdp3_onixs/MDP3Listener.h | 42 ++++-- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 4 + 8 files changed, 328 insertions(+), 29 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index fd68a6626..f3c362de0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -215,7 +215,7 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/server/sparkd", - "args": ["test", "[md]"], + "args": ["test", "[mbo]"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/server/", "environment": [], diff --git a/src/lib/md/IntPrice.h b/src/lib/md/IntPrice.h index 1f563548c..dfca4e73f 100644 --- a/src/lib/md/IntPrice.h +++ b/src/lib/md/IntPrice.h @@ -60,6 +60,12 @@ class IntPrice { return delta.mantissa() < 0; } } + bool operator <=(const IntPrice& other) { + return *this == other || *this < other; + } + bool operator >=(const IntPrice& other) { + return !(*this < other); + } IntPrice operator - (const IntPrice& o) const { return op(o, [](int64_t m1, int64_t m2) { return m1 - m2; }); } diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 4fec20880..1079df5bc 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -1,4 +1,4 @@ -#include "MBOProcessor.h" +#include "MBOProcessor.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/ConcurrentPool.h" #include "lib/config/config.h" @@ -12,8 +12,8 @@ #include -namespace bts { -namespace md { +namespace bts { +namespace md { struct SecurityKeyHashCompare { static size_t hash(const SecurityKey& k) { @@ -51,8 +51,8 @@ void print_pool_stats(const utils::ConcurrentPool& pool) { } bid_percentiles << bid_sz[count-1] << " "; ask_percentiles << ask_sz[count-1] << " "; - LOG(md,Info, - "MBO statistics: count: {} - %[50,75,90,95,99,100] bid: [{}] - ask: [{}]", + LOG(md,Info, + "MBO statistics: count: {} - %[50,75,90,95,99,100] bid: [{}] - ask: [{}]", count, bid_percentiles.str(), ask_percentiles.str()); } @@ -60,11 +60,13 @@ typedef tbb::concurrent_hash_map pool; MapType snapshots; + utils::Dispatcher& dispatcher; utils::DispatcherTimer timer; - MBOProcessorImpl() : - pool(2048,512), - snapshots(), - timer(utils::Dispatcher::background(), 600*1000, [&]() { print_pool_stats(pool); }) + MBOProcessorImpl(utils::Dispatcher& d) : + pool(2048,512), + snapshots(), + dispatcher(d), + timer(d, 600*1000, [&]() { print_pool_stats(pool); }) { if (config::get().enabled()) { timer.start(); @@ -75,8 +77,8 @@ struct MBOProcessorImpl { // // Processor -MBOProcessor::MBOProcessor() : - _me(std::make_shared()), +MBOProcessor::MBOProcessor(utils::Dispatcher* d) : + _me(std::make_shared(d == nullptr ? utils::Dispatcher::background() : *d)), _producer(_me), _consumer(_me) {} @@ -86,9 +88,10 @@ MBOReader& MBOProcessor::reader() { return _consumer; } // // Writer + MBOWriter::MBOWriter(std::shared_ptr& me): _me(me) { } MBOWriter::~MBOWriter() { } -MBOSnapshot* MBOWriter::allocate_snapshot() { +MBOSnapshot* MBOWriter::allocate_snapshot(size_t, size_t) { return _me->pool.allocate(); } void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot* snapshot) { @@ -102,6 +105,68 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot* snapshot) { } } +bool MBOWriter::validate_snapshot(SecurityKey key, MBOSnapshot* control) { + #define REQUIRE(x) if (!(x)) { LOG(md,Error, "{}", std::string(#x)); return false; } + // retrieve current snapshot and check they are equal + MBOReader reader(_me); + auto snap = reader.acquire_snapshot(key); + REQUIRE (snap->bids.size() == control->bids.size()) + REQUIRE (snap->asks.size() == control->asks.size()) + for (size_t x = 0; x < snap->bids.size(); ++x) { + REQUIRE (snap->bids[x].id == control->bids[x].id); + REQUIRE (snap->bids[x].price == control->bids[x].price); + REQUIRE (snap->bids[x].size == control->bids[x].size); + } + for (size_t x = 0; x < snap->asks.size(); ++x) { + REQUIRE (snap->asks[x].id == control->asks[x].id); + REQUIRE (snap->asks[x].price == control->asks[x].price); + REQUIRE (snap->asks[x].size == control->asks[x].size); + } + #undef REQUIRE + return true; +} +void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { + _me->dispatcher.post([this,&state,update]() { + this->apply_incremental(state,update); + }); +} +void MBOWriter::apply_incremental(MBOWorkingState& state, const inc::Update& update) { + // fast part + apply_incremental(state, update); + // determine if we need to publish + // slow part + auto snap = allocate_snapshot(state.bid.size(), state.ask.size()); + copy_to_snapshot(state, snap); + publish_snapshot(state.security, snap); +} + +void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { + if (dest == nullptr) return; + // copy working state to snapshot + dest->updated = state.updated; + auto type_map = [](auto pair) { + return SnapshotOrder { pair.second.id, std::get(pair.first), pair.second.size }; + }; + std::transform(state.bid.rbegin(), state.bid.rend(), std::back_inserter(dest->bids), type_map); + std::transform(state.ask.begin(), state.ask.end(), std::back_inserter(dest->asks), type_map); +} + +void apply_incremental(MBOWorkingState& state, const inc::Update& update) { + // apply the update to the working state + MBOWorkingState::Side& collection = (update.type == inc::UpdateType::Bid) ? state.bid : state.ask; + auto key = std::make_tuple(update.price, update.priority); + switch(update.action) { + case inc::UpdateAction::New: + case inc::UpdateAction::Change: + collection[key] = { update.order_id, update.size }; + break; + case inc::UpdateAction::Delete: + collection.erase(key); + break; + } +} + + // // Reader @@ -123,4 +188,4 @@ void MBOReader::release_snapshot(MBOSnapshot**s) const { _me->pool.release(s); } -}} +}} diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 17246fd21..04c3747d7 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -2,16 +2,20 @@ #include "lib/domain_model/SecurityKey.h" #include "lib/domain_model/Security.h" #include "lib/utils/TimeStamp.h" -#include "lib/utils/ValueWrapper.h" +#include "lib/utils/Dispatcher.h" #include "IntPrice.h" #include "ExchangeOrderKey.h" +#include "google/btree/btree_map.h" + #include #include +#include +#include namespace bts { namespace md { - + struct SnapshotOrder { ExchangeOrderKey id; IntPrice price; @@ -39,6 +43,59 @@ struct MBOSnapshot { } }; +namespace inc { + + // according to this link MBO book only has these updates types + // https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP+3.0+-+Market+Data+Snapshot+Full+Refresh#MDP3.0-MarketDataSnapshotFullRefresh-MarketbyOrder(MBO)DataBlock + enum class UpdateType { + Bid, // 0 + Ask, // 1 + // ImpliedBid, // 'E' + // ImpliedAsk, // 'F' + // BookReset // 'J' + }; + + // OrderUpdateAction=37708 + enum class UpdateAction { + New, //0 + Change, //1 + Delete //2 + }; + + typedef int64_t Priority; + struct Update { + ExchangeOrderKey order_id; + Priority priority; + IntPrice price; + int32_t size; + UpdateType type; + UpdateAction action; + }; +} + +struct MBOWorkingState { + typedef std::tuple key_t; + struct Data { + ExchangeOrderKey id; + int32_t size; + }; + //todo: b-tree this + typedef std::map Side; + + SecurityKey security; + utils::TimeStamp updated; + Side bid,ask; +}; +void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* destination); + +void apply_incremental(MBOWorkingState&, const inc::Update&); +template +void apply_incremental(MBOWorkingState& state, const V& vec) { + for (const auto& update : vec) { + apply_incremental(state, update); + } +} + struct MBOProcessorImpl; class MBOProcessor; @@ -47,16 +104,27 @@ class MBOWriter { friend class MBOProcessor; MBOWriter(std::shared_ptr&); ~MBOWriter(); + public: + // + // Snapshot api + MBOSnapshot* allocate_snapshot(size_t bid_count=0, size_t ask_count=0); + void publish_snapshot(SecurityKey, MBOSnapshot*); + bool validate_snapshot(SecurityKey, MBOSnapshot*); - MBOSnapshot* allocate_snapshot(); + // + // Incremental api - void publish_snapshot(SecurityKey key, MBOSnapshot* snapshot); + // Enqueue the update to the mbo work queue. + void enqueue_incremental(MBOWorkingState&, const inc::Update&); + // Update working state and publish directly. + void apply_incremental(MBOWorkingState&, const inc::Update&); }; class MBOReader { std::shared_ptr _me; friend class MBOProcessor; + friend class MBOWriter; MBOReader(std::shared_ptr&); ~MBOReader(); public: @@ -70,7 +138,7 @@ class MBOProcessor { MBOReader _consumer; public: - MBOProcessor(); + MBOProcessor(utils::Dispatcher* _=nullptr); MBOWriter& writer(); MBOReader& reader(); }; diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp index 70c4492e9..95517feb5 100644 --- a/src/lib/md/test/mbo_tests.cpp +++ b/src/lib/md/test/mbo_tests.cpp @@ -1,12 +1,17 @@ #ifndef _WIN32 #include "catch/catch.hpp" +#include "../IntPrice.h" #include "../MBOProcessor.h" + +#include +#include + namespace bts { namespace md { namespace test { int mbo_tests = 0; -TEST_CASE("mbo basic", "[md][mbo]") { +TEST_CASE("MBO publish snapshot", "[md][mbo]") { MBOProcessor p; // publish a snapshot auto sec = SecurityKey(99); @@ -23,5 +28,130 @@ TEST_CASE("mbo basic", "[md][mbo]") { REQUIRE(s->asks.size() == 0); } +std::vector gen_test_orders(uint64_t id=100, inc::UpdateType type=inc::UpdateType::Bid) { + std::vector result; + inc::Priority priority = 10000; + for (int p = 100; p < 501; p+=100) { + // 6 sizes + for (int32_t s = 1; s <= 5; ++s) { + IntPrice price { p + 5 , -2}; + auto order_id = ExchangeOrderKey(++id); + auto size = s; + priority++; + result.push_back( inc::Update { + order_id, + priority, + price, + size, + type, + inc::UpdateAction::New + }); + // std::cout << std::endl << order_id.value() << ": " << price.price() << "-" << size; + } + } + std::random_shuffle(result.begin(), result.end()); + return result; +} + +TEST_CASE("MBO state to snapshot", "[md][mbo]") { + // convert state to snapshot + MBOWorkingState state; + + state.security = SecurityKey(42); + state.updated = utils::TimeStamp::now(); + + auto bids = gen_test_orders(1000, inc::UpdateType::Bid); + apply_incremental(state, bids); + auto asks = gen_test_orders(2000, inc::UpdateType::Ask); + apply_incremental(state, asks); + + MBOSnapshot result; + copy_to_snapshot(state, &result); + + REQUIRE(result.updated == state.updated); + REQUIRE(result.bids.size() == state.bid.size()); + REQUIRE(result.asks.size() == state.ask.size()); + REQUIRE(bids.size() == result.bids.size()); + REQUIRE(asks.size() == result.asks.size()); + for (size_t x = 0; x < result.bids.size() - 1; ++x) { + // std::cout << std::endl << result.bids[x].id.value() << ": " << result.bids[x].price.price() << "-" << result.bids[x].size; + REQUIRE(result.bids[x].price >= result.bids[x+1].price); + } + for (size_t x = 0; x < result.asks.size() - 1; ++x) { + // std::cout << std::endl << result.asks[x].id.value() << ": " << result.asks[x].price.price() << "-" << result.asks[x].size; + REQUIRE(result.asks[x].price <= result.asks[x+1].price); + } + +} + +template +std::vector concat(const std::vector& a, const std::vector& b) { + std::vector res; + std::copy(a.begin(), a.end(), std::back_inserter(res)); + std::copy(b.begin(), b.end(), std::back_inserter(res)); + return res; +} + +TEST_CASE("MBO incremental", "[md][mbo]") { + // basic incremental test + MBOWorkingState state; + + auto orders = + concat( gen_test_orders(128, inc::UpdateType::Bid), + gen_test_orders(256, inc::UpdateType::Ask)); + + std::random_shuffle(orders.begin(), orders.end()); + apply_incremental(state, orders); + + REQUIRE(state.bid.size() + state.ask.size() == orders.size()); + + // generate some updates/deletes + std::vector updates; + // - change all size=2 to size=6 + // - delete all size=3 + size_t removed = 0; + for (const auto& e : orders) { + if (e.size == 2) { + updates.push_back( inc::Update { + e.order_id, + e.priority, + e.price, + 6, + e.type, + inc::UpdateAction::Change + }); + } + else if (e.size == 3) { + ++removed; + updates.push_back( inc::Update { + e.order_id, + e.priority, + e.price, + e.size, + e.type, + inc::UpdateAction::Delete + }); + } + } + apply_incremental(state,updates); + + REQUIRE(state.bid.size() + state.ask.size() == (orders.size() - removed)); + + MBOSnapshot result; + copy_to_snapshot(state, &result); + + auto all = concat (result.bids, result.asks); + std::map counts; + for (const auto& order : all) { + REQUIRE(order.size != 3); + REQUIRE(order.size != 2); + counts[order.size] ++; + } + REQUIRE(counts[6] > 0); + REQUIRE(counts[6] == counts[1]); + REQUIRE(counts[5] == counts[4]); + REQUIRE(counts[6] == counts[5]); +} + }}} #endif \ No newline at end of file diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 5f255fc56..d225abfde 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -60,7 +60,7 @@ int Run::execute(std::vector const &) { persistence::PersistenceServiceServer settings_saver(sc, "settings"); user::AuthServiceServer login_server(sc); - md::MBOProcessor mbo; + md::MBOProcessor mbo(&utils::Dispatcher::get_named("MBO")); md::run_market_datapublishers(&mbo.writer()); api::synthetic::SyntheticSecurityService synthetic_service(sc); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index c18a823d7..cd757f5c6 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -62,16 +62,42 @@ public: // int qty; // a.displayQty(qty); // uint8_t ref; - // a.referenceId(ref); - // const auto& ref_entry = args.message().entries()[(size_t)ref]; - // std::cout - // << a.orderId() - // << qty - // << ref_entry.securityId() - // ; + // if (a.referenceId(ref)) { + // const auto& ref_entry = args.message().entries()[(size_t)ref]; + // LOG(md,Info, "INCREMENTAL32: {} {} {}", + // a.orderId(), + // qty, + // ref_entry.securityId()); + // } // } // }; - // void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args&) override {}; + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args& args) override { + for(const auto& a : args.message().orderIdEntries()) { + int qty; + a.displayQty(qty); + uint8_t ref; + if (a.referenceId(ref)) { + const auto& ref_entry = args.message().entries()[(size_t)ref]; + LOG(md,Info, "INCREMENTAL46: {} {} {}", + a.orderId(), + qty, + ref_entry.securityId()); + } + } + }; + + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook44Args&) override { + if (_mbo) { + LOG(md,Info, "GOT SNAPSHOT44"); + } + } + + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook53Args&) override { + if (_mbo) { + LOG(md,Info, "GOT SNAPSHOT53"); + } + } + // diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index ac8da7ff3..025de1078 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -129,7 +129,11 @@ onixs::SessionSettings session_settings_for_channel(const string& channel_id) { } if (!options.join_snapshot_on_gap()) { + #ifdef DEBUG + settings.gapAndErrorHandling(onixs::GapAndErrorHandlingOptions::RecoverMarketState); + #else settings.gapAndErrorHandling(onixs::GapAndErrorHandlingOptions::ContinueProcessing); + #endif } if (options.use_natural_refresh()) { settings.joinRecovery(onixs::JoinRecoveryOptions::Disabled); From 3cf701a211f5a61e9742195dbd7c82a39eaf617a Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 18 Apr 2019 14:42:26 -0400 Subject: [PATCH 063/131] swap map for btree --- externals | 2 +- src/lib/md/MBOProcessor.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/externals b/externals index d0e8645b5..67a79263b 160000 --- a/externals +++ b/externals @@ -1 +1 @@ -Subproject commit d0e8645b5ef7380314eddc68114a9f0eaa379318 +Subproject commit 67a79263ba116a7646beb8368cef015530bea027 diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 04c3747d7..d1b94fd11 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -79,8 +79,8 @@ struct MBOWorkingState { ExchangeOrderKey id; int32_t size; }; - //todo: b-tree this - typedef std::map Side; + // typedef std::map Side; + typedef btree::btree_map Side; SecurityKey security; utils::TimeStamp updated; From d2a9d2170071ab7fcab7c784848dde129ac79044 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 22 Apr 2019 15:26:50 -0400 Subject: [PATCH 064/131] Handle incremental MBO messages from cme --- src/lib/md/MBOProcessor.cpp | 63 ++++++- src/lib/md/MBOProcessor.h | 24 ++- src/lib/md/test/mbo_tests.cpp | 12 +- .../platforms/mdp3_onixs/CMESecurityIndex.h | 4 +- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 3 +- .../md/platforms/mdp3_onixs/MDP3Listener.h | 172 +++++++++++++++--- 6 files changed, 228 insertions(+), 50 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 1079df5bc..b7909759f 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -94,14 +94,17 @@ MBOWriter::~MBOWriter() { } MBOSnapshot* MBOWriter::allocate_snapshot(size_t, size_t) { return _me->pool.allocate(); } -void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot* snapshot) { - bts_assert(snapshot != nullptr); +void MBOWriter::release_snapshot(MBOSnapshot** s) { + _me->pool.release(s); +} +void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { + bts_assert(*snapshot != nullptr); MapType::accessor accessor; _me->snapshots.insert(accessor, key); - std::swap(accessor->second, snapshot); - if (snapshot != nullptr) { + std::swap(accessor->second, *snapshot); + if (*snapshot != nullptr) { // Note: relase() must happen inside accessor lifetime to ensure correctness - _me->pool.release(&snapshot); + _me->pool.release(snapshot); } } @@ -125,19 +128,27 @@ bool MBOWriter::validate_snapshot(SecurityKey key, MBOSnapshot* control) { #undef REQUIRE return true; } + void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { - _me->dispatcher.post([this,&state,update]() { + std::stringstream ss; + ss << update; + LOG(md,Info, "Update: {}", ss.str()); + + // _me->dispatcher.post([this,&state,update]() { this->apply_incremental(state,update); - }); + // }); } void MBOWriter::apply_incremental(MBOWorkingState& state, const inc::Update& update) { // fast part apply_incremental(state, update); // determine if we need to publish // slow part - auto snap = allocate_snapshot(state.bid.size(), state.ask.size()); - copy_to_snapshot(state, snap); - publish_snapshot(state.security, snap); + auto key = SecurityKey(state.handle.key()); + if (key.value() != invalid_security_key) { + auto snap = allocate_snapshot(state.bid.size(), state.ask.size()); + copy_to_snapshot(state, snap); + publish_snapshot(key, &snap); + } } void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { @@ -166,6 +177,38 @@ void apply_incremental(MBOWorkingState& state, const inc::Update& update) { } } +std::ostream& operator<<(std::ostream& os, const inc::Update& update) { + os << "Update: { " + << update.order_id.value() << " " + << update.priority << " " + << update.price.price() << " " + << update.size << " "; + + switch(update.type) { + case inc::UpdateType::Bid: + os << "bid "; + break; + case inc::UpdateType::Ask: + os << "ask "; + break; + } + + switch(update.action) { + case inc::UpdateAction::New: + os << "NEW"; + break; + case inc::UpdateAction::Change: + os << "CH"; + break; + case inc::UpdateAction::Delete: + os << "DEL"; + break; + } + + os << "}"; + return os; +} + // // Reader diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index d1b94fd11..ceb38f1ea 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -5,6 +5,7 @@ #include "lib/utils/Dispatcher.h" #include "IntPrice.h" #include "ExchangeOrderKey.h" +#include "MDProducerHandle.h" #include "google/btree/btree_map.h" @@ -48,8 +49,8 @@ namespace inc { // according to this link MBO book only has these updates types // https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP+3.0+-+Market+Data+Snapshot+Full+Refresh#MDP3.0-MarketDataSnapshotFullRefresh-MarketbyOrder(MBO)DataBlock enum class UpdateType { - Bid, // 0 - Ask, // 1 + Bid = 0, + Ask = 1, // ImpliedBid, // 'E' // ImpliedAsk, // 'F' // BookReset // 'J' @@ -57,12 +58,12 @@ namespace inc { // OrderUpdateAction=37708 enum class UpdateAction { - New, //0 - Change, //1 - Delete //2 + New = 0, + Change = 1, + Delete = 2 }; - typedef int64_t Priority; + typedef uint64_t Priority; struct Update { ExchangeOrderKey order_id; Priority priority; @@ -73,6 +74,8 @@ namespace inc { }; } +std::ostream& operator<<(std::ostream& os, const inc::Update&); + struct MBOWorkingState { typedef std::tuple key_t; struct Data { @@ -82,10 +85,14 @@ struct MBOWorkingState { // typedef std::map Side; typedef btree::btree_map Side; - SecurityKey security; + MBOWorkingState(MDProducerHandle& h) : handle(h), updated(), bid(),ask() {} + + MDProducerHandle handle; utils::TimeStamp updated; Side bid,ask; }; + + void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* destination); void apply_incremental(MBOWorkingState&, const inc::Update&); @@ -109,7 +116,8 @@ class MBOWriter { // // Snapshot api MBOSnapshot* allocate_snapshot(size_t bid_count=0, size_t ask_count=0); - void publish_snapshot(SecurityKey, MBOSnapshot*); + void release_snapshot(MBOSnapshot**); + void publish_snapshot(SecurityKey, MBOSnapshot**); bool validate_snapshot(SecurityKey, MBOSnapshot*); // diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp index 95517feb5..0a8433de6 100644 --- a/src/lib/md/test/mbo_tests.cpp +++ b/src/lib/md/test/mbo_tests.cpp @@ -2,6 +2,7 @@ #include "catch/catch.hpp" #include "../IntPrice.h" #include "../MBOProcessor.h" +#include "Co2TestInstrument.h" #include #include @@ -18,7 +19,7 @@ TEST_CASE("MBO publish snapshot", "[md][mbo]") { { auto s = p.writer().allocate_snapshot(); s->bids.push_back({ ExchangeOrderKey(1), { 100, 0 }, 42 }); - p.writer().publish_snapshot(sec, s); + p.writer().publish_snapshot(sec, &s); } auto s = p.reader().acquire_snapshot(sec); @@ -55,9 +56,10 @@ std::vector gen_test_orders(uint64_t id=100, inc::UpdateType type=i TEST_CASE("MBO state to snapshot", "[md][mbo]") { // convert state to snapshot - MBOWorkingState state; + Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; + auto handle = instr.producer(); + MBOWorkingState state(handle); - state.security = SecurityKey(42); state.updated = utils::TimeStamp::now(); auto bids = gen_test_orders(1000, inc::UpdateType::Bid); @@ -94,7 +96,9 @@ std::vector concat(const std::vector& a, const std::vector& b) { TEST_CASE("MBO incremental", "[md][mbo]") { // basic incremental test - MBOWorkingState state; + Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; + auto handle = instr.producer(); + MBOWorkingState state(handle); auto orders = concat( gen_test_orders(128, inc::UpdateType::Bid), diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h b/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h index e0e9d3b56..d1915f85d 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h @@ -1,15 +1,17 @@ #pragma once #include "lib/md/SecurityIndex.h" +#include "lib/md/MBOProcessor.h" #include namespace bts { namespace md { struct SecurityData { - SecurityData(uint32_t id, MDProducerHandle& h) : id(id), handle(h), electronic_volume(0) {} + SecurityData(uint32_t id, MDProducerHandle& h) : id(id), handle(h), mbo_state(h), electronic_volume(0) { } uint32_t id; // exchange id MDProducerHandle handle; + MBOWorkingState mbo_state; uint32_t electronic_volume; }; diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index 85867b973..5f541305f 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -134,7 +134,8 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con for (const auto& ask : book.offers()) { snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); } - _mbo->publish_snapshot(SecurityKey(key), snapshot); + // _mbo->publish_snapshot(SecurityKey(key), snapshot); + bts_assert_d(_mbo->validate_snapshot(SecurityKey(key), snapshot)); } } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index cd757f5c6..b7f92544e 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -19,6 +19,52 @@ namespace bts { namespace md { +inline inc::UpdateType mbo_update_type(onixs::EntryTypeBook::Enum otype) { + switch(otype) { + case onixs::EntryTypeBook::Bid: + case onixs::EntryTypeBook::ImpliedBid: + return inc::UpdateType::Bid; + break; + case onixs::EntryTypeBook::Offer: + case onixs::EntryTypeBook::ImpliedOffer: + return inc::UpdateType::Ask; + break; + default: + bts_assert_msg(false, "Unexpected update type"); + } +} + +inline inc::UpdateAction mbo_update_action(onixs::OrderUpdateAction::Enum a) { + switch(a) { + case onixs::OrderUpdateAction::New: + return inc::UpdateAction::New; + case onixs::OrderUpdateAction::Update: + return inc::UpdateAction::Change; + case onixs::OrderUpdateAction::Delete: + return inc::UpdateAction::Delete; + default: + bts_assert_msg(false, "Unexpected update action"); + } +} + +inline inc::UpdateAction mbo_update_action(onixs::UpdateAction::Enum a) { + switch(a) { + case onixs::UpdateAction::New: + return inc::UpdateAction::New; + case onixs::UpdateAction::Change: + return inc::UpdateAction::Change; + case onixs::UpdateAction::Delete: + return inc::UpdateAction::Delete; + case onixs::UpdateAction::DeleteThru: + case onixs::UpdateAction::DeleteFrom: + case onixs::UpdateAction::Overlay: + return inc::UpdateAction::Delete; + default: + bts_assert_msg(false, "Unexpected update action"); + } +} + + class MDP3Listener : public onixs::HandlerListener, public onixs::MarketDataListener, @@ -57,44 +103,118 @@ public: // // Handle MBO incremental updates directly + // void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args& args) override { - // for(const auto& a : args.message().orderIdEntries()) { - // int qty; - // a.displayQty(qty); - // uint8_t ref; - // if (a.referenceId(ref)) { - // const auto& ref_entry = args.message().entries()[(size_t)ref]; - // LOG(md,Info, "INCREMENTAL32: {} {} {}", - // a.orderId(), - // qty, - // ref_entry.securityId()); - // } - // } // }; void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args& args) override { - for(const auto& a : args.message().orderIdEntries()) { + if (!_mbo) return; + + for(const auto& entry : args.message().orderIdEntries()) { int qty; - a.displayQty(qty); + entry.displayQty(qty); uint8_t ref; - if (a.referenceId(ref)) { - const auto& ref_entry = args.message().entries()[(size_t)ref]; - LOG(md,Info, "INCREMENTAL46: {} {} {}", - a.orderId(), - qty, - ref_entry.securityId()); + if (entry.referenceId(ref)) { + const auto& ref_entry = args.message().entries()[(int)ref]; + auto data = _index.get_security(ref_entry.securityId()); + if (!data) continue; + + auto order_id = ExchangeOrderKey(entry.orderId()); + inc::Priority priority; + bts_assert(entry.orderPriority(priority)); + + onixs::Decimal oprice; + ref_entry.entryPx(oprice); + auto price = convert_price(oprice, data->handle.security().display_factor); + + int32_t size; + bts_assert(entry.displayQty(size)); + auto type = mbo_update_type(ref_entry.entryType()); + + auto action = mbo_update_action(entry.orderUpdateAction()); + + _mbo->enqueue_incremental( + data->mbo_state, + inc::Update { + order_id, + priority, + price, + size, + type, + action + }); } } }; - void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook44Args&) override { - if (_mbo) { - LOG(md,Info, "GOT SNAPSHOT44"); + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshOrderBook43Args&) override { + LOG(md,Info, "GOT IROB43"); + } + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshOrderBook47Args& args) override { + if (!_mbo) return; + for (const auto& entry : args.message().entries()) { + auto data = _index.get_security(entry.securityId()); + if (!data) continue; + + // order id + uint64_t temp; + bts_assert( + entry.orderId(temp)); + auto order_id = ExchangeOrderKey(temp); + + inc::Priority priority; + bts_assert( + entry.orderPriority(priority)); + + onixs::Decimal onxprice; + bts_assert( + entry.entryPx(onxprice)); + auto price = convert_price(onxprice, data->handle.security().display_factor); + + int32_t size; + bts_assert( + entry.displayQty(size)); + + auto type = mbo_update_type(entry.entryType()); + auto action = mbo_update_action(entry.updateAction()); + + _mbo->enqueue_incremental( + data->mbo_state, + inc:: Update { + order_id, + priority, + price, + size, + type, + action + }); } } - void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook53Args&) override { - if (_mbo) { - LOG(md,Info, "GOT SNAPSHOT53"); + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook44Args&) override { + LOG(md,Info, "GOT SNAPSHOT44"); + } + + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook53Args& args) override { + if (!_mbo) return; + auto data = _index.get_security(args.message().securityId()); + if (!data) return; + for (const auto& entry : args.message().entries()) { + auto order_id = ExchangeOrderKey(entry.orderId()); + inc::Priority priority; + bts_assert(entry.orderPriority(priority)); + auto price = convert_price(entry.entryPx(), data->handle.security().display_factor); + auto size = entry.displayQty(); + inc::UpdateType type = mbo_update_type(entry.entryType()); + _mbo->enqueue_incremental( + data->mbo_state, + inc::Update { + order_id, + priority, + price, + size, + type, + inc::UpdateAction::New + }); } } From 5b71918a1f465d18af13893035224af8f546a5a8 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 22 Apr 2019 16:41:48 -0400 Subject: [PATCH 065/131] Revert "Make sure every sparkcmd command respects -configserver" This reverts commit dab95d7e91a02fd8ddb1cccdcdafddf3ce8a784f. --- src/client/sparkcmd/Program.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/sparkcmd/Program.cs b/src/client/sparkcmd/Program.cs index 790308e27..c4b3c8cd3 100644 --- a/src/client/sparkcmd/Program.cs +++ b/src/client/sparkcmd/Program.cs @@ -9,7 +9,6 @@ using System.Linq; namespace sparkcmd { class Program { static int Main(string[] args) { - mbts.config.Configuration.initialize(); var commands = ConsoleCommandDispatcher.FindCommandsInSameAssemblyAs(typeof(Program)); return ConsoleCommandDispatcher.DispatchCommand(commands, args, Console.Out); } From 80e337c5996ecc9b61cf77c41051ddf2278159a6 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 22 Apr 2019 16:45:36 -0400 Subject: [PATCH 066/131] sparkcmd should respect '-configserver' --- src/client/sparkcmd/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/sparkcmd/Program.cs b/src/client/sparkcmd/Program.cs index c4b3c8cd3..504ce8daf 100644 --- a/src/client/sparkcmd/Program.cs +++ b/src/client/sparkcmd/Program.cs @@ -9,6 +9,7 @@ using System.Linq; namespace sparkcmd { class Program { static int Main(string[] args) { + mbts.utils.BTSNativeMethods.provide_command_line(args, args.Length); var commands = ConsoleCommandDispatcher.FindCommandsInSameAssemblyAs(typeof(Program)); return ConsoleCommandDispatcher.DispatchCommand(commands, args, Console.Out); } From ca6c0f984c680b87848406f39e51f2d003e6028d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 23 Apr 2019 09:11:30 -0400 Subject: [PATCH 067/131] mbo bugfixes --- src/lib/md/MBOProcessor.cpp | 65 +++++++++++-------- src/lib/md/MBOProcessor.h | 12 +++- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 2 +- 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index b7909759f..6c2efd967 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -109,22 +109,28 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { } bool MBOWriter::validate_snapshot(SecurityKey key, MBOSnapshot* control) { - #define REQUIRE(x) if (!(x)) { LOG(md,Error, "{}", std::string(#x)); return false; } + // #define REQUIRE(x) if (!(x)) { LOG(md,Error, "{}", std::string(#x)); return false; } + #define REQUIRE(x) bts_assert(x); + // retrieve current snapshot and check they are equal - MBOReader reader(_me); - auto snap = reader.acquire_snapshot(key); - REQUIRE (snap->bids.size() == control->bids.size()) - REQUIRE (snap->asks.size() == control->asks.size()) - for (size_t x = 0; x < snap->bids.size(); ++x) { - REQUIRE (snap->bids[x].id == control->bids[x].id); - REQUIRE (snap->bids[x].price == control->bids[x].price); - REQUIRE (snap->bids[x].size == control->bids[x].size); - } - for (size_t x = 0; x < snap->asks.size(); ++x) { - REQUIRE (snap->asks[x].id == control->asks[x].id); - REQUIRE (snap->asks[x].price == control->asks[x].price); - REQUIRE (snap->asks[x].size == control->asks[x].size); - } + _me->dispatcher.post([this,key,control]() { + MBOReader reader(_me); + auto snap = reader.acquire_snapshot(key); + if (snap) { + REQUIRE (snap->bids.size() == control->bids.size()) + REQUIRE (snap->asks.size() == control->asks.size()) + for (size_t x = 0; x < snap->bids.size(); ++x) { + REQUIRE (snap->bids[x].id == control->bids[x].id); + REQUIRE (snap->bids[x].price == control->bids[x].price); + REQUIRE (snap->bids[x].size == control->bids[x].size); + } + for (size_t x = 0; x < snap->asks.size(); ++x) { + REQUIRE (snap->asks[x].id == control->asks[x].id); + REQUIRE (snap->asks[x].price == control->asks[x].price); + REQUIRE (snap->asks[x].size == control->asks[x].size); + } + } + }); #undef REQUIRE return true; } @@ -132,17 +138,17 @@ bool MBOWriter::validate_snapshot(SecurityKey key, MBOSnapshot* control) { void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { std::stringstream ss; ss << update; - LOG(md,Info, "Update: {}", ss.str()); + LOG(md,Info, "security:{} : {}", state.handle.key(), ss.str()); - // _me->dispatcher.post([this,&state,update]() { - this->apply_incremental(state,update); - // }); + _me->dispatcher.post([this,&state,update]() { + // fast part + apply_incremental(state,update); + // slow part + publish_incremental(state); + }); } -void MBOWriter::apply_incremental(MBOWorkingState& state, const inc::Update& update) { - // fast part - apply_incremental(state, update); +void MBOWriter::publish_incremental(MBOWorkingState& state) { // determine if we need to publish - // slow part auto key = SecurityKey(state.handle.key()); if (key.value() != invalid_security_key) { auto snap = allocate_snapshot(state.bid.size(), state.ask.size()); @@ -168,8 +174,11 @@ void apply_incremental(MBOWorkingState& state, const inc::Update& update) { auto key = std::make_tuple(update.price, update.priority); switch(update.action) { case inc::UpdateAction::New: + collection.insert(std::make_pair(key, MBOWorkingState::Data { update.order_id, update.size })); + break; case inc::UpdateAction::Change: collection[key] = { update.order_id, update.size }; + break; break; case inc::UpdateAction::Delete: collection.erase(key); @@ -179,8 +188,8 @@ void apply_incremental(MBOWorkingState& state, const inc::Update& update) { std::ostream& operator<<(std::ostream& os, const inc::Update& update) { os << "Update: { " - << update.order_id.value() << " " - << update.priority << " " + << "id:" << update.order_id.value() << " " + << "pri:" << update.priority << " " << update.price.price() << " " << update.size << " "; @@ -195,13 +204,13 @@ std::ostream& operator<<(std::ostream& os, const inc::Update& update) { switch(update.action) { case inc::UpdateAction::New: - os << "NEW"; + os << "+"; break; case inc::UpdateAction::Change: - os << "CH"; + os << "C"; break; case inc::UpdateAction::Delete: - os << "DEL"; + os << "-"; break; } diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index ceb38f1ea..00c09d888 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -7,7 +7,9 @@ #include "ExchangeOrderKey.h" #include "MDProducerHandle.h" +#ifndef _WIN32 #include "google/btree/btree_map.h" +#endif #include #include @@ -82,8 +84,12 @@ struct MBOWorkingState { ExchangeOrderKey id; int32_t size; }; - // typedef std::map Side; + +#ifdef _WIN32 + typedef std::map Side; +#else typedef btree::btree_map Side; +#endif MBOWorkingState(MDProducerHandle& h) : handle(h), updated(), bid(),ask() {} @@ -94,8 +100,8 @@ struct MBOWorkingState { void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* destination); - void apply_incremental(MBOWorkingState&, const inc::Update&); + template void apply_incremental(MBOWorkingState& state, const V& vec) { for (const auto& update : vec) { @@ -126,7 +132,7 @@ class MBOWriter { // Enqueue the update to the mbo work queue. void enqueue_incremental(MBOWorkingState&, const inc::Update&); // Update working state and publish directly. - void apply_incremental(MBOWorkingState&, const inc::Update&); + void publish_incremental(MBOWorkingState&); }; class MBOReader { diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index 5f541305f..d40b26eb7 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -135,7 +135,7 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); } // _mbo->publish_snapshot(SecurityKey(key), snapshot); - bts_assert_d(_mbo->validate_snapshot(SecurityKey(key), snapshot)); + // bts_assert_d(_mbo->validate_snapshot(SecurityKey(key), snapshot)); } } From 9cc4b5aa87cbb75d1c266ab552f1d2e77dcb3324 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 24 Apr 2019 10:08:53 -0400 Subject: [PATCH 068/131] Update unit/system tests command line parser --- .../sparkcmd/Cmd/ScriptClientCommand.cs | 4 --- src/client/sparkcmd/Cmd/ServiceTail.cs | 1 + src/client/sparkcmd/Cmd/TestCommand.cs | 25 ++++++++----------- src/client/sparkcmd/Program.cs | 7 +++--- src/client/sparkcmd/sparkcmd.csproj | 1 + 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/client/sparkcmd/Cmd/ScriptClientCommand.cs b/src/client/sparkcmd/Cmd/ScriptClientCommand.cs index 5f00cf392..57cf26389 100644 --- a/src/client/sparkcmd/Cmd/ScriptClientCommand.cs +++ b/src/client/sparkcmd/Cmd/ScriptClientCommand.cs @@ -23,10 +23,6 @@ namespace sparkcmd { } public class ScriptClientCommand { - bool _server = false; - string _host; - string _logfile; - public static int Run(ScriptClientOptions o) { if (o.server) { ScriptClient.runserver(); diff --git a/src/client/sparkcmd/Cmd/ServiceTail.cs b/src/client/sparkcmd/Cmd/ServiceTail.cs index 926bc5a6c..d7c47c6d8 100644 --- a/src/client/sparkcmd/Cmd/ServiceTail.cs +++ b/src/client/sparkcmd/Cmd/ServiceTail.cs @@ -8,6 +8,7 @@ using System.Text; using System.Threading.Tasks; namespace bts { + [Verb("tail", HelpText="'console dsqa'")] class TailServiceOptions { [Option('m', "method", Required=true, HelpText = "'[ServiceName].[Method]' to call")] public string method { get;set; } diff --git a/src/client/sparkcmd/Cmd/TestCommand.cs b/src/client/sparkcmd/Cmd/TestCommand.cs index 89dbf7498..b5b919f0a 100644 --- a/src/client/sparkcmd/Cmd/TestCommand.cs +++ b/src/client/sparkcmd/Cmd/TestCommand.cs @@ -8,28 +8,23 @@ using System.Threading.Tasks; using mbts.config; using System.Linq; using CommandLine; +using Newtonsoft.Json; namespace sparkcmd { [Verb("test", HelpText="Run Tests")] public class TestCommandOptions { - [Option('t', "timeout", HelpText="Timeout unit tets", Default=0)] + [Option('t', "timeout", HelpText="Timeout unit tets", Default=0u)] public uint timeout { get; set; } [Option('l', "list", HelpText="List tests")] public bool list { get; set; } [Option('i', "insular", HelpText="Tests run sparkd and use their own config (if specified)")] public bool self_contained { get; set; } - // [Option("unit", HelpText="Run unit tests")] - // public bool unit { get; set; } - // [Option("system", HelpText="Run system tests")] - // public bool system { get; set; } - // [Option("shell", HelpText="Run shell tests")] - // public bool shell { get; set; } + [Option("type", HelpText ="Run all tests of type [Unit|System|Shell]")] + public TestType type { get; set; } - [Value(0, MetaName="TestType")] - public string type { get; set; } [Value(1, MetaName="Tests", HelpText="Test list")] public IEnumerable tests { get; set; } @@ -46,8 +41,10 @@ namespace sparkcmd { } public static int Run(TestCommandOptions o) { + Console.WriteLine(JsonConvert.SerializeObject(o)); + if (o.list) { - list_tests(test_type(o.type)); + list_tests((o.type)); return 0; } @@ -60,20 +57,20 @@ namespace sparkcmd { } } else if (o.tests.Count() > 0) { - return run_tests(Enumerator.match(tests)); + return run_tests(o, Enumerator.match(o.tests)); } else { - return run_tests(Enumerator.get_of_type(test_type)); + return run_tests(o, Enumerator.get_of_type(o.type)); } } static Lazy _enumerator = new Lazy(() => new TestEnumerator("spark_test")); static TestEnumerator Enumerator => _enumerator.Value; - static int run_tests(IEnumerable tests) { + static int run_tests(TestCommandOptions o, IEnumerable tests) { var runner = new TestRunner(); // waits for all child processes to exit or time out - return runner.run_tests(tests, _timeout, self_contained); + return runner.run_tests(tests, o.timeout, o.self_contained); } static void list_tests(TestType test_type) { diff --git a/src/client/sparkcmd/Program.cs b/src/client/sparkcmd/Program.cs index 14b3df9bb..3f73e04c2 100644 --- a/src/client/sparkcmd/Program.cs +++ b/src/client/sparkcmd/Program.cs @@ -24,14 +24,15 @@ namespace sparkcmd { // } static int Main(string[] args) { // note: this can be made to infer the options type list from the list of delegates - return CommandLine.Parser.Default - .ParseArguments(args) - .MapResult( + return Parser.Default + .ParseArguments(args) + .MapResult( Configcheck.Run, ScriptClientCommand.Run, TailService.Run, ServiceTest.Run, SettingsManagerCommand.Run, + TestCommand.Run, errors => 1); } } diff --git a/src/client/sparkcmd/sparkcmd.csproj b/src/client/sparkcmd/sparkcmd.csproj index 1a8c7f607..75fd218c8 100644 --- a/src/client/sparkcmd/sparkcmd.csproj +++ b/src/client/sparkcmd/sparkcmd.csproj @@ -23,6 +23,7 @@ + From 34b7c99c81535d5f83d7d8be43895cb5d34417ae Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 24 Apr 2019 11:05:26 -0400 Subject: [PATCH 069/131] Toggle mbostats off by default --- src/lib/config/protos/toggles.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/config/protos/toggles.proto b/src/lib/config/protos/toggles.proto index 2925a8721..7985bbac7 100644 --- a/src/lib/config/protos/toggles.proto +++ b/src/lib/config/protos/toggles.proto @@ -36,7 +36,7 @@ message VolumeAtPrice{ optional bool enabled = 1 [default = true]; } message MBOStats { - optional bool enabled = 1 [default = true]; + optional bool enabled = 1 [default = false]; } message PlacementInQueue { optional bool enabled = 1 [default = true]; From 34401960f9301c4cc7f9e2571c3876b6dc852133 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 24 Apr 2019 11:08:46 -0400 Subject: [PATCH 070/131] Fix mbo bid sort order --- src/lib/md/MBOProcessor.cpp | 46 +++++++++++++++++++++++++++---- src/lib/md/MBOProcessor.h | 26 +++++++++++++++++- src/lib/md/test/mbo_tests.cpp | 51 +++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 6 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 6c2efd967..f6ef8e2f5 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -164,13 +164,12 @@ void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { auto type_map = [](auto pair) { return SnapshotOrder { pair.second.id, std::get(pair.first), pair.second.size }; }; - std::transform(state.bid.rbegin(), state.bid.rend(), std::back_inserter(dest->bids), type_map); + std::transform(state.bid.begin(), state.bid.end(), std::back_inserter(dest->bids), type_map); std::transform(state.ask.begin(), state.ask.end(), std::back_inserter(dest->asks), type_map); } -void apply_incremental(MBOWorkingState& state, const inc::Update& update) { - // apply the update to the working state - MBOWorkingState::Side& collection = (update.type == inc::UpdateType::Bid) ? state.bid : state.ask; +template +void apply_incremental(C& collection, const inc::Update& update) { auto key = std::make_tuple(update.price, update.priority); switch(update.action) { case inc::UpdateAction::New: @@ -185,8 +184,45 @@ void apply_incremental(MBOWorkingState& state, const inc::Update& update) { break; } } +void apply_incremental(MBOWorkingState& state, const inc::Update& update) { + // apply the update to the working state + if (update.type == inc::UpdateType::Bid) { + apply_incremental(state.bid, update); + } + else { + apply_incremental(state.ask, update); + } +} -std::ostream& operator<<(std::ostream& os, const inc::Update& update) { + +// +// Reader + +MBOReader::MBOReader(std::shared_ptr& me): _me(me) { } +MBOReader::~MBOReader() { } +MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { + // get a result eagerly so we don't have to do it while accessor has the entry locked + auto result = _me->pool.allocate(); + MapType::const_accessor accessor; + if (_me->snapshots.find(accessor, key) && accessor->second != nullptr) { + result->copy_from(*accessor->second); + } + else { + release_snapshot(&result); + } + return result; +} +void MBOReader::release_snapshot(MBOSnapshot**s) const { + _me->pool.release(s); +} + +}} + +namespace std { + using namespace bts; + using namespace bts::md; + +ostream& operator<<(ostream& os, const inc::Update& update) { os << "Update: { " << "id:" << update.order_id.value() << " " << "pri:" << update.priority << " " diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 00c09d888..e6bd0213b 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -78,12 +78,25 @@ namespace inc { std::ostream& operator<<(std::ostream& os, const inc::Update&); + struct MBOWorkingState { typedef std::tuple key_t; struct Data { ExchangeOrderKey id; int32_t size; }; + struct LessBid { + bool operator()(const key_t& left, const key_t& right) const { + const auto& left_price = std::get(left); + const auto& right_price = std::get(right); + if (left_price == right_price) { + return + std::get(left) < + std::get(right); + } + return !(left_price < right_price); + } + }; #ifdef _WIN32 typedef std::map Side; @@ -95,7 +108,14 @@ struct MBOWorkingState { MDProducerHandle handle; utils::TimeStamp updated; - Side bid,ask; + +#ifdef _WIN32 + std::map bid; + std::map ask; +#else + btree::btree_map bid; + btree::btree_map ask; +#endif }; @@ -159,3 +179,7 @@ public: }} + +namespace std { +std::ostream& operator<<(std::ostream& os, const bts::md::inc::Update&); +} diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp index 0a8433de6..d6e11580c 100644 --- a/src/lib/md/test/mbo_tests.cpp +++ b/src/lib/md/test/mbo_tests.cpp @@ -157,5 +157,56 @@ TEST_CASE("MBO incremental", "[md][mbo]") { REQUIRE(counts[6] == counts[5]); } +TEST_CASE("MBO sort order", "[md][mbo]") { + Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; + auto handle = instr.producer(); + MBOWorkingState state(handle); + + state.updated = utils::TimeStamp::now(); + + std::vector updates; + // multiple updates at the same price + for (unsigned x=0; x < 100; ++x) { + // bid + auto p = 1000 + x; + updates.push_back( inc::Update { + ExchangeOrderKey(1000 + x), + 1000 + x, + { 1000, -1 }, + (int)x+1, + inc::UpdateType::Bid, + inc::UpdateAction::New + }); + // ask + if (x < 75) { // ensure bid.size != ask.size + updates.push_back( inc::Update { + ExchangeOrderKey(2000 + x), + 2000 + x, + { 2000, -1 }, + (int)x+1, + inc::UpdateType::Ask, + inc::UpdateAction::New + }); + } + } + std::random_shuffle(updates.begin(), updates.end()); + + apply_incremental(state, updates); + + MBOSnapshot result; + copy_to_snapshot(state, &result); + + // Since all orders are at the same price we can assert they are in priority order + // Since priority is not explicit in the shapshot we set order_id to the same value + + for (size_t x = 0; x < result.bids.size() - 1; ++x) { + REQUIRE(result.bids[x].id.value() < result.bids[x+1].id.value()); + } + for (size_t x = 0; x < result.asks.size() - 1; ++x) { + REQUIRE(result.asks[x].id.value() < result.asks[x+1].id.value()); + } + +} + }}} #endif \ No newline at end of file From f65f34c2fed0f0841540bbd280f3a1e6ec50a83a Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 24 Apr 2019 14:12:50 -0400 Subject: [PATCH 071/131] Limit returned MBOSnapshot size to avoid memory leaking --- src/lib/md/MBOProcessor.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index e6bd0213b..9962208f3 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -41,6 +41,13 @@ struct MBOSnapshot { void reset() { updated = utils::TimeStamp(); + constexpr int max = 256; + if (bids.size() > max) { + bids.resize(max); + } + if (asks.size() > max) { + asks.resize(max); + } bids.clear(); asks.clear(); } From 8a1180835c4611fb96c70f7816d2b185ddc337a3 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 25 Apr 2019 11:06:39 -0400 Subject: [PATCH 072/131] MBO diagnostics/logging --- src/lib/md/MBOProcessor.cpp | 72 +++++++------------ src/lib/md/MBOProcessor.h | 3 +- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 2 +- .../md/platforms/mdp3_onixs/MDP3Listener.h | 23 +++--- 4 files changed, 38 insertions(+), 62 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index f6ef8e2f5..fa1789d5d 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -108,44 +108,42 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { } } -bool MBOWriter::validate_snapshot(SecurityKey key, MBOSnapshot* control) { - // #define REQUIRE(x) if (!(x)) { LOG(md,Error, "{}", std::string(#x)); return false; } - #define REQUIRE(x) bts_assert(x); +bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) { + #define REQUIRE(x,y) \ + if (!(x)) { \ + LOG(md,Error, "Validation error exch:{} sec:{} {} {}", exchid, key.value(), std::string(#x), y); \ + return false; \ + } + // #define REQUIRE(x) bts_assert(x); // retrieve current snapshot and check they are equal - _me->dispatcher.post([this,key,control]() { - MBOReader reader(_me); - auto snap = reader.acquire_snapshot(key); - if (snap) { - REQUIRE (snap->bids.size() == control->bids.size()) - REQUIRE (snap->asks.size() == control->asks.size()) - for (size_t x = 0; x < snap->bids.size(); ++x) { - REQUIRE (snap->bids[x].id == control->bids[x].id); - REQUIRE (snap->bids[x].price == control->bids[x].price); - REQUIRE (snap->bids[x].size == control->bids[x].size); - } - for (size_t x = 0; x < snap->asks.size(); ++x) { - REQUIRE (snap->asks[x].id == control->asks[x].id); - REQUIRE (snap->asks[x].price == control->asks[x].price); - REQUIRE (snap->asks[x].size == control->asks[x].size); - } + MBOReader reader(_me); + auto snap = reader.acquire_snapshot(key); + if (snap) { + REQUIRE (snap->bids.size() == control->bids.size(), (int)snap->bids.size() - (int)control->bids.size()); + REQUIRE (snap->asks.size() == control->asks.size(), (int)snap->asks.size() - (int)control->asks.size()); + for (size_t x = 0; x < snap->bids.size(); ++x) { + REQUIRE (snap->bids[x].id == control->bids[x].id, x); + REQUIRE (snap->bids[x].size == control->bids[x].size, x); } - }); + for (size_t x = 0; x < snap->asks.size(); ++x) { + REQUIRE (snap->asks[x].id == control->asks[x].id, x); + REQUIRE (snap->asks[x].size == control->asks[x].size, x); + } + } #undef REQUIRE return true; } void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { - std::stringstream ss; - ss << update; - LOG(md,Info, "security:{} : {}", state.handle.key(), ss.str()); + LOG(md,Info, "sec:{} : {}", state.handle.key(), update); - _me->dispatcher.post([this,&state,update]() { + // _me->dispatcher.post([this,&state,update]() { // fast part apply_incremental(state,update); // slow part publish_incremental(state); - }); + // }); } void MBOWriter::publish_incremental(MBOWorkingState& state) { // determine if we need to publish @@ -254,26 +252,4 @@ ostream& operator<<(ostream& os, const inc::Update& update) { return os; } - -// -// Reader - -MBOReader::MBOReader(std::shared_ptr& me): _me(me) { } -MBOReader::~MBOReader() { } -MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { - // get a result eagerly so we don't have to do it while accessor has the entry locked - auto result = _me->pool.allocate(); - MapType::const_accessor accessor; - if (_me->snapshots.find(accessor, key) && accessor->second != nullptr) { - result->copy_from(*accessor->second); - } - else { - release_snapshot(&result); - } - return result; -} -void MBOReader::release_snapshot(MBOSnapshot**s) const { - _me->pool.release(s); -} - -}} +} \ No newline at end of file diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index 9962208f3..eaf644cbf 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -83,7 +83,6 @@ namespace inc { }; } -std::ostream& operator<<(std::ostream& os, const inc::Update&); struct MBOWorkingState { @@ -151,7 +150,7 @@ class MBOWriter { MBOSnapshot* allocate_snapshot(size_t bid_count=0, size_t ask_count=0); void release_snapshot(MBOSnapshot**); void publish_snapshot(SecurityKey, MBOSnapshot**); - bool validate_snapshot(SecurityKey, MBOSnapshot*); + bool validate_snapshot(uint32_t, SecurityKey, MBOSnapshot*); // // Incremental api diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index d40b26eb7..62a771ee3 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -135,7 +135,7 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); } // _mbo->publish_snapshot(SecurityKey(key), snapshot); - // bts_assert_d(_mbo->validate_snapshot(SecurityKey(key), snapshot)); + _mbo->validate_snapshot(sec.id(), SecurityKey(key), snapshot); } } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index b7f92544e..13b70e1d5 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -104,8 +104,9 @@ public: // // Handle MBO incremental updates directly - // void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args& args) override { - // }; + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args&) override { + LOG(md,Error, "GOT IRB32"); + }; void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args& args) override { if (!_mbo) return; @@ -113,9 +114,13 @@ public: int qty; entry.displayQty(qty); uint8_t ref; + LOG(md,Info, "GOT IRB46:"); if (entry.referenceId(ref)) { - const auto& ref_entry = args.message().entries()[(int)ref]; - auto data = _index.get_security(ref_entry.securityId()); + // entries is indexed by 0 but ref is indexed by 1 :facepalm: + const auto& ref_entry = args.message().entries()[(int)ref-1]; + uint32_t secid = ref_entry.securityId(); + LOG(md,Info, "> GOT IRB46 exch:{} rptSeq:{}", secid, ref_entry.rptSeq()); + auto data = _index.get_security(secid); if (!data) continue; auto order_id = ExchangeOrderKey(entry.orderId()); @@ -395,9 +400,7 @@ void MDP3Listener::onFuture(onixs::Handler&, const onixs::Security& osec, const tracepoint (bts_trace,trace_order,(char*) "onDefinition.future", _latest_sequence, (uint16_t) _latest_sequence, 0); const auto& sec = def.message(); auto exchange_id = sec.securityId(); - // LOG(md, Info, "onDefinition future {} {}", - // std::string(sec.asset().items(), sec.asset().size()), - // std::string(sec.symbol().items(), sec.symbol().size())); + // LOG(md, Info, "onDefinition future exch:{}", exchange_id); if (filter_security(exchange_id, sec) != ADD) return; bts_assert(exchange_id == osec.id()); @@ -425,7 +428,7 @@ void MDP3Listener::onOption(onixs::Handler&, const onixs::Security&, const Instr tracepoint (bts_trace,trace_order,(char*) "onDefinition.option", _latest_sequence, (uint16_t) _latest_sequence, 0); const auto& sec = def.message(); auto exchange_id = sec.securityId(); - // LOG(Info, MarketData) << "onDefinition option " << exchange_id; + // LOG(md, Info, "onDefinition option exch:{}", exchange_id); if (filter_security(exchange_id, sec) != ADD) return; FixedOptionSecurity fsec; @@ -469,9 +472,7 @@ void MDP3Listener::onSpread(onixs::Handler&, const onixs::Security&, const Instr tracepoint (bts_trace,trace_order,(char*) "onDefinition.spread", _latest_sequence, (uint16_t) _latest_sequence, 0); const auto& sec = spread_args.message(); auto exchange_id = sec.securityId(); - // LOG(md, Info, "onDefinition spread {} {}", - // std::string(sec.asset().items(), sec.asset().size()), - // std::string(sec.symbol().items(), sec.symbol().size())); + // LOG(md, Info, "onDefinition spread exch:{}", exchange_id); if (filter_security(exchange_id, sec) != ADD) return; // Filter out any already expired spreads From ad4fd1318923ae8535d69051854e67886856332d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 25 Apr 2019 14:25:49 -0400 Subject: [PATCH 073/131] MBO: Add seconary index to handle mbo change --- src/lib/md/MBOProcessor.cpp | 45 +++++---- src/lib/md/MBOProcessor.h | 9 +- src/lib/md/test/mbo_tests.cpp | 93 ++++++++++++++++++- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 1 - .../md/platforms/mdp3_onixs/MDP3Listener.h | 2 - 5 files changed, 118 insertions(+), 32 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index fa1789d5d..3ec4f31ca 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -109,10 +109,11 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { } bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) { - #define REQUIRE(x,y) \ - if (!(x)) { \ - LOG(md,Error, "Validation error exch:{} sec:{} {} {}", exchid, key.value(), std::string(#x), y); \ - return false; \ + // TO USE THIS VALIDATION YOU MUST MAKE enqueue_incremental syncronous + #define REQUIRE(a,b,y) \ + if ((a) != (b)) { \ + LOG(md,Error, "Validation error exch:{} sec:{} {}!={} other:{}", exchid, key.value(), a, b, y); \ + return false; \ } // #define REQUIRE(x) bts_assert(x); @@ -120,15 +121,15 @@ bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* MBOReader reader(_me); auto snap = reader.acquire_snapshot(key); if (snap) { - REQUIRE (snap->bids.size() == control->bids.size(), (int)snap->bids.size() - (int)control->bids.size()); - REQUIRE (snap->asks.size() == control->asks.size(), (int)snap->asks.size() - (int)control->asks.size()); + REQUIRE (snap->bids.size(), control->bids.size(), '!'); + REQUIRE (snap->asks.size(), control->asks.size(), '!'); for (size_t x = 0; x < snap->bids.size(); ++x) { - REQUIRE (snap->bids[x].id == control->bids[x].id, x); - REQUIRE (snap->bids[x].size == control->bids[x].size, x); + REQUIRE (snap->bids[x].id, control->bids[x].id, x); + REQUIRE (snap->bids[x].size, control->bids[x].size, x); } for (size_t x = 0; x < snap->asks.size(); ++x) { - REQUIRE (snap->asks[x].id == control->asks[x].id, x); - REQUIRE (snap->asks[x].size == control->asks[x].size, x); + REQUIRE (snap->asks[x].id, control->asks[x].id, x); + REQUIRE (snap->asks[x].size, control->asks[x].size, x); } } #undef REQUIRE @@ -167,32 +168,42 @@ void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { } template -void apply_incremental(C& collection, const inc::Update& update) { +void apply_incremental(C& collection, std::unordered_map& index, const inc::Update& update) { auto key = std::make_tuple(update.price, update.priority); switch(update.action) { case inc::UpdateAction::New: collection.insert(std::make_pair(key, MBOWorkingState::Data { update.order_id, update.size })); + index[update.order_id] = key; break; - case inc::UpdateAction::Change: - collection[key] = { update.order_id, update.size }; - break; + case inc::UpdateAction::Change: { + // look up old key + auto r = index.find(update.order_id); + if (r != index.end()) { + auto oldkey = r->second; + collection.erase(oldkey); + index.erase(r); + index[update.order_id] = key; + } + else { bts_assert_d(false); } + collection.insert(std::make_pair(key, MBOWorkingState::Data { update.order_id, update.size })); + } break; case inc::UpdateAction::Delete: collection.erase(key); + index.erase(update.order_id); break; } } void apply_incremental(MBOWorkingState& state, const inc::Update& update) { // apply the update to the working state if (update.type == inc::UpdateType::Bid) { - apply_incremental(state.bid, update); + apply_incremental(state.bid, state.orderid_index, update); } else { - apply_incremental(state.ask, update); + apply_incremental(state.ask, state.orderid_index, update); } } - // // Reader diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h index eaf644cbf..f3a92b59b 100644 --- a/src/lib/md/MBOProcessor.h +++ b/src/lib/md/MBOProcessor.h @@ -84,7 +84,6 @@ namespace inc { } - struct MBOWorkingState { typedef std::tuple key_t; struct Data { @@ -104,16 +103,12 @@ struct MBOWorkingState { } }; -#ifdef _WIN32 - typedef std::map Side; -#else - typedef btree::btree_map Side; -#endif - MBOWorkingState(MDProducerHandle& h) : handle(h), updated(), bid(),ask() {} MDProducerHandle handle; utils::TimeStamp updated; + // Used for 'change' messages which update price + std::unordered_map orderid_index; #ifdef _WIN32 std::map bid; diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp index d6e11580c..a275a54c0 100644 --- a/src/lib/md/test/mbo_tests.cpp +++ b/src/lib/md/test/mbo_tests.cpp @@ -76,14 +76,11 @@ TEST_CASE("MBO state to snapshot", "[md][mbo]") { REQUIRE(bids.size() == result.bids.size()); REQUIRE(asks.size() == result.asks.size()); for (size_t x = 0; x < result.bids.size() - 1; ++x) { - // std::cout << std::endl << result.bids[x].id.value() << ": " << result.bids[x].price.price() << "-" << result.bids[x].size; REQUIRE(result.bids[x].price >= result.bids[x+1].price); } for (size_t x = 0; x < result.asks.size() - 1; ++x) { - // std::cout << std::endl << result.asks[x].id.value() << ": " << result.asks[x].price.price() << "-" << result.asks[x].size; REQUIRE(result.asks[x].price <= result.asks[x+1].price); } - } template @@ -162,8 +159,6 @@ TEST_CASE("MBO sort order", "[md][mbo]") { auto handle = instr.producer(); MBOWorkingState state(handle); - state.updated = utils::TimeStamp::now(); - std::vector updates; // multiple updates at the same price for (unsigned x=0; x < 100; ++x) { @@ -205,7 +200,95 @@ TEST_CASE("MBO sort order", "[md][mbo]") { for (size_t x = 0; x < result.asks.size() - 1; ++x) { REQUIRE(result.asks[x].id.value() < result.asks[x+1].id.value()); } +} +TEST_CASE("MBO change price and size", "[md][mbo]") { + // When an order's size goes up it gets a new priority + Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; + auto handle = instr.producer(); + MBOWorkingState state(handle); + + constexpr auto bid = inc::UpdateType::Bid; + constexpr auto ask = inc::UpdateType::Ask; + constexpr auto add = inc::UpdateAction::New; + constexpr auto remove = inc::UpdateAction::Delete; + constexpr auto change = inc::UpdateAction::Change; + auto p = [](double price) -> IntPrice { return IntPrice(price * 1000, -3); }; + + // snapshot + std::vector snapshot = { + { ExchangeOrderKey(701267154552), 1160179309, p(2.975), 1, ask, add}, + { ExchangeOrderKey(701267991871), 1161068569, p(2.925), 9, bid, add}, + { ExchangeOrderKey(701267963307), 1161037482, p(2.975), 1, ask, add}, + { ExchangeOrderKey(701267991967), 1161068670, p(2.900), 1, bid, add}, + { ExchangeOrderKey(701267928346), 1160999982, p(3.000), 1, ask, add}, + { ExchangeOrderKey(701267993125), 1161069878, p(2.900), 3, bid, add}, + { ExchangeOrderKey(701267037972), 1160052467, p(2.875), 1, bid, add}, + { ExchangeOrderKey(701267099137), 1160119020, p(2.875), 5, bid, add}, + { ExchangeOrderKey(701267299079), 1160330214, p(2.875), 1, bid, add}, + { ExchangeOrderKey(701267308619), 1160340068, p(2.875), 5, bid, add}, + { ExchangeOrderKey(701267310436), 1160341952, p(2.875), 3, bid, add}, + { ExchangeOrderKey(701267872642), 1160939720, p(2.875), 1, bid, add}, + { ExchangeOrderKey(701267895533), 1160964252, p(2.875), 1, bid, add}, + { ExchangeOrderKey(701267953989), 1161027476, p(2.875), 3, bid, add}, + { ExchangeOrderKey(701267993135), 1161069890, p(2.875), 2, bid, add}, + { ExchangeOrderKey(701266410447), 1159379621, p(2.850), 3, bid, add}, + { ExchangeOrderKey(701267324626), 1160356597, p(2.850), 3, bid, add}, + { ExchangeOrderKey(701267992564), 1161069297, p(2.850), 1, bid, add}, + { ExchangeOrderKey(701266420294), 1159389827, p(2.775), 5, bid, add}, + { ExchangeOrderKey(701267334462), 1160366796, p(2.775), 5, bid, add}, + { ExchangeOrderKey(701266411103), 1159380297, p(2.625), 5, bid, add}, + { ExchangeOrderKey(701267325284), 1160357275, p(2.625), 5, bid, add}, + }; + + apply_incremental(state, snapshot); + REQUIRE(state.bid.size() == 19); + REQUIRE(state.ask.size() == 3); + + std::vector incremental = { + { ExchangeOrderKey(701267993738), 1161070510, p(2.925), 2, bid, add}, + { ExchangeOrderKey(701267993759), 1161070532, p(2.975), 3, ask, add}, + { ExchangeOrderKey(701267993884), 1161070662, p(2.950), 3, ask, add}, + { ExchangeOrderKey(701267994228), 1161071029, p(2.925), 2, bid, add}, + { ExchangeOrderKey(701267994243), 1161071044, p(2.925), 4, bid, add}, + { ExchangeOrderKey(701267994228), 1161071029, p(2.925), 2, bid, remove}, + { ExchangeOrderKey(701267993135), 1161071281, p(2.875), 3, bid, change}, + }; + apply_incremental(state, incremental); + REQUIRE(state.bid.size() == 21); + REQUIRE(state.ask.size() == 5); + + { + auto iter = state.bid.begin(); + auto next = state.bid.begin(); + next++; + for (; next != state.bid.end(); ++iter, ++next) { + auto cur_price = std::get(iter->first); + auto cur_prio = std::get<1>(iter->first); + auto next_price = std::get(next->first); + auto next_prio = std::get<1>(next->first); + REQUIRE(cur_price >= next_price); + if (cur_price == next_price) { + REQUIRE(cur_prio < next_prio); + } + } + } + { + auto iter = state.ask.begin(); + auto next = state.ask.begin(); + next++; + for (; next != state.ask.end(); ++iter, ++next) { + auto cur_price = std::get(iter->first); + auto cur_prio = std::get<1>(iter->first); + auto next_price = std::get(next->first); + auto next_prio = std::get<1>(next->first); + REQUIRE(cur_price <= next_price); + if (cur_price == next_price) { + REQUIRE(cur_prio < next_prio); + } + } + } + } }}} diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index 62a771ee3..b06b185f7 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -134,7 +134,6 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con for (const auto& ask : book.offers()) { snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); } - // _mbo->publish_snapshot(SecurityKey(key), snapshot); _mbo->validate_snapshot(sec.id(), SecurityKey(key), snapshot); } } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index 13b70e1d5..d368c8661 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -114,12 +114,10 @@ public: int qty; entry.displayQty(qty); uint8_t ref; - LOG(md,Info, "GOT IRB46:"); if (entry.referenceId(ref)) { // entries is indexed by 0 but ref is indexed by 1 :facepalm: const auto& ref_entry = args.message().entries()[(int)ref-1]; uint32_t secid = ref_entry.securityId(); - LOG(md,Info, "> GOT IRB46 exch:{} rptSeq:{}", secid, ref_entry.rptSeq()); auto data = _index.get_security(secid); if (!data) continue; From a123ac2ed8d4701c8764f8dab50f1f5b9c9bf6e5 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 25 Apr 2019 15:14:27 -0400 Subject: [PATCH 074/131] Turn on MBO background processing --- src/lib/config/protos/MarketData.proto | 2 +- src/lib/md/MBOProcessor.cpp | 38 ++++++++++--------- src/server/sparkd/cmds/Run.cpp | 3 +- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 8 ++-- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index 607bfd1a2..250936649 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -124,7 +124,7 @@ message CMEOptions optional bool load_security_db = 33 [default = true]; reserved 34; // optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED]; - optional bool maintain_mbo = 35 [default = true]; + optional bool validate_mbo = 35 [default = false]; } optional string onixs_license = 1 [default = "{default}"]; diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 3ec4f31ca..1e0c29dcd 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -109,42 +109,44 @@ void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { } bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) { - // TO USE THIS VALIDATION YOU MUST MAKE enqueue_incremental syncronous + // bts_assert_msg(false, "TO USE THIS VALIDATION YOU MUST MAKE enqueue_incremental SYNCRONOUS"); #define REQUIRE(a,b,y) \ if ((a) != (b)) { \ LOG(md,Error, "Validation error exch:{} sec:{} {}!={} other:{}", exchid, key.value(), a, b, y); \ return false; \ } - // #define REQUIRE(x) bts_assert(x); // retrieve current snapshot and check they are equal - MBOReader reader(_me); - auto snap = reader.acquire_snapshot(key); - if (snap) { - REQUIRE (snap->bids.size(), control->bids.size(), '!'); - REQUIRE (snap->asks.size(), control->asks.size(), '!'); - for (size_t x = 0; x < snap->bids.size(); ++x) { - REQUIRE (snap->bids[x].id, control->bids[x].id, x); - REQUIRE (snap->bids[x].size, control->bids[x].size, x); + _me->dispatcher.post([this,exchid,key,control]() { + MBOReader reader(_me); + auto snap = reader.acquire_snapshot(key); + if (snap) { + REQUIRE (snap->bids.size(), control->bids.size(), '!'); + REQUIRE (snap->asks.size(), control->asks.size(), '!'); + for (size_t x = 0; x < snap->bids.size(); ++x) { + REQUIRE (snap->bids[x].id, control->bids[x].id, x); + REQUIRE (snap->bids[x].size, control->bids[x].size, x); + } + for (size_t x = 0; x < snap->asks.size(); ++x) { + REQUIRE (snap->asks[x].id, control->asks[x].id, x); + REQUIRE (snap->asks[x].size, control->asks[x].size, x); + } } - for (size_t x = 0; x < snap->asks.size(); ++x) { - REQUIRE (snap->asks[x].id, control->asks[x].id, x); - REQUIRE (snap->asks[x].size, control->asks[x].size, x); - } - } + return true; + }); #undef REQUIRE return true; } void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { - LOG(md,Info, "sec:{} : {}", state.handle.key(), update); + LOG(md,Diag, "sec:{} : {}", state.handle.key(), update); - // _me->dispatcher.post([this,&state,update]() { + _me->dispatcher.post([this,&state,update]() { // fast part apply_incremental(state,update); // slow part publish_incremental(state); - // }); + }); } void MBOWriter::publish_incremental(MBOWorkingState& state) { // determine if we need to publish diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index d225abfde..547e04fc0 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -61,7 +61,8 @@ int Run::execute(std::vector const &) { user::AuthServiceServer login_server(sc); md::MBOProcessor mbo(&utils::Dispatcher::get_named("MBO")); - md::run_market_datapublishers(&mbo.writer()); + auto* mbowriter = config::get().enabled() ? &mbo.writer() : nullptr; + md::run_market_datapublishers(mbowriter); api::synthetic::SyntheticSecurityService synthetic_service(sc); md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service, &mbo.reader()); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 025de1078..e1880efc4 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -117,11 +117,11 @@ map> create_feed_engines(const vector& symbols, MBOWriter* mbo) { } for (auto& pair : _handlers) { - onixs::SessionSettings settings = session_settings_for_channel(pair.first); + onixs::SessionSettings settings = session_settings_for_channel(pair.first, mbo); pair.second->start(settings); } LOG(md,Info, "** CME MDP3 Publisher started **"); @@ -210,7 +210,7 @@ unique_ptr MDP3Main::prepare_handler(const string& channel_id, M settings.bookManagement().directBooks().maintain(true); settings.bookManagement().impliedBooks().maintain(true); settings.bookManagement().consolidatedBooks().maintain(false); - if (options.maintain_mbo() && mbo != nullptr) { + if (options.validate_mbo() && mbo != nullptr) { settings.bookManagement().mboBooks().maintain(true); } From dfd0cfc2171982a9ab9250739b5a05db94042008 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 25 Apr 2019 17:16:35 -0400 Subject: [PATCH 075/131] templatize mbo callbacks for older templates --- .../md/platforms/mdp3_onixs/MDP3Listener.h | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index d368c8661..1fdb93393 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -104,10 +104,14 @@ public: // // Handle MBO incremental updates directly - void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args&) override { - LOG(md,Error, "GOT IRB32"); + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args& args) override { + onIncrementalRefreshBook(args); }; void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args& args) override { + onIncrementalRefreshBook(args); + } + template + void onIncrementalRefreshBook(const Args& args) { if (!_mbo) return; for(const auto& entry : args.message().orderIdEntries()) { @@ -147,12 +151,16 @@ public: }); } } - }; + } - void onMessage(onixs::Handler&, const onixs::IncrementalRefreshOrderBook43Args&) override { - LOG(md,Info, "GOT IROB43"); + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshOrderBook43Args& args) override { + onIncrementalRefreshOrderBook(args); } void onMessage(onixs::Handler&, const onixs::IncrementalRefreshOrderBook47Args& args) override { + onIncrementalRefreshOrderBook(args); + } + template + void onIncrementalRefreshOrderBook(const Args& args) { if (!_mbo) return; for (const auto& entry : args.message().entries()) { auto data = _index.get_security(entry.securityId()); @@ -193,11 +201,14 @@ public: } } - void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook44Args&) override { - LOG(md,Info, "GOT SNAPSHOT44"); + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook44Args& args) override { + onSnapshotFullRefreshOrderBook(args); } - void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook53Args& args) override { + onSnapshotFullRefreshOrderBook(args); + } + template + void onSnapshotFullRefreshOrderBook(const Args& args) { if (!_mbo) return; auto data = _index.get_security(args.message().securityId()); if (!data) return; From b129bea1296d57f19ebaef6c51a77ccf4ef2766f Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 26 Apr 2019 10:04:07 -0500 Subject: [PATCH 076/131] PR edit - method signature all on one line --- src/client/spark/Orders/OrdersModel.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 5be617a10..9793e8353 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -239,8 +239,7 @@ namespace mbts.spark { public void Dispose() => Disposable.dispose(ref _dispose); - static ImmutableDictionary - _update_working_orders(ImmutableDictionary target, OrderUpdate table) { + static ImmutableDictionary _update_working_orders(ImmutableDictionary target, OrderUpdate table) { var clone = target.ToBuilder(); foreach (var o in table.removed) { if (o.order.is_active()) { @@ -263,8 +262,7 @@ namespace mbts.spark { return clone.ToImmutableDictionary(); } - static ImmutableDictionary - _update_working_orders(ImmutableDictionary target, OrderUpdatePiq table) { + static ImmutableDictionary _update_working_orders(ImmutableDictionary target, OrderUpdatePiq table) { var clone = target.ToBuilder(); foreach (var o in table.removed) { if (o.order.is_active()) { From b102b271b03d2e8c46db630afee4bf17e7595c47 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 26 Apr 2019 10:08:10 -0500 Subject: [PATCH 077/131] PR edit - use single_order instead; this will be gone from attach/detach --- src/client/spark/view/Spark/Ladder/LadderOrderVM.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs b/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs index 2768b017b..a78ff0443 100644 --- a/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs @@ -112,7 +112,7 @@ namespace mbts.spark { var single_order = _orders.Values.First().order; var single_order_metadata = _parent.model.order_gateway.factory.get_metadata(single_order); border = single_order_metadata.Color.rgb_brush(); - is_exchange_order = OrderUtils.is_exchange_order(_orders.Values.First().order); + is_exchange_order = OrderUtils.is_exchange_order(single_order); } else { border = new SolidColorBrush(color); is_exchange_order = false; From 43a18941a40c72393f8804d6df6c027a9b8b3595 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 26 Apr 2019 16:16:59 -0400 Subject: [PATCH 078/131] Change debug assert to logged warning in PIQ --- src/lib/md/MBOProcessor.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index 1e0c29dcd..fe1627a74 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -184,10 +184,12 @@ void apply_incremental(C& collection, std::unordered_mapsecond; collection.erase(oldkey); index.erase(r); - index[update.order_id] = key; } - else { bts_assert_d(false); } + else { + LOG(md,Warn, "MBO Update w/out Add for: {}", update); + } collection.insert(std::make_pair(key, MBOWorkingState::Data { update.order_id, update.size })); + index[update.order_id] = key; } break; case inc::UpdateAction::Delete: @@ -242,10 +244,10 @@ ostream& operator<<(ostream& os, const inc::Update& update) { switch(update.type) { case inc::UpdateType::Bid: - os << "bid "; + os << "B "; break; case inc::UpdateType::Ask: - os << "ask "; + os << "A "; break; } From 92f67d1d2d67394b38f7ff2c602cbe24b80ce6f3 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 26 Apr 2019 16:40:59 -0400 Subject: [PATCH 079/131] piq: Add security key to log message --- src/lib/md/MBOProcessor.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp index fe1627a74..88cc1f3f2 100644 --- a/src/lib/md/MBOProcessor.cpp +++ b/src/lib/md/MBOProcessor.cpp @@ -170,7 +170,7 @@ void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { } template -void apply_incremental(C& collection, std::unordered_map& index, const inc::Update& update) { +void apply_incremental(uint32_t security_key, C& collection, std::unordered_map& index, const inc::Update& update) { auto key = std::make_tuple(update.price, update.priority); switch(update.action) { case inc::UpdateAction::New: @@ -186,7 +186,7 @@ void apply_incremental(C& collection, std::unordered_map Date: Sun, 28 Apr 2019 16:31:36 -0400 Subject: [PATCH 080/131] Lean fixes --- src/lib/domain_model/Security.h | 4 ++ .../orders/algo/algo_orders/LeanAlgo.cpp | 49 ++++++++++++------- .../orders/algo/algo_orders/LeanAlgo.h | 1 + .../spreader_orders/spreader/SpreaderUtils.h | 15 ++++++ 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/src/lib/domain_model/Security.h b/src/lib/domain_model/Security.h index 25e348517..c7eacbc41 100644 --- a/src/lib/domain_model/Security.h +++ b/src/lib/domain_model/Security.h @@ -167,5 +167,9 @@ inline bool test_price_equality(double price_1, double price_2, inline bool gr_than_eq(double p1, double p2, double tick_size) { return test_price_equality(p1, p2, tick_size) || p1 >= p2; } +inline bool gr_than(double p1, double p2, double tick_size) { + return gr_than_eq(p1, p2, tick_size) && + !test_price_equality(p1,p2,tick_size); +} } // namespace securities } // namespace bts diff --git a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp index 36ec9ef69..de9eb5883 100644 --- a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp @@ -17,7 +17,7 @@ using namespace api::orderman; enum class LeanAlgo::Status { WAITING_FOR_MD, WORKING, DONE }; struct LeanAlgo::PImpl { - IAlgoServices *algo; + IAlgoServices* algo_services; double tick_size = os::nan(); Status status = Status::WAITING_FOR_MD; Price initial_observed_price = Price(os::nan()); @@ -29,7 +29,7 @@ struct LeanAlgo::PImpl { LeanAlgo::LeanAlgo() : me(new PImpl()) {} LeanAlgo::~LeanAlgo() {} -IAlgoServices &LeanAlgo::algo() const { return *me->algo; } +IAlgoServices &LeanAlgo::algo() const { return *me->algo_services; } // initialization ///////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// @@ -40,7 +40,8 @@ OrderOpResult LeanAlgo::do_before_launch(IBeforeLaunchAlgoServices &serv) { bts_assert(def.custom_def().has_lean()); // validate that lean is properly formed if (serv.is_attaching()) { - } else { + } + else { auto child_def = def; *child_def.mutable_custom_def() = def.custom_def().lean().child(); @@ -48,14 +49,14 @@ OrderOpResult LeanAlgo::do_before_launch(IBeforeLaunchAlgoServices &serv) { } return {}; } -void LeanAlgo::do_launch(IAlgoServices&algo) { - me->algo = &algo; +void LeanAlgo::do_launch(IAlgoServices& algo) { + me->algo_services = &algo; bts_assert(me->child_template); me->child_key = algo.launch_child(*me->child_template, algo.price(), algo.quantity()); algo.subscribe_book(SecurityKey(algo.definition().security_key())); } void LeanAlgo::do_launch_attached(IAlgoServices&algo,OrderKey k, ChildState st) { - me->algo = &algo; + me->algo_services = &algo; me->child_key = k; algo.subscribe_book(SecurityKey(algo.definition().security_key())); on_child_change(k, st); @@ -126,7 +127,7 @@ void LeanAlgo::on_child_change(OrderKey, const ChildState &st) { return; } } -// helpers //////////////////////////////////////////////////////////////////// +// accessor helpers /////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// spark::QuantityMeasure LeanAlgo::_cross_threshold() const { return algo().definition().custom_def().lean().cross_threshold(); @@ -152,6 +153,28 @@ md::PriceSize LeanAlgo::_opp_inside_market() const { api::Side LeanAlgo::_opp_side() const { return securities::other_side(algo().definition().side()); } +bool LeanAlgo::_within_chase_band(Price price) const{ + return securities::is_better(_opp_side(), + price, + me->initial_observed_price) || + static_cast(std::abs( + securities::price_diff_to_ticks(price.value(), + me->initial_observed_price.value(), + me->tick_size))) < _max_chase_ticks(); +} +// actions //////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +void LeanAlgo::_analyze_and_act() { + bts_assert(me->status == LeanAlgo::Status::WORKING); + if (_book_moved_toward()) + _on_toward_me(); + else if (_book_moved_away()) { + me->observed_price = Price(_opp_inside_market().price()); + _try_cross(); + } + else + _try_cross(); +} void LeanAlgo::_on_toward_me() { switch (_on_toward_behaviour()) { case LeanOrderDefinition_OnToward_CANCEL: @@ -186,20 +209,12 @@ void LeanAlgo::_on_toward_me() { break; } } -void LeanAlgo::_analyze_and_act() { - bts_assert(me->status == LeanAlgo::Status::WORKING); - if (_book_moved_toward()) - _on_toward_me(); - else if (_book_moved_away()) { - me->observed_price = Price(_opp_inside_market().price()); - _try_cross(); - } else - _try_cross(); -} void LeanAlgo::_try_cross() { + Price price=me->observed_price; md::PriceSize opp_inside = _opp_inside_market(); Quantity trigger_quantity = SpreaderUtils::_quantity_threshold( _cross_threshold(), Quantity(algo().quantity())); + if (opp_inside.size() != 0 && opp_inside.size() < trigger_quantity.value()) { // cross threshold is good, check // for chase diff --git a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h index 1e38b515f..3872500d1 100644 --- a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h +++ b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h @@ -31,6 +31,7 @@ class LeanAlgo : public IAlgoOrder { bool _book_moved_away() const; md::PriceSize _opp_inside_market() const; api::Side _opp_side() const; + bool _within_chase_band(Price price) const; void _analyze_and_act(); void _try_cross(); void _on_toward_me(); diff --git a/src/lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h b/src/lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h index fecbf9430..6f1c05c30 100644 --- a/src/lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h +++ b/src/lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h @@ -261,6 +261,21 @@ public: price_qty = Quantity(0); cumm_qty = Quantity(0); return false; } + // returns true if the price (3rd arg) + // crosses into the side (2nd arg) of the + // book (1rst arg). if the side of the + // book is empty, the function returns + // false. + static bool crosses_book(const Book& book, + api::Side side, + Price price, + double tick_size){ + const auto& book_side = book.side(side); + return (side==api::SIDE_BUY && book_side[0].size() > 0 && + securities::gr_than_eq(book_side[0].price(),price.value(),tick_size)) || + (side==api::SIDE_SELL && book_side[0].size() > 0 && + securities::gr_than_eq(price.value(),book_side[0].price(),tick_size)); + } // find the top of the synthetic book static Price top_of_book_price (const ISpreaderManager&, api::Side){ From 1dba1034b6422416eda4189061d1c837fa1c2621 Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Sun, 28 Apr 2019 23:01:42 -0400 Subject: [PATCH 081/131] Lean simplified a bit more --- src/lib/domain_model/Security.h | 5 ++ .../orders/algo/algo_orders/LeanAlgo.cpp | 54 +++++++++++++------ .../orders/algo/algo_orders/LeanAlgo.h | 3 ++ 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/src/lib/domain_model/Security.h b/src/lib/domain_model/Security.h index c7eacbc41..869556bb8 100644 --- a/src/lib/domain_model/Security.h +++ b/src/lib/domain_model/Security.h @@ -61,6 +61,11 @@ inline Price improve_price(api::Side side, Price p, Price offset) { return Price(0); } +inline Price worsen_price(api::Side side, Price p, + double tick_size,uint32_t offset_ticks) { + return improve_price(side, p, Price(tick_size*offset_ticks)); +} + inline Price worsen_price(api::Side side, Price p, Price offset) { return improve_price(other_side(side), p, offset); } diff --git a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp index de9eb5883..82b87b9cc 100644 --- a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp @@ -24,7 +24,8 @@ struct LeanAlgo::PImpl { Price observed_price = Price(os::nan()); OrderKey child_key = null_order_key(); ChildTemplate *child_template = nullptr; - Book last_book; + md::PriceSize last_opp_inside=md::PriceSize(); + Book curr_book; }; LeanAlgo::LeanAlgo() : me(new PImpl()) {} @@ -79,7 +80,6 @@ void LeanAlgo::do_update(const api::orderman::Update &update) { *child_update.mutable_custom() = lean_update.child(); } algo().update_child(me->child_key, child_update); - switch (me->status) { case LeanAlgo::Status::WAITING_FOR_MD: break; @@ -93,7 +93,7 @@ void LeanAlgo::do_update(const api::orderman::Update &update) { // callbacks ////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// void LeanAlgo::on_book(SecurityKey, const Book &book) { - me->last_book = book; + me->curr_book = book; switch (me->status) { case LeanAlgo::Status::WAITING_FOR_MD: if (_opp_inside_market().size() != 0) { @@ -140,15 +140,17 @@ LeanAlgo::_on_toward_behaviour() const { return algo().definition().custom_def().lean().trade_toward_behavior(); } bool LeanAlgo::_book_moved_toward() const { - return securities::is_better( - _opp_side(), Price(_opp_inside_market().price()), me->observed_price); + return securities::is_better(_opp_side(), + Price(_opp_inside_market().price()), + Price(me->last_opp_inside.price())); } bool LeanAlgo::_book_moved_away() const { - return securities::is_better(_opp_side(), me->observed_price, + return securities::is_better(_opp_side(), + Price(me->last_opp_inside.price()), Price(_opp_inside_market().price())); } md::PriceSize LeanAlgo::_opp_inside_market() const { - return me->last_book.side(_opp_side())[0]; + return me->curr_book.side(_opp_side())[0]; } api::Side LeanAlgo::_opp_side() const { return securities::other_side(algo().definition().side()); @@ -162,18 +164,41 @@ bool LeanAlgo::_within_chase_band(Price price) const{ me->initial_observed_price.value(), me->tick_size))) < _max_chase_ticks(); } +Price LeanAlgo::_chase_limit() const{ + return securities::worsen_price(_opp_side(),me->initial_observed_price, + me->tick_size,_max_chase_ticks()); +} +LeanAlgo::MarketChange LeanAlgo::_market_change() const{ + md::PriceSize opp_inside=_opp_inside_market(); + if(opp_inside.size()==0) + return MarketChange::NO_MARKET; + else if (_book_moved_away()) + return MarketChange::AWAY; + else if (_book_moved_toward()) + return MarketChange::TOWARD; + else + return MarketChange::SIZE_CHANGE; +} // actions //////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// void LeanAlgo::_analyze_and_act() { bts_assert(me->status == LeanAlgo::Status::WORKING); - if (_book_moved_toward()) + auto market_change=_market_change(); + me->last_opp_inside=_opp_inside_market(); + switch(market_change){ + case MarketChange::NO_MARKET: + _try_cross(); + break; + case MarketChange::SIZE_CHANGE: + _try_cross(); + break; + case MarketChange::AWAY: + _try_cross(); + break; + case MarketChange::TOWARD: _on_toward_me(); - else if (_book_moved_away()) { - me->observed_price = Price(_opp_inside_market().price()); - _try_cross(); + break; } - else - _try_cross(); } void LeanAlgo::_on_toward_me() { switch (_on_toward_behaviour()) { @@ -200,7 +225,6 @@ void LeanAlgo::_on_toward_me() { case LeanOrderDefinition_OnToward_NO_CHANGE: LOG(OrderMan, Diag, "LeanAlgo {} ignoring opposite move towards order.", algo().order_key().value()); - // Change of behaviour : nothing == reprice me->observed_price = Price(_opp_inside_market().price()); _try_cross(); break; @@ -210,7 +234,7 @@ void LeanAlgo::_on_toward_me() { } } void LeanAlgo::_try_cross() { - Price price=me->observed_price; + //Price price=me->observed_price; md::PriceSize opp_inside = _opp_inside_market(); Quantity trigger_quantity = SpreaderUtils::_quantity_threshold( _cross_threshold(), Quantity(algo().quantity())); diff --git a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h index 3872500d1..8edadb5c4 100644 --- a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h +++ b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h @@ -21,6 +21,7 @@ class LeanAlgo : public IAlgoOrder { void on_child_change(OrderKey, ChildState const &) override; private: + enum class MarketChange {NO_MARKET,SIZE_CHANGE,AWAY,TOWARD}; struct PImpl; enum class Status; IAlgoServices &algo() const ; @@ -32,6 +33,8 @@ class LeanAlgo : public IAlgoOrder { md::PriceSize _opp_inside_market() const; api::Side _opp_side() const; bool _within_chase_band(Price price) const; + Price _chase_limit() const; + MarketChange _market_change() const; void _analyze_and_act(); void _try_cross(); void _on_toward_me(); From 342d89724e69f9efda4f65f462cb127e49ff89bb Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 29 Apr 2019 09:13:09 -0400 Subject: [PATCH 082/131] Update release notes --- doc/user/content/release_notes/_index.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index db2b50254..29fc0e4b3 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -4,6 +4,11 @@ weight = 3 sort_by = "weight" insert_anchor_links = "right" +++ +## 1.5.9 +### User Visible Changes + - You can now view your order's position in queue in the ladder + - Fix a bug in the order popup in the ladder which could lead to a crash. + ## 1.5.8 ### User Visible Changes - Fixed a bug where it was possible to crash the UI by entering an order through the order ticket if Enter was used to dismiss the dialog. From 61745d0ed79767e47d688e361502146e4ac00cec Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 29 Apr 2019 09:35:13 -0400 Subject: [PATCH 083/131] Update release notes --- doc/user/content/release_notes/_index.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index 29fc0e4b3..0d403e851 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -4,11 +4,15 @@ weight = 3 sort_by = "weight" insert_anchor_links = "right" +++ -## 1.5.9 +## 1.5.10 ### User Visible Changes - You can now view your order's position in queue in the ladder - Fix a bug in the order popup in the ladder which could lead to a crash. +## 1.5.9 +### User Visible Changes + - Fix for stunnel connections on Windows 7 + ## 1.5.8 ### User Visible Changes - Fixed a bug where it was possible to crash the UI by entering an order through the order ticket if Enter was used to dismiss the dialog. From 5a1cc4cf2b85088c7ac4f2a7d8b7b4291f133a11 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 29 Apr 2019 09:38:02 -0400 Subject: [PATCH 084/131] Update release notes --- doc/user/content/release_notes/_index.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index 0d403e851..3c158addf 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -4,9 +4,11 @@ weight = 3 sort_by = "weight" insert_anchor_links = "right" +++ -## 1.5.10 +## 1.6.0 ### User Visible Changes +#### New Features - You can now view your order's position in queue in the ladder +#### Bug Fixes - Fix a bug in the order popup in the ladder which could lead to a crash. ## 1.5.9 From e84c7d4f8c9d2fd9f66893e39af15d661d512648 Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Mon, 29 Apr 2019 13:01:31 -0400 Subject: [PATCH 085/131] Cleaning up lean again --- .../orders/algo/algo_orders/LeanAlgo.cpp | 42 +++++++++++-------- .../orders/algo/algo_orders/LeanAlgo.h | 4 +- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp index 82b87b9cc..4c0810c28 100644 --- a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp @@ -129,8 +129,10 @@ void LeanAlgo::on_child_change(OrderKey, const ChildState &st) { } // accessor helpers /////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// -spark::QuantityMeasure LeanAlgo::_cross_threshold() const { - return algo().definition().custom_def().lean().cross_threshold(); +Quantity LeanAlgo::_cross_threshold() const{ + auto qty_measure=algo().definition().custom_def().lean().cross_threshold(); + return SpreaderUtils::_quantity_threshold( + qty_measure, Quantity(algo().quantity())); } uint32_t LeanAlgo::_max_chase_ticks() const { return algo().definition().custom_def().lean().max_chase_ticks(); @@ -187,12 +189,14 @@ void LeanAlgo::_analyze_and_act() { me->last_opp_inside=_opp_inside_market(); switch(market_change){ case MarketChange::NO_MARKET: + // go to the limit and park there _try_cross(); break; case MarketChange::SIZE_CHANGE: _try_cross(); break; case MarketChange::AWAY: + me->observed_price = Price(_opp_inside_market().price()); _try_cross(); break; case MarketChange::TOWARD: @@ -225,7 +229,6 @@ void LeanAlgo::_on_toward_me() { case LeanOrderDefinition_OnToward_NO_CHANGE: LOG(OrderMan, Diag, "LeanAlgo {} ignoring opposite move towards order.", algo().order_key().value()); - me->observed_price = Price(_opp_inside_market().price()); _try_cross(); break; default: @@ -233,26 +236,29 @@ void LeanAlgo::_on_toward_me() { break; } } -void LeanAlgo::_try_cross() { - //Price price=me->observed_price; - md::PriceSize opp_inside = _opp_inside_market(); - Quantity trigger_quantity = SpreaderUtils::_quantity_threshold( - _cross_threshold(), Quantity(algo().quantity())); - - if (opp_inside.size() != 0 && opp_inside.size() < trigger_quantity.value()) { - // cross threshold is good, check - // for chase - uint32_t diff = std::abs(securities::price_diff_to_ticks( - opp_inside.price(), me->initial_observed_price.value(), - me->tick_size)); - if (diff <= _max_chase_ticks()) { - // payup +// this assumes the opposite side is there +// and will always try to cross into the other side +// IF the qty threshold AND chase band condition is met. +// it returns true if it successfully crossed. +bool LeanAlgo::_try_cross() { + if(_within_chase_band(me->observed_price)){ + Quantity level_qty, cumm_qty; + SpreaderUtils::find_qty_at_price_level(me->curr_book, + _opp_side(), + me->observed_price, + me->tick_size, + level_qty, + cumm_qty); + Quantity trigger_quantity = _cross_threshold(); + if(cumm_qty.value()observed_price.value()), child_update.set_quantity(algo().quantity().value()); algo().update_child(me->child_key, child_update); + return true; } } + return false; } } // namespace orderman } // namespace bts diff --git a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h index 8edadb5c4..4a2a2760e 100644 --- a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h +++ b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h @@ -25,7 +25,7 @@ class LeanAlgo : public IAlgoOrder { struct PImpl; enum class Status; IAlgoServices &algo() const ; - spark::QuantityMeasure _cross_threshold() const; + Quantity _cross_threshold() const; uint32_t _max_chase_ticks() const; api::orderman::LeanOrderDefinition::OnToward _on_toward_behaviour() const; bool _book_moved_toward() const; @@ -36,7 +36,7 @@ class LeanAlgo : public IAlgoOrder { Price _chase_limit() const; MarketChange _market_change() const; void _analyze_and_act(); - void _try_cross(); + bool _try_cross(); void _on_toward_me(); private: From 221814cc1125dbec6430b9984f6d86edf89f0e08 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 29 Apr 2019 16:30:18 -0400 Subject: [PATCH 086/131] Add OrderStress tool for perf testing --- src/client/sparkcmd/Cmd/OrderStress.cs | 157 +++++++++++++++++++++++++ src/client/sparkcmd/Program.cs | 5 +- 2 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 src/client/sparkcmd/Cmd/OrderStress.cs diff --git a/src/client/sparkcmd/Cmd/OrderStress.cs b/src/client/sparkcmd/Cmd/OrderStress.cs new file mode 100644 index 000000000..43f8d9765 --- /dev/null +++ b/src/client/sparkcmd/Cmd/OrderStress.cs @@ -0,0 +1,157 @@ +using bts.api; +using bts.api.core; +using bts.api.orderman; +using bts.co2; +using bts.test; +using CommandLine; +using mbts.co2; +using mbts.config; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace bts { + + [Verb("orderstress", HelpText = "Create a bunch of orders")] + public class OrderStressOptions { + // Maybe options to create fewer orders or sleep + + [Option('s', "sleep", HelpText = "Sleep ms between order submission", Default = 100)] + public int sleep { get; set; } + + [Option('d', "depth", HelpText = "How many orders per side", Default = 10u)] + public uint depth { get; set; } + + [Option('q', "quanityt", HelpText = "Order quantity", Default = 1u)] + public uint quantity { get; set; } + }; + + public struct Market { + public SecurityImmutable security; + public IntPrice? bid; + public IntPrice? ask; + } + + public static class OrderStress { + static IEnumerable all_markets(ServiceContext ctx) { + var channels = + UserSymbol.get_all() + .Select(u => + new SecurityChannel() { + Time = 0, + Flags = (int)(MDRequest.Types.Flags.Direct | MDRequest.Types.Flags.Implied), + Type = u.type, + Symbol = u.symbol, + Exchange = u.exchange + } + ); + var request = new MDRequest() { Host = "system_test" }; + request.Channels.AddRange(channels); + + var relay = ctx.locate(); + System.Threading.Thread.Sleep(100); + var result = relay.requestmd(request); + + foreach (var ch in result.Channels) { + foreach (var update in ch.Updates) { + if (update.Security != null) { + var m = new Market() { + security = new SecurityImmutable(update.Security), + bid = best( + (a,b) => a > b ? a : b, + update.Direct?.Bids?.FirstOrDefault(), update.Implied?.Bids?.FirstOrDefault()), + ask = best( + (a,b) => a < b ? a : b, + update.Direct?.Asks?.FirstOrDefault(), update.Implied?.Asks?.FirstOrDefault()) + }; + if (m.bid.HasValue || m.ask.HasValue) { + yield return m; + } + } + } + } + } + + private static IntPrice? best(Func comp, params PriceSizeMessage[] ps) { + IntPrice? result = null; + foreach (var p in ps) { + if (p != null) { + var ip = new IntPrice(p.Mantissa, (sbyte)p.Exponent); + if (result == null) { + result = ip; + } + else { + result = comp(result.Value, ip); + } + } + } + return result; + } + + static Create mkorder(uint security, double price, uint size, Side side) { + return new Create { + Price = price, + Quantity = size, + NewOrder = new OrderDefinition { + SecurityKey = security, + Side = side, + TimeInForce = TimeInForce.Day, + ManualOrderIndicator = ManualOrderIndicator.Yes, + CtiCode = CTICode.CtiCode1, + CustomerOrFirm = CustomerOrFirm.Firm, + AccountId = 1, + UserId = 1, + CustomDef = new CustomOrderDefinition { + Limit = new LimitOrderDefinition() + } + } + }; + } + + public static int Run(OrderStressOptions o) { + + var ctx = new ServiceContext(); + ConsoleEx.WriteLine("Please log in to Spark", ConsoleColor.Yellow); + ConsoleEx.Write("Username: ", ConsoleColor.Yellow); + var name = Console.ReadLine(); + ConsoleEx.Write("Password: ", ConsoleColor.Yellow); + var pw = Console.ReadLine(); + + var login_result = ctx.login(name, pw); + if (!login_result.Result.success) { + ConsoleEx.WriteLine("Invalid Credentials", ConsoleColor.Red); + } + + if (Configuration.get().type.ToUpper() == "PROD") { + ConsoleEx.WriteLine("Refusing to run on production system", ConsoleColor.Red); + return -1; + } + + var orders = ctx.locate(); + foreach (var m in all_markets(ctx)) { + ConsoleEx.WriteLine(m.security.security_desc(), ConsoleColor.White); + var tick = m.security.ContractSpec.LowTick; + if (m.bid.HasValue) { + // start at bid and place 1 order at 10 levels + for (var x=0; x < o.depth; ++x) { + var price = m.bid.Value.price() - (tick * x); + ConsoleEx.WriteLine($"Buy {o.quantity} at {price}", ConsoleColor.Blue); + orders.create_order_async(mkorder(m.security.Id, price, o.quantity, Side.Buy)); + System.Threading.Thread.Sleep(o.sleep); + } + } + if (m.ask.HasValue) { + for (var x = 0; x < o.depth; ++x) { + var price = m.ask.Value.price() + (tick * x); + ConsoleEx.WriteLine($"Sell {o.quantity} at {price}", ConsoleColor.Red); + orders.create_order_async(mkorder(m.security.Id, price, o.quantity, Side.Sell)); + System.Threading.Thread.Sleep(o.sleep); + } + } + } + return 3; + } + } + +} \ No newline at end of file diff --git a/src/client/sparkcmd/Program.cs b/src/client/sparkcmd/Program.cs index 9c56c1746..a91fed7f2 100644 --- a/src/client/sparkcmd/Program.cs +++ b/src/client/sparkcmd/Program.cs @@ -26,14 +26,15 @@ namespace sparkcmd { mbts.utils.BTSNativeMethods.provide_command_line(args, args.Length); // note: this can be made to infer the options type list from the list of delegates return Parser.Default - .ParseArguments(args) - .MapResult( + .ParseArguments(args) + .MapResult( Configcheck.Run, ScriptClientCommand.Run, TailService.Run, ServiceTest.Run, SettingsManagerCommand.Run, TestCommand.Run, + OrderStress.Run, errors => 1); } } From 4279e3d388429de7e40359140f67ec655b42c100 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 29 Apr 2019 16:36:16 -0400 Subject: [PATCH 087/131] Fix broken managed md unit test --- src/client/spark_api/Co2/Co2NativeMethods.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/spark_api/Co2/Co2NativeMethods.cs b/src/client/spark_api/Co2/Co2NativeMethods.cs index 495230306..f744b9963 100644 --- a/src/client/spark_api/Co2/Co2NativeMethods.cs +++ b/src/client/spark_api/Co2/Co2NativeMethods.cs @@ -35,6 +35,7 @@ namespace mbts.co2 public readonly IntPtr hot; public readonly IntPtr cold_data; public readonly IntPtr cold_header; + private readonly IntPtr unused_mbo; } [DllImport("libspark")] From 6066e2162c35359f73ea9d7449ccc2cffa934955 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 29 Apr 2019 17:28:23 -0400 Subject: [PATCH 088/131] SPARK-2197 Correction to PIQ calculation per taha (+test coverage) --- src/lib/md/MBOServiceServer.cpp | 6 +++--- src/lib/md/MBOServiceServer.h | 2 ++ src/lib/md/test/mbo_tests.cpp | 25 +++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp index e25715ee4..6cabb35a0 100644 --- a/src/lib/md/MBOServiceServer.cpp +++ b/src/lib/md/MBOServiceServer.cpp @@ -148,14 +148,14 @@ std::optional search_orders(const ExchangeOrderKey oid, const std::vec const auto& oo = orders[x]; if (oo.id == oid) { if (oo.price == price) return { position }; - else return 0 ; + else return 0; } else if (price == oo.price) { - position += 1; + position += oo.size; } else { price = oo.price; - position = 1; + position = oo.size; } } return std::nullopt; diff --git a/src/lib/md/MBOServiceServer.h b/src/lib/md/MBOServiceServer.h index d0f590a18..6eccab38c 100644 --- a/src/lib/md/MBOServiceServer.h +++ b/src/lib/md/MBOServiceServer.h @@ -2,6 +2,7 @@ #include "lib/api/public/ServiceContext.h" #include "lib/api/services/MBOService.server.pb.h" #include +#include namespace bts { namespace utils { class Dispatcher; } @@ -24,4 +25,5 @@ class MBOServiceServer : public MBOServiceBase { }; + }} //namespace diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp index a275a54c0..69abb194e 100644 --- a/src/lib/md/test/mbo_tests.cpp +++ b/src/lib/md/test/mbo_tests.cpp @@ -9,6 +9,10 @@ namespace bts { namespace md { + +// fwd declare from MBOServiceServer.cpp +std::optional search_orders(const ExchangeOrderKey oid, const std::vector& orders); + namespace test { int mbo_tests = 0; @@ -202,6 +206,7 @@ TEST_CASE("MBO sort order", "[md][mbo]") { } } + TEST_CASE("MBO change price and size", "[md][mbo]") { // When an order's size goes up it gets a new priority Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; @@ -288,7 +293,27 @@ TEST_CASE("MBO change price and size", "[md][mbo]") { } } } + + // Do some tests on the snapshot + MBOSnapshot result; + copy_to_snapshot(state, &result); + + auto ask30 = search_orders(ExchangeOrderKey(701267928346), result.asks); + REQUIRE(ask30); + REQUIRE(*ask30 == 0); + + auto bid2875 = search_orders(ExchangeOrderKey(701267993135), result.bids); + REQUIRE(bid2875); + REQUIRE(*bid2875 == 20); + bid2875 = search_orders(ExchangeOrderKey(701267299079), result.bids); + REQUIRE(bid2875); + REQUIRE(*bid2875 == 6); + + auto not_bid = search_orders(ExchangeOrderKey(701967299879), result.bids); + auto not_ask = search_orders(ExchangeOrderKey(701967299879), result.asks); + REQUIRE(!not_bid); + REQUIRE(!not_ask); } }}} From 98315d8e15e90c60470a51550a67183786f8c150 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 29 Apr 2019 13:38:38 -0500 Subject: [PATCH 089/131] refactor 'GridOrLadderSettingsSelectorKeys' to 'GridSettingsTemplateKeys' --- .../spark/Modules/Trading/Ladder/VM/LadderSettingsVM.cs | 2 +- .../view/Controls/DIYGrid/Settings/GridSettingsVM.cs | 2 +- .../DIYGrid/Settings/SimpleGridSettingsDialog.xaml | 8 ++++---- .../DIYGrid/Settings/SimpleGridSettingsDialog.xaml.cs | 5 +++-- .../Controls/DIYGrid/Settings/SimpleGridSettingsVM.cs | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/client/spark/Modules/Trading/Ladder/VM/LadderSettingsVM.cs b/src/client/spark/Modules/Trading/Ladder/VM/LadderSettingsVM.cs index 1dd5abd37..534145cbe 100644 --- a/src/client/spark/Modules/Trading/Ladder/VM/LadderSettingsVM.cs +++ b/src/client/spark/Modules/Trading/Ladder/VM/LadderSettingsVM.cs @@ -195,7 +195,7 @@ namespace mbts.spark { } - public string key => GridOrLadderSettingsSelectorKeys.Ladder; + public string key => GridSettingsTemplateKeys.Ladder; public void load(LadderSettings settings) { _current = settings; diff --git a/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsVM.cs b/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsVM.cs index fc859fd0d..78fb6e804 100644 --- a/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsVM.cs +++ b/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsVM.cs @@ -115,7 +115,7 @@ namespace mbts.spark { new DialogSkirtItem("Apply", new DelegateCommand(() => committed?.Invoke())) }; - public string key => ui.spark.GridOrLadderSettingsSelectorKeys.SimpleGrid; + public string key => ui.spark.GridSettingsTemplateKeys.SimpleGrid; public TSettingsData get_result() { var animation_map = predicate_animations diff --git a/src/client/spark/view/Controls/DIYGrid/Settings/SimpleGridSettingsDialog.xaml b/src/client/spark/view/Controls/DIYGrid/Settings/SimpleGridSettingsDialog.xaml index 26dff2922..19b7cdef2 100644 --- a/src/client/spark/view/Controls/DIYGrid/Settings/SimpleGridSettingsDialog.xaml +++ b/src/client/spark/view/Controls/DIYGrid/Settings/SimpleGridSettingsDialog.xaml @@ -65,13 +65,13 @@ - - + + - + @@ -89,7 +89,7 @@ Grid.Row="1" VerticalAlignment="Top" Content="{Binding instance_vm}" - ContentTemplateSelector="{StaticResource GridOrLadderSettingsSelector}" + ContentTemplateSelector="{StaticResource GridSettingsTemplateSelector}" /> committed?.Invoke())) }; - public string key => ui.spark.GridOrLadderSettingsSelectorKeys.SimpleGrid; + public string key => ui.spark.GridSettingsTemplateKeys.SimpleGrid; public GridSettings create() => new GridSettings(font_family.name(), font_size, font_weight, font_color.color.rgb_color(), selection_highlight.to_proto(), columns.Select(c => c.create())); From 7ce7ebbf966fbeec094c8abd32bb4bed6a827760 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 29 Apr 2019 13:46:34 -0500 Subject: [PATCH 090/131] remove erroneous/legacy RadioButton default style from SimpleGridSettingsDialog.xaml --- .../Settings/SimpleGridSettingsDialog.xaml | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/src/client/spark/view/Controls/DIYGrid/Settings/SimpleGridSettingsDialog.xaml b/src/client/spark/view/Controls/DIYGrid/Settings/SimpleGridSettingsDialog.xaml index 19b7cdef2..b1d8b532e 100644 --- a/src/client/spark/view/Controls/DIYGrid/Settings/SimpleGridSettingsDialog.xaml +++ b/src/client/spark/view/Controls/DIYGrid/Settings/SimpleGridSettingsDialog.xaml @@ -15,44 +15,6 @@ - - - - @@ -311,7 +307,10 @@ - + diff --git a/src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml b/src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml index 10493f539..c6bc3dd1f 100644 --- a/src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml +++ b/src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml @@ -111,6 +111,7 @@ From 04eee0f4b778f22a0118e9b05a4f8dff2928bafa Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Wed, 1 May 2019 14:00:18 -0400 Subject: [PATCH 128/131] added logs in Lean to detect the out-of-the-gate fill situation --- .../orderman/orders/algo/algo_orders/LeanAlgo.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp index 4c0810c28..e69ca3e62 100644 --- a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp @@ -54,6 +54,10 @@ void LeanAlgo::do_launch(IAlgoServices& algo) { me->algo_services = &algo; bts_assert(me->child_template); me->child_key = algo.launch_child(*me->child_template, algo.price(), algo.quantity()); + LOG(OrderMan, Info, + "LeanAlgo::do_launch key:{} child_key:{} price:{} qty:{}", + algo.order_key().value(),me->child_key, + algo.price(),algo.quantity()); algo.subscribe_book(SecurityKey(algo.definition().security_key())); } void LeanAlgo::do_launch_attached(IAlgoServices&algo,OrderKey k, ChildState st) { @@ -110,7 +114,12 @@ void LeanAlgo::on_book(SecurityKey, const Book &book) { break; } } -void LeanAlgo::on_child_change(OrderKey, const ChildState &st) { +void LeanAlgo::on_child_change(OrderKey key, const ChildState &st) { + LOG(OrderMan, Info, + "LeanAlgo::on_child_change key:{} child_key:{} status:{} total_fills:{} " + "last_price:{}", + algo().order_key().value(),key, + static_cast(st.status),st.total_fills,st.last_price); algo().update_fill_state(st.total_fills, st.last_price); switch (st.status) { case ChildStatus::WORKING: @@ -254,6 +263,10 @@ bool LeanAlgo::_try_cross() { Update child_update; child_update.set_price(me->observed_price.value()), child_update.set_quantity(algo().quantity().value()); + LOG(OrderMan, Info, "LeanAlgo::_try_cross updating child " + "key:{} child_key:{} price:{} qty:{}", + algo().order_key().value(),me->child_key, + child_update.price(),child_update.quantity()); algo().update_child(me->child_key, child_update); return true; } From 791774490c8967b98ba2d83f4ddfa144769deaca Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Wed, 1 May 2019 13:31:21 -0400 Subject: [PATCH 129/131] Clang build fixes. --- src/lib/api/public/ClientChannel.h | 2 +- src/lib/utils/ConcurrentPool.h | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/lib/api/public/ClientChannel.h b/src/lib/api/public/ClientChannel.h index 25cbd019f..fb5fd31c7 100644 --- a/src/lib/api/public/ClientChannel.h +++ b/src/lib/api/public/ClientChannel.h @@ -93,7 +93,7 @@ public: } else { bool first_update = true; - auto cb = [observer, first_update, method, qualifier](EventType event_type, const StreamUpdate* changes, int change_count) mutable { + auto cb = [observer, first_update, qualifier](EventType event_type, const StreamUpdate* changes, int change_count) mutable { if (event_type == EventType::TABLE_UPDATED) { std::vector> added_items, changed_items; std::vector added_keys, diff --git a/src/lib/utils/ConcurrentPool.h b/src/lib/utils/ConcurrentPool.h index 92d86e7ea..c0f9851bd 100644 --- a/src/lib/utils/ConcurrentPool.h +++ b/src/lib/utils/ConcurrentPool.h @@ -8,7 +8,7 @@ namespace utils { template class ConcurrentPool { - tbb::concurrent_vector> _all; + tbb::concurrent_vector _all; tbb::concurrent_queue _free; size_t _initial_size; size_t _grow_size; @@ -19,7 +19,10 @@ class ConcurrentPool { reserve(initial_size); } - ~ConcurrentPool() { } + ~ConcurrentPool() { + for (auto b : _all) + delete [] b; + } T* allocate() { T* result = nullptr; @@ -34,13 +37,13 @@ class ConcurrentPool { } void reserve(size_t count) { - std::unique_ptr buffer {new T[count]}; + auto buffer = new T[count]; _outstanding += count; for (size_t x = 0; x < count; ++x) { T* temp = &buffer[x]; release(&temp); } - _all.push_back(std::move(buffer)); + _all.push_back(buffer); } void release(T** item) { @@ -57,8 +60,7 @@ class ConcurrentPool { for (auto iter = _all.begin(); iter != _all.end(); ++iter) { size_t size = (iter == _all.begin()) ? _initial_size : _grow_size; for (size_t x = 0; x < size; ++x) { - const std::unique_ptr& bucket = *iter; - const T* start = bucket.get(); + const T* start = *iter; cb(start[x]); } } From e2f3bfd61a74454deb60f5b814e691a9bc235634 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Wed, 1 May 2019 13:31:29 -0400 Subject: [PATCH 130/131] Fixe user manual generation script. --- doc/user/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/user/build.sh b/doc/user/build.sh index 5bf4d5ae3..80e6dd052 100644 --- a/doc/user/build.sh +++ b/doc/user/build.sh @@ -7,7 +7,6 @@ zola_path=$(readlink -f "../../externals/linux/bin/zola/zola") mkdir -p $temp_path cp -r ./* $temp_path -cp -r ../../src/sdk/doc/* $temp_path/content/bts_spark_sdk/Server/ cd $temp_path $zola_path build -u '/docs/' -o $1 >$tmpfile 2>&1 From bddb152d76c9be4b9f9450f6749e10b410107e21 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Wed, 1 May 2019 09:17:05 -0400 Subject: [PATCH 131/131] Working towards a composite order. --- doc/user/CMakeDeps.cmake | 2 +- src/lib/CMakeDeps.cmake | 2 +- src/lib/api/services/OrderService.proto | 5 + .../orderman/order_factory/OrderFactory.cpp | 4 + src/lib/orderman/orders/CompositeOrder.cpp | 149 ++++++++++++++++++ src/lib/orderman/orders/CompositeOrder.h | 31 ++++ src/web/CMakeDeps.cmake | 2 +- 7 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 src/lib/orderman/orders/CompositeOrder.cpp create mode 100644 src/lib/orderman/orders/CompositeOrder.h diff --git a/doc/user/CMakeDeps.cmake b/doc/user/CMakeDeps.cmake index 5f3811807..1b1ac7fc0 100644 --- a/doc/user/CMakeDeps.cmake +++ b/doc/user/CMakeDeps.cmake @@ -1 +1 @@ -set(sources /home/ppinto/Dropbox/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Help/index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Launcher/index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Logging-In-to-the-System/index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/index.md;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/My-Windows/index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Settings/index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Spreader/index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/index.md;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/bts_spark_sdk/Client/Overview.md;content/bts_spark_sdk/Client/Threading.md;content/bts_spark_sdk/Client/_index.md;content/bts_spark_sdk/Server/_index.md;content/bts_spark_sdk/_index.md;content/release_notes/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html) \ No newline at end of file +set(sources /home/ppinto/Dropbox/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Algos/_index.md;content/bts_spark/Glossary/index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Help/index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Launcher/index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Logging-In-to-the-System/index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/index.md;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/My-Windows/index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Settings/index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Spreader/index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/index.md;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html) \ No newline at end of file diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index 03ccc1f50..914b61f82 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServices.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/core/IOrderServices.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orderman_utils/orderman_utils.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/algo/AlgoWrapperOrder.cpp;orderman/orders/algo/IAlgoOrder.cpp;orderman/orders/algo/IAlgoServices.cpp;orderman/orders/algo/IBeforeLaunchAlgoServices.cpp;orderman/orders/algo/algo_orders/CoverAlgo.cpp;orderman/orders/algo/algo_orders/IcebergAlgo.cpp;orderman/orders/algo/algo_orders/JoinAlgo.cpp;orderman/orders/algo/algo_orders/LeanAlgo.cpp;orderman/orders/algo/algo_orders/TopAlgo.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/BookOrderBase.cpp;orderman/orders/book_algos/OCOOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/PricerOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegFillState.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/HedgerManager.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServices.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/core/IOrderServices.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orderman_utils/orderman_utils.cpp;orderman/orders/CompositeOrder.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/algo/AlgoWrapperOrder.cpp;orderman/orders/algo/IAlgoOrder.cpp;orderman/orders/algo/IAlgoServices.cpp;orderman/orders/algo/IBeforeLaunchAlgoServices.cpp;orderman/orders/algo/algo_orders/CoverAlgo.cpp;orderman/orders/algo/algo_orders/IcebergAlgo.cpp;orderman/orders/algo/algo_orders/JoinAlgo.cpp;orderman/orders/algo/algo_orders/LeanAlgo.cpp;orderman/orders/algo/algo_orders/TopAlgo.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/BookOrderBase.cpp;orderman/orders/book_algos/OCOOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/PricerOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegFillState.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/HedgerManager.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file diff --git a/src/lib/api/services/OrderService.proto b/src/lib/api/services/OrderService.proto index d3381209c..c4986cda2 100644 --- a/src/lib/api/services/OrderService.proto +++ b/src/lib/api/services/OrderService.proto @@ -96,6 +96,7 @@ message CustomOrderDefinition OffsetOrderDefinition offset = 24; OCOOrderDefinition oco = 25; PricerOrderDefinition pricer = 26; + CompositeOrderDefinition composite = 27; DynamicOrderDefinition dynamic = 1000; } @@ -182,6 +183,10 @@ message PricerOrderDefinition { double price_factor = 2; CustomOrderDefinition child = 3; } +message CompositeOrderDefinition { + string name = 1; + CustomOrderDefinition definition = 2; +} message DynamicOrderDefinition { string dynamic_order_type = 1; map order_params = 2; diff --git a/src/lib/orderman/order_factory/OrderFactory.cpp b/src/lib/orderman/order_factory/OrderFactory.cpp index 821f15a11..1a11a8549 100644 --- a/src/lib/orderman/order_factory/OrderFactory.cpp +++ b/src/lib/orderman/order_factory/OrderFactory.cpp @@ -23,6 +23,7 @@ #include "lib/orderman/orders/book_algos/OffsetOrder.h" #include "lib/orderman/orders/book_algos/PricerOrder.h" #include "lib/orderman/orders/spreader_orders/SpreaderOrder.h" +#include "lib/orderman/orders/CompositeOrder.h" #include #include @@ -199,6 +200,9 @@ Order &OrderFactory::allocate(OrderKey k, OrderKey pk, } break; } + case CustomOrderDefinition::kComposite: + order_ptr.reset(new orderman::CompositeOrder()); + break; case CustomOrderDefinition::CUSTOM_NOT_SET: bts_assert(false); break; diff --git a/src/lib/orderman/orders/CompositeOrder.cpp b/src/lib/orderman/orders/CompositeOrder.cpp new file mode 100644 index 000000000..34ba96ede --- /dev/null +++ b/src/lib/orderman/orders/CompositeOrder.cpp @@ -0,0 +1,149 @@ +#include "lib/orderman/orders/CompositeOrder.h" +#include "lib/api/services/OrderService.pb.h" +#include "lib/orderman/actors/order_manager/ChildOrder.h" +#include "lib/orderman/actors/order_manager/OrderBuilder.h" +#include "lib/orderman/orders/PreActivationOrderServices.h" + +#include "lib/utils/bts_assert.h" + +namespace bts { +namespace orderman { + +using namespace api::orderman; + +struct CompositeOrder::PImpl { + ChildOrder child_order; + OrderBuilder order_builder; + OrderServices *services; + api::orderman::OrderState state; +}; + +CompositeOrder::CompositeOrder() : me(new PImpl()) {} +CompositeOrder::~CompositeOrder() {} + +void CompositeOrder::on_init_new(api::orderman::OrderDefinition const &def, + const PreActivationOrderServices &serv) { + bts_assert(def.custom_def().has_composite()); + api::orderman::OrderDefinition child_def = def; + *child_def.mutable_custom_def() = def.custom_def(); + me->order_builder = OrderBuilder{key(), child_def, serv.order_factory, 1}; + me->state.set_status(ORDER_STATUS_INACTIVE); +} + +void CompositeOrder::on_init_recovered(OrderRecord const &rec, + const PreActivationOrderServices &) { + + me->state = rec.state(); + // Infer internal status from persisted external status + switch (me->state.status()) { + + case ORDER_STATUS_WORKING: + case ORDER_STATUS_PENDING_WORKING: + case ORDER_STATUS_DISCONNECTED: + me->state.set_status(ORDER_STATUS_ERROR); + me->state.set_last_msg("System restarted with active composite order"); + return; + + // Terminal states + case ORDER_STATUS_ERROR: + case ORDER_STATUS_REJECTED_BY_SPARK: + case ORDER_STATUS_COMPLETED: + case ORDER_STATUS_CANCELED: + break; + + // It should not be possible to reach these + case ORDER_STATUS_INACTIVE: + case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: + case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: + bts_assert(false); + break; + } +} + +OrderOpResult CompositeOrder::on_activate(Price p, Quantity q, + OrderServices &services) { + me->services = &services; + switch (me->state.status()) { + case ORDER_STATUS_INACTIVE: + me->order_builder.activate(services); + me->child_order = me->order_builder.build(); + me->child_order.activate(p, q); + services.subscribe_to_child_updates([this](auto const &u) { + this->on_child_order_update(u.key, u.quick_rebuild_state()); + }); + return {}; + + // Terminal states + case ORDER_STATUS_ERROR: + case ORDER_STATUS_REJECTED_BY_SPARK: + case ORDER_STATUS_COMPLETED: + case ORDER_STATUS_CANCELED: + return {}; + + // It should not be possible to reach these + case ORDER_STATUS_DISCONNECTED: + case ORDER_STATUS_WORKING: + case ORDER_STATUS_PENDING_WORKING: + case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: + case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: + bts_assert(false); + return {}; + } + + bts_assert(false); + return {}; +} + +OrderOpResult CompositeOrder::on_update(Price p, Quantity q) { + if (is_working()) + me->child_order.update(p, q); + return {}; +} + +OrderOpResult CompositeOrder::on_update(api::orderman::Update const &u) { + if (is_working()) { + Update child_update; + child_update.set_price(u.price()); + child_update.set_quantity(u.quantity()); + child_update.set_order_key(me->child_order.key().value()); + + if (u.has_custom()) { + bts_assert(u.custom().has_composite()); + auto const &cupdate = u.custom().composite().definition(); + *child_update.mutable_custom() + ->mutable_composite() + ->mutable_definition() = cupdate; + } + me->child_order.update(child_update); + } + return {}; +} + +OrderOpResult CompositeOrder::on_detach() { + if (is_working()) + me->child_order.detach(); + return {}; +} + +OrderOpResult CompositeOrder::on_cancel() { + if (is_working()) + me->child_order.cancel(); + return {}; +} + +void CompositeOrder::on_child_order_update( + OrderKey, api::orderman::OrderState const &st) { + me->services->report_state(st); +} + +bool CompositeOrder::is_working() const { + return + me->state.status() == ORDER_STATUS_WORKING || + me->state.status() == ORDER_STATUS_PENDING_WORKING || + me->state.status() == ORDER_STATUS_DISCONNECTED ; +} + +void CompositeOrder::on_attached(api::orderman::OrderDefinition const &) {} + +} // namespace orderman +} // namespace bts diff --git a/src/lib/orderman/orders/CompositeOrder.h b/src/lib/orderman/orders/CompositeOrder.h new file mode 100644 index 000000000..6470f10ce --- /dev/null +++ b/src/lib/orderman/orders/CompositeOrder.h @@ -0,0 +1,31 @@ +#pragma once + +#include "lib/orderman/orders/Order.h" + +namespace bts { +namespace orderman { + +class CompositeOrder : public Order { + public: + CompositeOrder(); + ~CompositeOrder(); + + void on_init_new(api::orderman::OrderDefinition const &, + const PreActivationOrderServices &) override; + void on_init_recovered(api::orderman::OrderRecord const &, + const PreActivationOrderServices &) override; + OrderOpResult on_activate(Price, Quantity, OrderServices &) override; + OrderOpResult on_update(Price, Quantity) override; + OrderOpResult on_update(api::orderman::Update const &) override; + OrderOpResult on_detach() override; + OrderOpResult on_cancel() override; + void on_attached(api::orderman::OrderDefinition const &new_parent) override; + + private: + struct PImpl; + std::unique_ptr me; + void on_child_order_update(OrderKey, api::orderman::OrderState const &); + bool is_working() const; +}; +} // namespace orderman +} // namespace bts diff --git a/src/web/CMakeDeps.cmake b/src/web/CMakeDeps.cmake index a717036b4..40156daa9 100644 --- a/src/web/CMakeDeps.cmake +++ b/src/web/CMakeDeps.cmake @@ -1 +1 @@ -set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/AlgoDefinitionService.proto;../lib/api/services/AlgoService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/node_modules/exit/test/fixtures/create-files.sh;api/node_modules/protobufjs/cli/node_modules/acorn-jsx/node_modules/acorn/bin/update_authors.sh;api/node_modules/watch/release.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs) \ No newline at end of file +set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/AlgoDefinitionService.proto;../lib/api/services/AlgoService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MBOService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/node_modules/exit/test/fixtures/create-files.sh;api/node_modules/protobufjs/cli/node_modules/acorn-jsx/node_modules/acorn/bin/update_authors.sh;api/node_modules/watch/release.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs) \ No newline at end of file