From 4af4af86778839a9dbbb70c67b86d6f21aab049d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 20 Dec 2018 15:49:41 -0500 Subject: [PATCH 01/54] SPARK-1731, SPARK-1947 - Don't load security db in md producer --- .../sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 425a3e8d3..26d24e5fb 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -3,7 +3,6 @@ #include "lib/md/cme/MDP3Utils.h" #include "lib/config/config.h" #include "lib/md/coconut2.h" -#include "lib/md/SecurityDB.h" #include "lib/os/os.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/proto_helper.h" @@ -343,16 +342,6 @@ unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, con listener->add_filter_symbol(symbol.symbol()); } - if (options.load_security_db()) { - // Load the list of known securities and add them to the index for fast startup - db::load_securities(channel.symbols, [&](const api::Security& security) { - with_security_like(security, [&](FixedSecurity& fs) { - convert_security(security, fs, true); - listener->add_security(fs); - }); - }); - } - handler.registerListener(*listener); handler.registerMarketDataListener(*listener); handler.registerSecurityListener(*listener); From 8a7174378a645bfa9dde8cf6915c918f654a57bd Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 1 Jan 2019 13:32:40 -0500 Subject: [PATCH 02/54] SPARK-1948 Implement SecurityCache --- src/lib/md/SecurityCache.cpp | 104 +++++++++++++++++++++++++++++++++++ src/lib/md/SecurityCache.h | 34 ++++++++++++ src/lib/md/SecurityDB.h | 6 +- src/lib/md/SecurityPtr.h | 12 ++++ 4 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 src/lib/md/SecurityCache.cpp create mode 100644 src/lib/md/SecurityCache.h create mode 100644 src/lib/md/SecurityPtr.h diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp new file mode 100644 index 000000000..4e5475515 --- /dev/null +++ b/src/lib/md/SecurityCache.cpp @@ -0,0 +1,104 @@ +#include "SecurityCache.h" +#include "SecurityDB.h" +#include "lib/utils/Dispatcher.h" + +#include +#include +#include + +/* +todo: + 0 - All new code uses SecurityKey + 1 - Implement SecurityCache. It is a simple structure + 2 - MDConsumer has a SecurityCache + 3 - scan_securities inserts securities into SecurityCache + + +*/ + +namespace bts { +namespace md { + +struct SecurityCacheImpl { + std::mutex cache_mutex; + std::unordered_map cache; + tbb::concurrent_queue update_queue; + void update_db(const SecurityPtr&); +}; + +SecurityPtr load_from_security_db(SecurityKey key) { + return db::find_security(key); +} + +void SecurityCacheImpl::update_db(const SecurityPtr& security) { + // Batch updates to the database for performance + // algo: + // push item into queue + // post work to dispatcher with delay + // When dispatcher services request drain update_queue. + // This will cause one update per security but most will be no-op + // This is a little simpler than running a timer and trying to stop it after security discovery is done + update_queue.push(security); + utils::Dispatcher::background().post([this]() { + if (!update_queue.empty()) { + std::vector work; + SecurityPtr temp; + while(update_queue.try_pop(temp)) { work.push_back(temp); } + if (!work.empty()) { + db::insert_or_replace(work); + } + } + }, 1000); +} + +////////////////// + +SecurityCache::SecurityCache() : _me(new SecurityCacheImpl()) {} +SecurityCache::~SecurityCache() {} + +SecurityPtr SecurityCache::try_get_security(SecurityKey key) const { + std::lock_guard lock(_me->cache_mutex); + auto iter = _me->cache.find(key); + if (iter != _me->cache.end()) { + return iter->second; + } + return nullptr; +} + +SecurityPtr SecurityCache::get_security(SecurityKey key) { + std::unique_lock lock(_me->cache_mutex); + auto iter = _me->cache.find(key); + if (iter != _me->cache.end()) { + return iter->second; + } + else { + // unlock for db read + lock.unlock(); + auto security = load_from_security_db(key); + if (security) { + return add_db_security(security); + } + return security; + } +} + +SecurityPtr SecurityCache::add_db_security(const SecurityPtr& security) { + bts_assert(security); + std::lock_guard lock(_me->cache_mutex); + SecurityKey key(security->id()); + auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security)); + // If result was inserted result.first->second is equal to security + // If the value was already present it must be a recently inserted 'live' security and we should use it + return result.first->second; +} +bool SecurityCache::add_live_security(const SecurityPtr& security) { + bts_assert(security); + std::lock_guard lock(_me->cache_mutex); + SecurityKey key(security->id()); + // Insert but don't overwrite existing + auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security)); + _me->update_db(security); + return result.second; +} + +}}//ns \ No newline at end of file diff --git a/src/lib/md/SecurityCache.h b/src/lib/md/SecurityCache.h new file mode 100644 index 000000000..388d66fd8 --- /dev/null +++ b/src/lib/md/SecurityCache.h @@ -0,0 +1,34 @@ +#pragma once + +#include "lib/domain_model/Security.pb.h" +#include "MDConsumerHandle.h" +#include "lib/domain_model/SecurityKey.h" +#include "SecurityPtr.h" +#include + +namespace bts { +namespace md { + +// Get security on demand +// Store security+handle +struct SecurityCacheImpl; +class SecurityCache { + std::unique_ptr _me; + +public: + SecurityCache(); + ~SecurityCache(); + + //Load security from cache with fallback to database + SecurityPtr get_security(SecurityKey key); + + // Get the security if it's in the cache otherwise return nullptr + SecurityPtr try_get_security(SecurityKey key) const; + + // returns true if the cache was updated + bool add_live_security(const SecurityPtr& security); + SecurityPtr add_db_security(const SecurityPtr& security); + +}; + +}}//ns \ No newline at end of file diff --git a/src/lib/md/SecurityDB.h b/src/lib/md/SecurityDB.h index 3e7b9e2ec..77948dcce 100644 --- a/src/lib/md/SecurityDB.h +++ b/src/lib/md/SecurityDB.h @@ -7,6 +7,8 @@ #include "lib/api/services/SyntheticSecurityService.pb.h" #include "common.h" #include "UserSymbol.h" +#include "lib/domain_model/SecurityKey.h" +#include "SecurityPtr.h" #include #include @@ -25,7 +27,9 @@ size_t load_securities( const std::set& symbols, const std::function& callback); -void store_if_new(const std::vector& handles); +SecurityPtr find_security(SecurityKey); + +void insert_or_replace(const std::vector&); // metadata std::vector load_metadata(); diff --git a/src/lib/md/SecurityPtr.h b/src/lib/md/SecurityPtr.h new file mode 100644 index 000000000..34c93965d --- /dev/null +++ b/src/lib/md/SecurityPtr.h @@ -0,0 +1,12 @@ +#pragma once + +#include "lib/domain_model/SecurityKey.h" +#include + +namespace bts { +namespace api {class Security;} +namespace md { + + typedef std::shared_ptr const SecurityPtr; + +}} \ No newline at end of file From 0af2b03ad7a9c91a3e4315a21dbc6d0f1fc427ca Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 1 Jan 2019 13:41:40 -0500 Subject: [PATCH 03/54] SPARK-1953 Remove SecurityDB from SecurityIDRecorder --- src/lib/md/SecurityIDRecorder.cpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index f8cc32613..afeea6f3b 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -5,7 +5,6 @@ #include "lib/config/config.h" #include "lib/utils/Sqlite_utils.h" #include "lib/logging/logging.h" -#include "lib/md/SecurityDB.h" #include #include @@ -279,19 +278,6 @@ void SecurityIDRecorderState::work_db() { sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } - // record the securities - if (!new_entries.empty() || !existing_securities.empty()) { - std::vector save_these; - save_these.reserve(new_entries.size() + existing_securities.size()); - for (const auto& e : new_entries) { - save_these.push_back(&e->security); - } - for (const auto& e : existing_securities) { - save_these.push_back(&e->security); - } - db::store_if_new(save_these); - } - if (!new_entries.empty()) { for (size_t x = 0; x < new_entries.size(); ++x) { Security& security = new_entries[x]->security; From 6953f8b351ac4327ea86be25be96c9f6916d0af3 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 1 Jan 2019 13:43:06 -0500 Subject: [PATCH 04/54] Remove obsolete comment --- src/lib/md/SecurityCache.cpp | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp index 4e5475515..2675128bb 100644 --- a/src/lib/md/SecurityCache.cpp +++ b/src/lib/md/SecurityCache.cpp @@ -6,16 +6,6 @@ #include #include -/* -todo: - 0 - All new code uses SecurityKey - 1 - Implement SecurityCache. It is a simple structure - 2 - MDConsumer has a SecurityCache - 3 - scan_securities inserts securities into SecurityCache - - -*/ - namespace bts { namespace md { @@ -26,10 +16,6 @@ struct SecurityCacheImpl { void update_db(const SecurityPtr&); }; -SecurityPtr load_from_security_db(SecurityKey key) { - return db::find_security(key); -} - void SecurityCacheImpl::update_db(const SecurityPtr& security) { // Batch updates to the database for performance // algo: @@ -74,7 +60,7 @@ SecurityPtr SecurityCache::get_security(SecurityKey key) { else { // unlock for db read lock.unlock(); - auto security = load_from_security_db(key); + auto security = db::find_security(key); if (security) { return add_db_security(security); } From ca21ac5f5e5150142a796993254e091e15e504e6 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 2 Jan 2019 15:29:31 -0500 Subject: [PATCH 05/54] SPARK-1949 - Convert SecuritydB to sqlitecpp --- src/lib/md/SecurityDB.cpp | 129 +++++++++++++++++--------------------- src/lib/md/SecurityPtr.h | 2 +- 2 files changed, 60 insertions(+), 71 deletions(-) diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index 03ac7d67e..d9a94c3e1 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -1,7 +1,9 @@ #include "SecurityDB.h" #ifndef WIN32 -#include "lib/utils/Sqlite_utils.h" +#include +#include + #include "lib/config/config.h" #include "lib/config/protos/system.pb.h" #include "lib/md/Shm/FixedSecurity.h" @@ -15,6 +17,7 @@ using std::string; using namespace bts::utils; +using namespace SQLite; namespace bts { namespace md { @@ -25,7 +28,7 @@ namespace db { * key, created_date, expiration_date */ -sqlite3_ptr open_db() { +std::unique_ptr open_db() { // Directory location on disk auto db_dir = config::get().data_path() + "databases/md/"; auto db_path = db_dir + "/securities.sqlite"; @@ -34,20 +37,14 @@ sqlite3_ptr open_db() { } // Create table if it does not exist - auto dbptr = sqlite::open_db(db_path.c_str()); - bts_assert(dbptr); - auto db = dbptr.get(); - sqlite3_busy_timeout(db, 9999); - sqlite::exec_nofail(db, "PRAGMA journal_mode=WAL"); - sqlite::exec_nofail(db, "PRAGMA syncronous=0"); + auto db = std::make_unique(db_path, OPEN_READWRITE | OPEN_CREATE); + db->setBusyTimeout(9999); + db->exec("PRAGMA journal_mode=WAL"); + db->exec("PRAGMA syncronous=0"); + db->exec("CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);"); + db->exec("CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); - sqlite::exec_nofail(db, - "CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);"); - - sqlite::exec_nofail(db, - "CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); - - return dbptr; + return db; } datetime::Date sec_expiration_cut_off() @@ -68,24 +65,22 @@ size_t load_securities( // Load all existing securities and call the callback auto db = dbptr.get(); - auto select_stmt = sqlite::prepare_query(db, "select key,symbol,venue,type,security_blob from securities where expiration > ?"); - auto select = select_stmt.get(); - sqlite3_bind_int(select, 1, sec_expiration_cut_off().to_short()); - auto result = sqlite3_step(select); + SQLite::Statement select_stmt(*db, "select key,symbol,venue,type,security_blob from securities where expiration > :exp"); + select_stmt.bind(":exp", sec_expiration_cut_off().to_short()); + Security security; - while (result == SQLITE_ROW) { - security_key security_key = sqlite3_column_int(select, 0); - auto symbol = std::string( - (char*)sqlite3_column_text(select, 1), - sqlite3_column_bytes(select, 1)); - auto venue = (api::Venue)sqlite3_column_int(select, 2); - auto type = (api::SecurityType)sqlite3_column_int(select, 3); + while (select_stmt.tryExecuteStep() == SQLITE_ROW) { + security_key security_key = select_stmt.getColumn("key"); + std::string symbol = select_stmt.getColumn("symbol"); + api::Venue venue = (api::Venue)(int) select_stmt.getColumn("venue"); + auto type = (api::SecurityType)(int) select_stmt.getColumn("type"); UserSymbol us(symbol, venue, type); if (symbols.find(us) != symbols.end()) { - auto data = sqlite3_column_blob(select, 4); - auto len = sqlite3_column_bytes(select, 4); + auto blob = select_stmt.getColumn("security_blob"); + // auto data = blob.getBlob(); + // auto len = blob.getBytes(); // Migrate from old tick stuff - if (security.ParseFromArray(data, len)) { + if (security.ParseFromArray(blob, blob.getBytes())) { if (security.has_glbx()) { if (security.contract_spec().main_fraction() == 0) { security.mutable_contract_spec()->set_main_fraction(security.glbx().do_not_use_main_fraction()); @@ -105,64 +100,64 @@ size_t load_securities( LOG(md, Warn, "Failed to parse security from database. id = {}", security_key); } } - result = sqlite3_step(select); } return count; } -void store_if_new(const std::vector& securities) { - auto dbptr = open_db(); - auto insert = sqlite::prepare_query(dbptr, "insert or ignore into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); - sqlite::exec_nofail(dbptr, "BEGIN TRANSACTION"); +SecurityPtr find_security(SecurityKey key) { + //todo: + (void)key; + return nullptr; +} + +void insert_or_replace(const std::vector& securities) { + auto db = open_db(); + SQLite::Statement insert(*db,"insert or replace into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); + SQLite::Transaction transaction(*db); std::string buffer; - for (const api::Security* psecurity : securities) { + for (const auto& psecurity : securities) { if (psecurity && psecurity->SerializeToString(&buffer)) { - SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, psecurity->id()), dbptr); - SQLITE_CHK(sqlite3_bind_text(insert.get(), 2, psecurity->symbol().c_str(), psecurity->symbol().size(), SQLITE_STATIC), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 3, psecurity->exchange()), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 4, securitytype_of(*psecurity)), dbptr); - SQLITE_CHK(sqlite3_bind_blob(insert.get(), 5, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 6, expiration(*psecurity).to_short()), dbptr); - sqlite3_step(insert.get()); - sqlite3_reset(insert.get()); + insert.bind(1, psecurity->id()); + insert.bind(2, psecurity->symbol()); + insert.bind(3, psecurity->exchange()); + insert.bind(4, securitytype_of(*psecurity)); + insert.bind(5, buffer); + insert.bind(6, expiration(*psecurity).to_short()); + insert.exec(); + insert.reset(); } } - sqlite::exec_nofail(dbptr, "COMMIT TRANSACTION"); + transaction.commit(); } std::vector load_metadata() { std::vector values; - auto dbptr = open_db(); - auto db = dbptr.get(); - auto select_stmt = sqlite::prepare_query(db, "select key,data from metadata"); - auto select = select_stmt.get(); - auto result = sqlite3_step(select); - while(result == SQLITE_ROW) { - uint32_t key = sqlite3_column_int(select, 0); - auto blob = sqlite3_column_blob(select, 1); - auto len = sqlite3_column_bytes(select, 1); - api::synthetic::Metadata data; - if (data.ParseFromArray(blob, len)) { - bts_assert(key == data.security_key()); - values.push_back(data); + auto db = open_db(); + SQLite::Statement select(*db, "select key,data from metadata"); + while(select.tryExecuteStep() == SQLITE_ROW) { + uint32_t key = select.getColumn(0); + auto blob = select.getColumn(1); + api::synthetic::Metadata meta; + if (meta.ParseFromArray(blob, blob.getBytes())) { + bts_assert(key == meta.security_key()); + values.push_back(meta); } else { LOG(md, Warn, "Failed to parse security metadata for security_key={}", key); } - result = sqlite3_step(select); } return values; } bool store_metadata(const api::synthetic::Metadata& data) { - auto dbptr = open_db(); - auto insert = sqlite::prepare_query(dbptr, "insert or replace into metadata (key, data) values (?,?)"); + auto db = open_db(); + SQLite::Statement insert(*db, "insert or replace into metadata (key, data) values (?,?)"); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, data.security_key()), dbptr); + insert.bind(1, data.security_key()); std::string buffer = data.SerializeAsString(); - SQLITE_CHK(sqlite3_bind_blob(insert.get(), 2, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr); - auto result = sqlite3_step(insert.get()); + insert.bind(2, buffer); + auto result = insert.tryExecuteStep(); if (result != SQLITE_DONE) { LOG(md, Warn, "Failed to insert metadata. sqlite result {}", result); return false; @@ -180,13 +175,7 @@ size_t load_securities( const std::function& callback) { return 0; } -void store_if_new(const std::vector& handles) {} -// std::vector load_metadata() { -// std::vector result; -// return result; -// } -// void store_metadata(const api::synthetic::Metadata&) { -// } +void insert_or_replace(const std::vector& handles) {} #endif diff --git a/src/lib/md/SecurityPtr.h b/src/lib/md/SecurityPtr.h index 34c93965d..228c9a198 100644 --- a/src/lib/md/SecurityPtr.h +++ b/src/lib/md/SecurityPtr.h @@ -7,6 +7,6 @@ namespace bts { namespace api {class Security;} namespace md { - typedef std::shared_ptr const SecurityPtr; + typedef std::shared_ptr SecurityPtr; }} \ No newline at end of file From 1126b772dd8eb046ec1564cfcaf0ae69e2486607 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 2 Jan 2019 17:26:39 -0500 Subject: [PATCH 06/54] SPARK-1952 Update MDConsumer to use SecurityPtr --- src/lib/md/MDConsumer.cpp | 42 +++++++++++++++++++++------------------ src/lib/md/MDConsumer.h | 3 +++ src/lib/md/SecurityDB.cpp | 1 + 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index dba9eda76..9ab52a906 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -39,15 +39,15 @@ bool match_symbol(const string& val, const string& b) { return b.empty() || (val == b); } -typedef shared_ptr PSecurity; +typedef shared_ptr MutSecurityPtr; struct SecurityLess { - bool operator()(const PSecurity& left, const PSecurity& right) { + bool operator()(const SecurityPtr& left, const SecurityPtr& right) { return left->id() < right->id(); } }; -PSecurity convert_psecurity(const FixedSecurity& security) { - PSecurity psec = make_shared(); +MutSecurityPtr convert_psecurity(const FixedSecurity& security) { + auto psec = make_shared(); convert_security(security, psec.get()); return psec; } @@ -562,10 +562,10 @@ public: // This is a terrible name size_t walk_sec_list(SecurityCallback cb, const string& symbol, SecurityType type) { size_t result = 0; - vector securities; + vector securities; for (auto& inst : _instances) { if (match_symbol(inst->symbol(), symbol) && match_type(inst->type(), type)) { - result += inst->walk_sec_list([&](PSecurity security) { securities.push_back(security); }, symbol, type); + result += inst->walk_sec_list([&](MutSecurityPtr security) { securities.push_back(security); }, symbol, type); } } auto complete_securities = fill_leg_details(securities); @@ -602,7 +602,8 @@ public: return empty_handle; } - map get_handles(const vector& securities) { + template //maybe const + map get_handles(const vector& securities) { set keys; for (const auto& sec : securities) { keys.insert(sec->id()); @@ -641,8 +642,8 @@ public: return handle; } - vector fill_leg_details(const vector& securities, int depth=0) { - vector result; + vector fill_leg_details(const vector& securities, int depth=0) { + vector result; if (securities.empty() || depth > 64) return result; // // Gather ids for legs @@ -655,14 +656,14 @@ public: } } // If there are no spreads everything is (by definition) resolved already - if (keys.empty()) return securities; + if (keys.empty()) return vector{securities.begin(), securities.end()}; // // gather handles for direct legs auto handles = get_handles(keys); // // Some of the legs will themselves be spreads - vector legs; + vector legs; legs.reserve(handles.size()); for (const auto& pair : handles) { if (pair.second.is_initialized()) { @@ -670,9 +671,8 @@ public: } } // recurse to find details - legs = fill_leg_details(legs, depth+1); - map leg_map; - for (auto& leg : legs) { + map leg_map; + for (auto& leg : fill_leg_details(legs, depth+1)) { leg_map.insert(make_pair(leg->id(), move(leg))); } @@ -705,7 +705,7 @@ public: if (_callbacks.empty() && _deferred_handles.empty()) return 0; // Call each _instance and pass it a callback for (auto& inst : _instances) { - vector found_securities; + vector found_securities; inst->scan_new_securities([&](const ColdEntry& entry) { // Check if any deferred handles need this security (could be sped up with get_handles() but it isn't hot auto iter = _deferred_handles.find(entry.security.key); @@ -719,10 +719,11 @@ public: if (resolve_legs && !found_securities.empty()) { sort(found_securities.begin(), found_securities.end(), SecurityLess()); - vector resolved_securities = fill_leg_details(found_securities); + vector resolved_securities = fill_leg_details(found_securities); sort(resolved_securities.begin(), resolved_securities.end(), SecurityLess()); + // Extract the list of unresolved securities - vector unresolved_securities; + vector unresolved_securities; set_difference(found_securities.begin(), found_securities.end(), resolved_securities.begin(), resolved_securities.end(), inserter(unresolved_securities, unresolved_securities.begin()), SecurityLess()); @@ -803,11 +804,14 @@ public: // Note: We have to lock over the initial walk because if scan_securities() finds a new security concurrently with this walk // it might mark it seen without this callback getting called. for (auto& inst : _instances) { - vector securities; + vector securities; inst->walk_old_securities([&](const ColdEntry& entry) { securities.push_back(convert_psecurity(entry.security)); }); - auto filled_securities = resolve_legs ? fill_leg_details(securities) : securities; + auto filled_securities = + resolve_legs ? + fill_leg_details(securities) : + vector{securities.begin(), securities.end()}; auto handles = get_handles(filled_securities); for (const auto& security : filled_securities) { auto find_result = handles.find(security->id()); diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 643ef2635..8a7d548c1 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -7,6 +7,7 @@ #include "MDConsumerHandle.h" #include "UserSymbol.h" #include "ISyntheticSecurityService.h" +#include "SecurityPtr.h" #include #include @@ -51,6 +52,8 @@ public: size_t scan_securities(bool resolve_legs = true); void on_security(SecurityCallback cb, bool resolve_legs = true); + // SecurityPtr get_security(SecurityKey key); + MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const; MDConsumerChannel get_channel(const UserSymbol&) const; diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index d9a94c3e1..c374765a9 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -176,6 +176,7 @@ size_t load_securities( return 0; } void insert_or_replace(const std::vector& handles) {} +SecurityPtr find_security(SecurityKey) { return nullptr; } #endif From 5481a3e9d64966b19439d629e82ef20c8dda991d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 2 Jan 2019 17:39:41 -0500 Subject: [PATCH 07/54] Update CMakeDeps --- src/lib/CMakeDeps.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index ec7dd860c..dced7a640 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/CoverOrder.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;orderman2/orders/NativeOrder.cpp;orderman2/orders/Order.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file From c4bbf5e219926b3e6880ed7704b49305a734eeee Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 3 Jan 2019 13:50:26 -0500 Subject: [PATCH 08/54] SPARK-1952 - convert md::SecurityCallback to SecurityPtr to avoid copying --- src/lib/md/Diag/MDPeriodicDumper.cpp | 6 +-- src/lib/md/MDConsumer.cpp | 12 ++--- src/lib/md/MDConsumer.h | 2 +- src/lib/md/MDConsumerInterop.cpp | 12 ++--- src/lib/md/MDRecoverableConsumer.cpp | 8 +-- src/lib/md/relay/QueryServer.cpp | 2 +- src/lib/md/test/md2_tests.cpp | 70 ++++++++++++------------- src/lib/orderman/actors/SecuritySrc.cpp | 10 ++-- 8 files changed, 60 insertions(+), 62 deletions(-) diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index 9fbfdc56c..1704b9719 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -23,7 +23,7 @@ namespace md { struct Entry { uint32_t seq; MDConsumerHandle handle; - std::unique_ptr security; + SecurityPtr security; }; std::string print_book( @@ -156,8 +156,8 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, // read all securities into a vector std::vector entries; - consumer.walk_security_list([&](const api::Security& sec, MDConsumerHandle handle) { - entries.push_back({ 0, handle, std::make_unique(sec) }); + consumer.walk_security_list([&](const SecurityPtr& sec, MDConsumerHandle handle) { + entries.push_back({ 0, handle, sec }); }); entries = filter_entries(entries); diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 9ab52a906..fcc6b4762 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -573,7 +573,7 @@ public: for (const auto& security : complete_securities) { auto find_result = handles.find(security->id()); if (find_result != handles.end()) { - cb(*security, find_result->second); + cb(security, find_result->second); } } return result; @@ -749,7 +749,7 @@ public: if (find_result != handles.end()) { // Pass new securities to each callback for (auto& cb : _callbacks) { - cb(*security, find_result->second); + cb(security, find_result->second); } } else { @@ -765,7 +765,7 @@ public: auto find_result = handles.find(security->id()); if (find_result != handles.end()) { for (auto& cb : _callbacks) { - cb(*security, find_result->second); + cb(security, find_result->second); } } else { @@ -789,7 +789,7 @@ public: } // call all subscribers for (auto& cb : _callbacks) { - cb(security, handle); + cb(std::make_shared(security), handle); } } } @@ -817,7 +817,7 @@ public: auto find_result = handles.find(security->id()); bts_assert_d(find_result != handles.end()); if (find_result != handles.end()) { - cb(*security, find_result->second); + cb(security, find_result->second); } } } @@ -826,7 +826,7 @@ public: for (const auto& item : _sci->get_old_synthetics()) { const auto& security = item->security; auto handle = MDConsumerHandle(item); - cb(security, handle); + cb(std::make_shared(security), handle); } } // store the callback diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 8a7d548c1..85e74f4b9 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -20,7 +20,7 @@ namespace bts { namespace md { /* typedef void (security_callback)(const Security& security); */ -typedef std::function SecurityCallback; +typedef std::function SecurityCallback; using std::size_t; class MDConsumerImpl; diff --git a/src/lib/md/MDConsumerInterop.cpp b/src/lib/md/MDConsumerInterop.cpp index 01b0616bb..7048354c5 100644 --- a/src/lib/md/MDConsumerInterop.cpp +++ b/src/lib/md/MDConsumerInterop.cpp @@ -64,10 +64,10 @@ typedef void(*security_cb)(security_key, const char*, size_t, MDConsumerHandle& CAPI size_t md_consumer_walk_security_list(MDConsumer* consumer, security_cb cb) { bts_assert(consumer); std::string buffer; - return consumer->walk_security_list([&](const Security& security, MDConsumerHandle handle) { + return consumer->walk_security_list([&](const SecurityPtr& security, MDConsumerHandle handle) { buffer.clear(); - security.SerializePartialToString(&buffer); - cb(security.id(), buffer.c_str(), buffer.size(), handle); + security->SerializePartialToString(&buffer); + cb(security->id(), buffer.c_str(), buffer.size(), handle); }); } @@ -94,10 +94,10 @@ void safe_call_clr_callback(security_cb cb, security_key key, const char* buffer CAPI void md_consumer_set_on_security(MDConsumer* consumer, security_cb cb) { bts_assert(consumer); - consumer->on_security([=](const Security& security, MDConsumerHandle handle) { + consumer->on_security([=](const SecurityPtr& security, MDConsumerHandle handle) { std::string buffer; - security.SerializePartialToString(&buffer); - safe_call_clr_callback(cb, security.id(), buffer.c_str(), buffer.size(), handle); + security->SerializePartialToString(&buffer); + safe_call_clr_callback(cb, security->id(), buffer.c_str(), buffer.size(), handle); }); } #endif diff --git a/src/lib/md/MDRecoverableConsumer.cpp b/src/lib/md/MDRecoverableConsumer.cpp index 31a554ee5..9e6250388 100644 --- a/src/lib/md/MDRecoverableConsumer.cpp +++ b/src/lib/md/MDRecoverableConsumer.cpp @@ -54,8 +54,8 @@ size_t MDRecoverableConsumer::walk_security_list(SecurityCallback callback) { void MDRecoverableConsumer::on_security(SecurityCallback cb) { // Add a filter to the callback to avoid calling the on_security callback with duplicates - auto filtered_cb = [this,cb](const Security& security, MDConsumerHandle handle) { - auto key = security.id(); + auto filtered_cb = [this,cb](const SecurityPtr& security, MDConsumerHandle handle) { + auto key = security->id(); if (_seen.find(key) == _seen.end()) { _seen.insert(key); cb(security, handle); @@ -101,11 +101,11 @@ bool MDRecoverableConsumer::try_recover() { _current->consumer.on_security(cb); } - _current->consumer.on_security([&](const Security& security, MDConsumerHandle newhandle) { + _current->consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle newhandle) { Mutex::scoped_lock l (_lock); for (auto& group : _old) { for (auto& handle : group->handles) { - if (handle->key() == security.id()) { + if (handle->key() == security->id()) { // update the handle! (if this is called guarunteed success) *handle = newhandle; bts_assert(handle->is_initialized()); diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index dea02db98..57ab06e89 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -184,7 +184,7 @@ public: void subscribe_securities() { auto id = _consumer.id(); - _consumer.on_security([this,id](const Security&, MDConsumerHandle handle) { + _consumer.on_security([this,id](const SecurityPtr&, MDConsumerHandle handle) { if (_consumer.id() == id) { auto iter = _groups.find(handle.get_usersymbol()); bts_assert(iter != _groups.end()); diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 0ec5f24a2..9d6ef027e 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -372,14 +372,14 @@ public: vector consumer_securities; // Poll for security list - none visible until they have keys - size_t count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); // Since no security has an id none are "public" REQUIRE(consumer_securities.size() == count); REQUIRE(count == 0); // Assign a unique ID to each security (from the producer) assign_ids(producers.begin(), producers.end()); // Now securities can be enumerated by the client - count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); REQUIRE(consumer_securities.size() == count); REQUIRE(count == 24); @@ -442,10 +442,10 @@ public: // Add the on_security callback vector securities; - consumer.on_security([&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() != invalid_security_key); - REQUIRE(security.id() == handle.key()); - securities.push_back(security); + consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() != invalid_security_key); + REQUIRE(security->id() == handle.key()); + securities.push_back(*security); }); // No securities have ids yet REQUIRE(securities.size() == 0); @@ -598,10 +598,10 @@ public: // Add the on_security callback - callback fires for existing securities vector securities; - consumer.on_security([&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() != invalid_security_key); - REQUIRE(security.id() == handle.key()); - securities.push_back(security); + consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() != invalid_security_key); + REQUIRE(security->id() == handle.key()); + securities.push_back(*security); }); // Found all securities REQUIRE(securities.size() == producers.size()); @@ -622,9 +622,9 @@ public: // Symbol list for the consumer MDRecoverableConsumer consumer(get_test_symbols(), nullptr); map> products; - consumer.on_security([&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() == handle.key()); - products.insert(make_pair(security.id(), consumer.get_managed_handle(security.id()))); + consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() == handle.key()); + products.insert(make_pair(security->id(), consumer.get_managed_handle(security->id()))); }); consumer.scan_securities(); REQUIRE(products.size() == producers.size()); @@ -740,7 +740,7 @@ public: MDConsumer consumer(symbols, nullptr); set keys; - consumer.walk_security_list([&](const Security& security, MDConsumerHandle) { keys.insert(security.id()); }); + consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) { keys.insert(security->id()); }); REQUIRE(keys.size() == producers.size()); auto map = consumer.get_handles(keys); REQUIRE(map.size() == keys.size()); @@ -783,16 +783,16 @@ public: csymbols.insert(UserSymbol("S4", VENUE_BTS, SECURITYTYPE_SPREAD)); MDConsumer consumer(csymbols, nullptr); - auto check_sec = [&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() == handle.key()); - auto iter = find_if(std::begin(all_secs_array), std::end(all_secs_array), [&](const Security& sec) { return sec.id() == security.id(); }); + auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() == handle.key()); + auto iter = find_if(std::begin(all_secs_array), std::end(all_secs_array), [&](const Security& sec) { return sec.id() == security->id(); }); REQUIRE(iter != end(all_secs_array)); - REQUIRE(iter->symbol() == (security.symbol())); - if (security.Contract_case() == Security::kSpread) { - REQUIRE(security.spread().normalized() == false); - REQUIRE(security.spread().legs_size() == iter->spread().legs_size()); - for (int x = 0; x < security.spread().legs_size(); ++x) { - REQUIRE((security.spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); + REQUIRE(iter->symbol() == (security->symbol())); + if (security->Contract_case() == Security::kSpread) { + REQUIRE(security->spread().normalized() == false); + REQUIRE(security->spread().legs_size() == iter->spread().legs_size()); + for (int x = 0; x < security->spread().legs_size(); ++x) { + REQUIRE((security->spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); } } }; @@ -829,16 +829,16 @@ public: csymbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); MDConsumer consumer(csymbols, nullptr); - auto check_sec = [&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() == handle.key()); - auto iter = find_if(begin(all_secs_array), end(all_secs_array), [&](const Security& sec) { return sec.id() == security.id(); }); + auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() == handle.key()); + auto iter = find_if(begin(all_secs_array), end(all_secs_array), [&](const Security& sec) { return sec.id() == security->id(); }); REQUIRE(iter != end(all_secs_array)); - REQUIRE((iter->id()) == (security.id())); - if (security.Contract_case() == Security::kSpread) { - REQUIRE(security.spread().normalized() == false); - REQUIRE(security.spread().legs_size() == iter->spread().legs_size()); - for (int x = 0; x < security.spread().legs_size(); ++x) { - REQUIRE((security.spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); + REQUIRE((iter->id()) == (security->id())); + if (security->Contract_case() == Security::kSpread) { + REQUIRE(security->spread().normalized() == false); + REQUIRE(security->spread().legs_size() == iter->spread().legs_size()); + for (int x = 0; x < security->spread().legs_size(); ++x) { + REQUIRE((security->spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); } } }; @@ -886,7 +886,7 @@ public: check_func = [&](const Security&) { REQUIRE(false); }; - consumer.on_security([&](const Security& sec, MDConsumerHandle) { ++security_count; check_func(sec); }); + consumer.on_security([&](const SecurityPtr& sec, MDConsumerHandle) { ++security_count; check_func(*sec); }); consumer.scan_securities(); REQUIRE(security_count == 0); @@ -1590,13 +1590,13 @@ namespace ImpliedTests { MDConsumer consumer(symbols, nullptr); vector consumer_securities; // Poll for security list - none visible until they have keys - size_t count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); // Since no security has an id none are "public" REQUIRE(consumer_securities.size() == count); REQUIRE(count == 0); assign_ids(producers.begin(), producers.end()); // Now securities can be enumerated by the client - count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); REQUIRE(consumer_securities.size() == count); REQUIRE(count == 24); vector consumers; diff --git a/src/lib/orderman/actors/SecuritySrc.cpp b/src/lib/orderman/actors/SecuritySrc.cpp index 8067ef85f..eacf58745 100644 --- a/src/lib/orderman/actors/SecuritySrc.cpp +++ b/src/lib/orderman/actors/SecuritySrc.cpp @@ -15,7 +15,7 @@ struct SecuritySrc::PImpl { mercury::Stream new_securities_stream; std::vector to_publish; md::MDConsumer md_consumer; - std::vector> securities_keep_alive; + std::vector securities_keep_alive; }; SecuritySrc::SecuritySrc(md::MDConsumer &md) : me(new PImpl(md)) {} @@ -24,11 +24,9 @@ SecuritySrc::~SecuritySrc() {} void SecuritySrc::init() { me->new_securities_stream = make_stream(); me->md_consumer.on_security( - [this](api::Security const &s, md::MDConsumerHandle) { - auto sec = std::make_unique(s); - auto sec_ptr = sec.get(); - me->securities_keep_alive.push_back(std::move(sec)); - me->to_publish.push_back(sec_ptr); + [this](md::SecurityPtr const &sec, md::MDConsumerHandle) { + me->securities_keep_alive.push_back(sec); + me->to_publish.push_back(sec.get()); }); subscribe([this](auto) { this->on_scan_securities(); }); From fe1b9f906ee37bd78609e922e4fe0ce7a49b463f Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 4 Jan 2019 16:07:08 -0500 Subject: [PATCH 09/54] SPARK-1961 SyntheticStrategy is not properly syncronized --- src/lib/md/MDConsumer.cpp | 13 ++++++++++--- src/lib/md/MDConsumerHandle.cpp | 7 ++++--- src/lib/md/MDConsumerHandle.h | 6 +++--- src/lib/md/test/synthetic_book_merge_tests.cpp | 2 +- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index fcc6b4762..e2eb29b0b 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -314,6 +314,7 @@ class SyntheticConsumerInst { bool _initialized; int _security_sub; int _meta_sub; + std::vector> _old_strategies; public: SyntheticConsumerInst(std::shared_ptr& ss, const IGetHandle& gh) : @@ -325,7 +326,8 @@ class SyntheticConsumerInst { _get_handle(gh), _initialized(false), _security_sub(0), - _meta_sub(0) + _meta_sub(0), + _old_strategies() { } @@ -418,8 +420,13 @@ private: // update security metadata auto iter = _data.find(m.security_key()); if (iter != _data.end()) { - iter->second->strategy.CopyFrom(m.strategy()); + auto updated = std::make_unique(); + updated->CopyFrom(m.strategy()); + iter->second->strategy.swap(updated); iter->second->updated = TimeStamp::now(); + // Keep the old strategy alive because it could be in use. + // metadata is changed so seldomly this is not a significant use of memory + if (updated != nullptr) _old_strategies.push_back(updated); } } @@ -475,7 +482,7 @@ private: if (legs.size() == (size_t)strategy.security->synthetic().legs_size()) { auto s = make_shared(); s->security.CopyFrom(*strategy.security); - s->strategy.CopyFrom(strategy.strategy); + s->strategy->CopyFrom(strategy.strategy); s->legs = legs; s->updated = TimeStamp::now(); result.insert( diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index 4062f1353..890a599c4 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -16,7 +16,7 @@ MDConsumerHandle::MDConsumerHandle() { _type = HANDLETYPE_VOID; static_assert(sizeof(MDConsumerHandle) <= 32, ""); } -MDConsumerHandle::MDConsumerHandle(const std::shared_ptr& def) { +MDConsumerHandle::MDConsumerHandle(const std::shared_ptr& def) { memset(this, 0, sizeof(MDConsumerHandle)); synthetic.def = def; _type = HANDLETYPE_SYNTHETIC; @@ -233,9 +233,10 @@ TimeStamp copy_synthetic_depth(const SyntheticStrategy& synth, size_t count, Pri std::array bids, asks; TimeStamp t; auto key = synth.legs[x].key(); - auto iter = synth.strategy.leg_params_map().find(key); + auto strategy = synth.strategy.get(); + auto iter = strategy->leg_params_map().find(key); const spark::LegSpreaderParams& leg_params = - iter == synth.strategy.leg_params_map().end() ? + iter == strategy->leg_params_map().end() ? spark::LegSpreaderParams::default_instance() : iter->second; diff --git a/src/lib/md/MDConsumerHandle.h b/src/lib/md/MDConsumerHandle.h index 2000ab7cd..770b8ee59 100644 --- a/src/lib/md/MDConsumerHandle.h +++ b/src/lib/md/MDConsumerHandle.h @@ -14,7 +14,7 @@ namespace md { class MDConsumerHandle; struct SyntheticStrategy { - spark::ExecutionStrategy strategy; + std::unique_ptr strategy; api::Security security; std::vector legs; TimeStamp updated; @@ -147,7 +147,7 @@ class BTS_EXPORT MDConsumerHandle { public: MDConsumerHandle(); - explicit MDConsumerHandle(const std::shared_ptr&); + explicit MDConsumerHandle(const std::shared_ptr&); ~MDConsumerHandle(); MDConsumerHandle(const MDConsumerHandle& other); @@ -335,7 +335,7 @@ private: const ColdHeader* cheader; }; struct SyntheticHandle { - std::shared_ptr def; + std::shared_ptr def; }; HandleType _type; union { diff --git a/src/lib/md/test/synthetic_book_merge_tests.cpp b/src/lib/md/test/synthetic_book_merge_tests.cpp index 4d174207a..822ebb907 100644 --- a/src/lib/md/test/synthetic_book_merge_tests.cpp +++ b/src/lib/md/test/synthetic_book_merge_tests.cpp @@ -135,7 +135,7 @@ tuple,vector> make_securities(c void set_leg_params(SyntheticStrategy& strat, uint32_t security_key, const spark::LegSpreaderParams& params) { - auto* map = strat.strategy.mutable_leg_params_map(); + auto* map = strat.strategy->mutable_leg_params_map(); map->insert({ security_key, params }); } From 7a6739c156d9ea05ff97a35721d647511ca207a7 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 4 Jan 2019 16:35:52 -0500 Subject: [PATCH 10/54] Fix build issue with uniqueptr --- src/lib/md/MDConsumer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index e2eb29b0b..5cc1d2abf 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -426,7 +426,7 @@ private: iter->second->updated = TimeStamp::now(); // Keep the old strategy alive because it could be in use. // metadata is changed so seldomly this is not a significant use of memory - if (updated != nullptr) _old_strategies.push_back(updated); + if (updated != nullptr) _old_strategies.push_back(std::move(updated)); } } From 501cd93fd67a096a17d6e7d8d0cb58857e6ed466 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 7 Jan 2019 15:52:37 -0500 Subject: [PATCH 11/54] SPARK-1950 Integrate SecurityCache into MDConsumer --- .vscode/launch.json | 2 +- src/lib/md/MDConsumer.cpp | 43 ++++++++++++++----- src/lib/md/MDConsumer.h | 5 ++- src/lib/md/MDConsumerHandle.cpp | 20 +++++---- src/lib/md/SecurityCache.cpp | 35 +++++++-------- src/lib/md/SecurityCache.h | 4 +- .../md/test/synthetic_book_merge_tests.cpp | 3 ++ 7 files changed, 71 insertions(+), 41 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index d3f66bbaf..cf754974d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -209,7 +209,7 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", - "args": ["test","AuthToken"], + "args": ["test", "[synthetic]"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", "environment": [], diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 5cc1d2abf..3a3eb8ab6 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -2,6 +2,7 @@ #include "MDConsumer.h" #include "MemorySegmentUtils.h" #include "PlatformLoader.h" +#include "SecurityCache.h" #include "lib/config/config.h" #include "lib/utils/SharedList.h" #include "lib/utils/WaitFreeQueue.h" @@ -530,8 +531,27 @@ private: // Pimple for MDConsumer /////////////////////////////////////////////////////////////////////////////////////////////////// class MDConsumerImpl { + typedef tbb::queuing_mutex Mutex; + Mutex _workmutex; + vector> _instances; + std::shared_ptr _sci; + map> _deferred_handles; + vector _callbacks; + SecurityCache _cache; + // co2::Platform _platform; + public: - MDConsumerImpl(const set& symbols, std::shared_ptr& ss, bool shared, co2::Platform platform = co2::PLATFORM_UNSET ) { + MDConsumerImpl( + const set& symbols, + std::shared_ptr& ss, + bool shared, co2::Platform platform = co2::PLATFORM_UNSET) : + _workmutex(), + _instances(), + _sci(), + _deferred_handles(), + _callbacks(), + _cache() + { std::ostringstream msg; for (const auto& symbol : symbols) { if (msg.tellp() > 0) { msg << ", "; } @@ -563,6 +583,11 @@ public: if (ss != nullptr) { _sci = std::make_shared(ss, [=](const set& keys) { return get_handles(keys); }); } + + // Add the SecurityCache to on_security so it can cache discovered securities + _callbacks.push_back([this](const SecurityPtr& security, MDConsumerHandle) { + _cache.add_live_security(security); + }); } ~MDConsumerImpl() { } @@ -766,7 +791,7 @@ public: } count += resolved_securities.size(); } - else /*!resolve_legs*/ { + else /* do NOT resolve_legs*/ { auto handles = get_handles(found_securities); for (const auto& security : found_securities) { auto find_result = handles.find(security->id()); @@ -801,6 +826,7 @@ public: } } + _cache.flush_to_disk(); return count; } @@ -843,6 +869,10 @@ public: scan_securities(resolve_legs); } + SecurityPtr get_security(SecurityKey key) { + return _cache.get_security(key); + } + MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const { for (auto& inst : _instances) { if ((inst->symbol() == symbol && inst->type() == type && inst->exchange() == exchange) || @@ -869,14 +899,6 @@ public: return result; } -private: - typedef tbb::queuing_mutex Mutex; - Mutex _workmutex; - vector> _instances; - std::shared_ptr _sci; - map> _deferred_handles; - vector _callbacks; - // co2::Platform _platform; }; @@ -922,6 +944,7 @@ size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol, size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); } void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); } +SecurityPtr MDConsumer::get_security(SecurityKey key) { return _pimpl->get_security(key); } MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType type, Venue exchange) const { return _pimpl->get_channel(symbol, type, exchange); diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 85e74f4b9..b1ea4f304 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -19,7 +19,6 @@ namespace bts { namespace md { -/* typedef void (security_callback)(const Security& security); */ typedef std::function SecurityCallback; using std::size_t; @@ -52,7 +51,9 @@ public: size_t scan_securities(bool resolve_legs = true); void on_security(SecurityCallback cb, bool resolve_legs = true); - // SecurityPtr get_security(SecurityKey key); + // Retrieve any security that has been seen in the past + // You still need to call scan_securities() periodically to discover new securities + SecurityPtr get_security(SecurityKey key); MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const; MDConsumerChannel get_channel(const UserSymbol&) const; diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index 890a599c4..891f2ee90 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -223,6 +223,17 @@ void copy_synthetic_side(size_t count, api::Side side, const api::ContractSpecif } } +const spark::LegSpreaderParams& get_spreader_params(const SyntheticStrategy& synth, security_key key) { + auto strategy = synth.strategy.get(); + if (strategy) { + auto iter = strategy->leg_params_map().find(key); + if (iter != strategy->leg_params_map().end()) { + return iter->second; + } + } + return spark::LegSpreaderParams::default_instance(); +} + TimeStamp copy_synthetic_depth(const SyntheticStrategy& synth, size_t count, PriceSize* bid, PriceSize* ask) { small_vector bid_leg_travs; small_vector ask_leg_travs; @@ -232,14 +243,7 @@ TimeStamp copy_synthetic_depth(const SyntheticStrategy& synth, size_t count, Pri for (size_t x = 0; x < synth.legs.size(); ++x) { std::array bids, asks; TimeStamp t; - auto key = synth.legs[x].key(); - auto strategy = synth.strategy.get(); - auto iter = strategy->leg_params_map().find(key); - const spark::LegSpreaderParams& leg_params = - iter == strategy->leg_params_map().end() ? - spark::LegSpreaderParams::default_instance() : - iter->second; - + const spark::LegSpreaderParams& leg_params = get_spreader_params(synth, synth.legs[x].key()); auto mdtype = leg_params.market_data_type(); HandleReadResult r; if (mdtype == spark::LegSpreaderParams::DIRECT) { diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp index 2675128bb..57a370221 100644 --- a/src/lib/md/SecurityCache.cpp +++ b/src/lib/md/SecurityCache.cpp @@ -13,28 +13,23 @@ struct SecurityCacheImpl { std::mutex cache_mutex; std::unordered_map cache; tbb::concurrent_queue update_queue; - void update_db(const SecurityPtr&); + void enqueue_update_db(const SecurityPtr&); + void flush(); }; -void SecurityCacheImpl::update_db(const SecurityPtr& security) { - // Batch updates to the database for performance - // algo: - // push item into queue - // post work to dispatcher with delay - // When dispatcher services request drain update_queue. - // This will cause one update per security but most will be no-op - // This is a little simpler than running a timer and trying to stop it after security discovery is done +void SecurityCacheImpl::enqueue_update_db(const SecurityPtr& security) { update_queue.push(security); - utils::Dispatcher::background().post([this]() { - if (!update_queue.empty()) { - std::vector work; - SecurityPtr temp; - while(update_queue.try_pop(temp)) { work.push_back(temp); } - if (!work.empty()) { - db::insert_or_replace(work); - } +} + +void SecurityCacheImpl::flush() { + if (!update_queue.empty()) { + std::vector work; + SecurityPtr temp; + while(update_queue.try_pop(temp)) { work.push_back(temp); } + if (!work.empty()) { + db::insert_or_replace(work); } - }, 1000); + } } ////////////////// @@ -83,8 +78,10 @@ bool SecurityCache::add_live_security(const SecurityPtr& security) { SecurityKey key(security->id()); // Insert but don't overwrite existing auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security)); - _me->update_db(security); + _me->enqueue_update_db(security); return result.second; } +void SecurityCache::flush_to_disk() { _me->flush(); } + }}//ns \ No newline at end of file diff --git a/src/lib/md/SecurityCache.h b/src/lib/md/SecurityCache.h index 388d66fd8..85de41b70 100644 --- a/src/lib/md/SecurityCache.h +++ b/src/lib/md/SecurityCache.h @@ -22,13 +22,15 @@ public: //Load security from cache with fallback to database SecurityPtr get_security(SecurityKey key); - // Get the security if it's in the cache otherwise return nullptr + // Get the security if it's in the cache otherwise return nullptr. Does not query security database. SecurityPtr try_get_security(SecurityKey key) const; // returns true if the cache was updated bool add_live_security(const SecurityPtr& security); SecurityPtr add_db_security(const SecurityPtr& security); + void flush_to_disk(); + }; }}//ns \ No newline at end of file diff --git a/src/lib/md/test/synthetic_book_merge_tests.cpp b/src/lib/md/test/synthetic_book_merge_tests.cpp index 822ebb907..87e8ce79a 100644 --- a/src/lib/md/test/synthetic_book_merge_tests.cpp +++ b/src/lib/md/test/synthetic_book_merge_tests.cpp @@ -135,6 +135,9 @@ tuple,vector> make_securities(c void set_leg_params(SyntheticStrategy& strat, uint32_t security_key, const spark::LegSpreaderParams& params) { + if (!strat.strategy) { + strat.strategy = std::make_unique(); + } auto* map = strat.strategy->mutable_leg_params_map(); map->insert({ security_key, params }); } From e36729814a31b3e11c4477e556a4ccd1fac48169 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 7 Jan 2019 16:53:04 -0500 Subject: [PATCH 12/54] SPARK-1731 Improve error handling in SecurityDB and implement db::find_security() --- src/lib/md/SecurityCache.cpp | 5 +- src/lib/md/SecurityDB.cpp | 94 ++++++++++++++++++++++-------------- src/lib/md/SecurityDB.h | 3 +- src/lib/utils/ResultEnum.h | 7 +++ 4 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 src/lib/utils/ResultEnum.h diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp index 57a370221..affe44c9e 100644 --- a/src/lib/md/SecurityCache.cpp +++ b/src/lib/md/SecurityCache.cpp @@ -27,7 +27,10 @@ void SecurityCacheImpl::flush() { SecurityPtr temp; while(update_queue.try_pop(temp)) { work.push_back(temp); } if (!work.empty()) { - db::insert_or_replace(work); + if (db::insert_or_replace(work) == Result::error) { + // If we fail to insert then retry later + for (auto& s : work) update_queue.push(std::move(s)); + } } } } diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index c374765a9..e16bb7584 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -56,15 +56,31 @@ datetime::Date sec_expiration_cut_off() return datetime::Date(pt.date().year(),pt.date().month(),pt.date().day()); } +void apply_compatibilit_mutations(Security& security) { + // Migrate from old tick stuff + if (security.has_glbx()) { + if (security.has_glbx()) { + if (security.contract_spec().main_fraction() == 0) { + security.mutable_contract_spec()->set_main_fraction(security.glbx().do_not_use_main_fraction()); + } + if (security.contract_spec().sub_fraction() == 0) { + security.mutable_contract_spec()->set_sub_fraction(security.glbx().do_not_use_sub_fraction()); + } + if (security.contract_spec().price_display_format() == 0) { + security.mutable_contract_spec()->set_price_display_format(security.glbx().do_not_use_price_display_format()); + } + } + } +} + size_t load_securities( const std::set& symbols, const std::function& callback) { size_t count = 0; - auto dbptr = open_db(); + auto db = open_db(); // Load all existing securities and call the callback - auto db = dbptr.get(); SQLite::Statement select_stmt(*db, "select key,symbol,venue,type,security_blob from securities where expiration > :exp"); select_stmt.bind(":exp", sec_expiration_cut_off().to_short()); @@ -77,21 +93,8 @@ size_t load_securities( UserSymbol us(symbol, venue, type); if (symbols.find(us) != symbols.end()) { auto blob = select_stmt.getColumn("security_blob"); - // auto data = blob.getBlob(); - // auto len = blob.getBytes(); - // Migrate from old tick stuff if (security.ParseFromArray(blob, blob.getBytes())) { - if (security.has_glbx()) { - if (security.contract_spec().main_fraction() == 0) { - security.mutable_contract_spec()->set_main_fraction(security.glbx().do_not_use_main_fraction()); - } - if (security.contract_spec().sub_fraction() == 0) { - security.mutable_contract_spec()->set_sub_fraction(security.glbx().do_not_use_sub_fraction()); - } - if (security.contract_spec().price_display_format() == 0) { - security.mutable_contract_spec()->set_price_display_format(security.glbx().do_not_use_price_display_format()); - } - } + apply_compatibilit_mutations(security); bts_assert(security_key == security.id()); count++; callback(security); @@ -106,29 +109,50 @@ size_t load_securities( } SecurityPtr find_security(SecurityKey key) { - //todo: - (void)key; + try { + auto db = open_db(); + SQLite::Statement select(*db, "select security_blob from securities where key = :key"); + select.bind(":key", key.value()); + if(select.tryExecuteStep() == SQLITE_ROW) { + auto result = std::make_shared(); + auto blob = select.getColumn(0); + if(result->ParseFromArray(blob, blob.getBytes())) { + apply_compatibilit_mutations(*result); + return result; + } + } + } + catch(const SQLite::Exception& e) { + LOG(md,Error, "Error reading security database. {}", std::string(e.what())); + } return nullptr; } -void insert_or_replace(const std::vector& securities) { - auto db = open_db(); - SQLite::Statement insert(*db,"insert or replace into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); - SQLite::Transaction transaction(*db); - std::string buffer; - for (const auto& psecurity : securities) { - if (psecurity && psecurity->SerializeToString(&buffer)) { - insert.bind(1, psecurity->id()); - insert.bind(2, psecurity->symbol()); - insert.bind(3, psecurity->exchange()); - insert.bind(4, securitytype_of(*psecurity)); - insert.bind(5, buffer); - insert.bind(6, expiration(*psecurity).to_short()); - insert.exec(); - insert.reset(); +Result insert_or_replace(const std::vector& securities) { + try { + auto db = open_db(); + SQLite::Statement insert(*db,"insert or replace into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); + SQLite::Transaction transaction(*db); + std::string buffer; + for (const auto& psecurity : securities) { + if (psecurity && psecurity->SerializeToString(&buffer)) { + insert.bind(1, psecurity->id()); + insert.bind(2, psecurity->symbol()); + insert.bind(3, psecurity->exchange()); + insert.bind(4, securitytype_of(*psecurity)); + insert.bind(5, buffer); + insert.bind(6, expiration(*psecurity).to_short()); + insert.exec(); + insert.reset(); + } } + transaction.commit(); } - transaction.commit(); + catch(const SQLite::Exception& e) { + LOG(md,Warn, "Error writing to security db (retry pending): {}", std::string(e.what())); + return Result::error; + } + return Result::ok; } @@ -175,7 +199,7 @@ size_t load_securities( const std::function& callback) { return 0; } -void insert_or_replace(const std::vector& handles) {} +Result insert_or_replace(const std::vector& handles) { return Result::error; } SecurityPtr find_security(SecurityKey) { return nullptr; } #endif diff --git a/src/lib/md/SecurityDB.h b/src/lib/md/SecurityDB.h index 77948dcce..b04f355da 100644 --- a/src/lib/md/SecurityDB.h +++ b/src/lib/md/SecurityDB.h @@ -3,6 +3,7 @@ #include "lib/domain_model/common.h" #include "lib/domain_model/Security.pb.h" #include "lib/utils/datetime.h" +#include "lib/utils/ResultEnum.h" #include "lib/md/MDProducerHandle.h" #include "lib/api/services/SyntheticSecurityService.pb.h" #include "common.h" @@ -29,7 +30,7 @@ size_t load_securities( SecurityPtr find_security(SecurityKey); -void insert_or_replace(const std::vector&); +Result insert_or_replace(const std::vector&); // metadata std::vector load_metadata(); diff --git a/src/lib/utils/ResultEnum.h b/src/lib/utils/ResultEnum.h new file mode 100644 index 000000000..3509ce0b2 --- /dev/null +++ b/src/lib/utils/ResultEnum.h @@ -0,0 +1,7 @@ +#pragma once +namespace bts { + enum class Result { + ok, + error + }; +} \ No newline at end of file From 2020f6af6d0d99ed99f40d0f6ec7e5d7e14c4a2b Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 11:28:19 -0500 Subject: [PATCH 13/54] SPARK-1951 Add SecurityCache access to QueryServer --- src/lib/api/services/MarketDataRelay.proto | 8 +++++++- src/lib/domain_model/Security.proto | 4 ++++ src/lib/md/MDConsumer.cpp | 6 +++--- src/lib/md/MDConsumer.h | 2 +- src/lib/md/relay/QueryServer.cpp | 19 +++++++++++++++++++ 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/lib/api/services/MarketDataRelay.proto b/src/lib/api/services/MarketDataRelay.proto index d93256c7c..d83daf33b 100644 --- a/src/lib/api/services/MarketDataRelay.proto +++ b/src/lib/api/services/MarketDataRelay.proto @@ -14,14 +14,20 @@ message ExpirationNameList { bts.api.Product product = 1; map names = 2; } - message ExpirationNameListReply { repeated ExpirationNameList securities = 2; } +message SecurityKeyList { + repeated uint32 keys = 1; +} + service MarketDataRelay { option (bts.api.service_description)="Poll the market data service for updates"; rpc requestmd(bts.co2.MDRequest) returns (bts.co2.MDReply); + rpc request_products(bts.api.VoidMessage) returns (ProductListReply); rpc request_names(bts.api.VoidMessage) returns (ExpirationNameListReply); + + rpc get_securities(SecurityKeyList) returns (bts.api.SecurityList); } diff --git a/src/lib/domain_model/Security.proto b/src/lib/domain_model/Security.proto index 5e921021e..8ee1db886 100644 --- a/src/lib/domain_model/Security.proto +++ b/src/lib/domain_model/Security.proto @@ -1,6 +1,7 @@ syntax="proto3"; package bts.api; option csharp_namespace="bts.api"; +option cc_enable_arenas=true; import "lib/api/apioptions.proto"; enum Venue { @@ -163,6 +164,9 @@ message Security { ICEExchangeData ice = 230; } } +message SecurityList { + repeated Security security = 1; +} message Product { bts.api.Venue exchange = 1; diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 3a3eb8ab6..449bfe420 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -537,7 +537,7 @@ class MDConsumerImpl { std::shared_ptr _sci; map> _deferred_handles; vector _callbacks; - SecurityCache _cache; + mutable SecurityCache _cache; // co2::Platform _platform; public: @@ -869,7 +869,7 @@ public: scan_securities(resolve_legs); } - SecurityPtr get_security(SecurityKey key) { + SecurityPtr get_security(SecurityKey key) const { return _cache.get_security(key); } @@ -944,7 +944,7 @@ size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol, size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); } void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); } -SecurityPtr MDConsumer::get_security(SecurityKey key) { return _pimpl->get_security(key); } +SecurityPtr MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key); } MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType type, Venue exchange) const { return _pimpl->get_channel(symbol, type, exchange); diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index b1ea4f304..a53b33665 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -53,7 +53,7 @@ public: // Retrieve any security that has been seen in the past // You still need to call scan_securities() periodically to discover new securities - SecurityPtr get_security(SecurityKey key); + SecurityPtr get_security(SecurityKey key) const; MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const; MDConsumerChannel get_channel(const UserSymbol&) const; diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 57ab06e89..6c3e2e457 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -388,6 +388,16 @@ public: } } + void get_securities(google::protobuf::RepeatedField keys, api::SecurityList* result) const { + bts_assert(result); + for (auto k : keys) { + auto ptr = _consumer.get_security(SecurityKey(k)); + if (ptr) { + result->add_security()->CopyFrom(*ptr); + } + } + } + }; struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { @@ -474,6 +484,15 @@ struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { reply(*preply); }); } + + virtual void get_securities(const api::AuthToken&, const SecurityKeyList& keys, const std::function& reply) override { + _threadpool.post([=]() { + google::protobuf::Arena arena; + api::SecurityList* preply = google::protobuf::Arena::CreateMessage(&arena); + _db.get_securities(keys.keys(), preply); + reply(*preply); + }); + } }; struct QueryServer { From 20fe97bbbaf02023a1892aa8a99f6501d456a135 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 11:45:18 -0500 Subject: [PATCH 14/54] Move ConsoleEx.cs because it's in the wrong place --- src/client/spark_core/{TestRunner => utils}/ConsoleEx.cs | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/client/spark_core/{TestRunner => utils}/ConsoleEx.cs (100%) diff --git a/src/client/spark_core/TestRunner/ConsoleEx.cs b/src/client/spark_core/utils/ConsoleEx.cs similarity index 100% rename from src/client/spark_core/TestRunner/ConsoleEx.cs rename to src/client/spark_core/utils/ConsoleEx.cs From aea38c168984d8be7be7b59be1a0e8a9b8f8a15a Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 12:54:38 -0500 Subject: [PATCH 15/54] SPARK-1962 - Add SecurityStore.cs --- src/client/spark/Daemons/ShellServices.cs | 5 +- src/client/spark/Security/SecurityStore.cs | 92 ++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 src/client/spark/Security/SecurityStore.cs diff --git a/src/client/spark/Daemons/ShellServices.cs b/src/client/spark/Daemons/ShellServices.cs index eb9deee90..c51a6b0da 100644 --- a/src/client/spark/Daemons/ShellServices.cs +++ b/src/client/spark/Daemons/ShellServices.cs @@ -5,7 +5,7 @@ using bts.api.orderman; using bts.api.persistence; using bts.api.position; using bts.api.synthetic; -using bts.api.user; +using bts.co2; using bts.metrics; using bts.support; using System.ComponentModel.Composition; @@ -41,5 +41,8 @@ namespace Spark { [Export] public MarketDataAuxiliaryService md_auxiliary_service => context.locate(); + + [Export] + public MarketDataRelay md_replay => context.locate(); } } \ No newline at end of file diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs new file mode 100644 index 000000000..3ef2673bf --- /dev/null +++ b/src/client/spark/Security/SecurityStore.cs @@ -0,0 +1,92 @@ +using bts.api; +using bts.co2; +using mbts; +using mbts.spark; +using System; +using System.Collections.Generic; +using System.ComponentModel.Composition; +using System.Threading.Tasks; +using System.Linq; +using security_key = System.UInt32; +using System.Collections.Concurrent; + +namespace bts { + [Export] + class SecurityStore : IDisposable { + readonly MarketDataRelay _md; + IDisposable _dispose; + IReadOnlyDictionary _live_securities; + readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); + + [ImportingConstructor] + public SecurityStore(MarketDataRelay md, IBus bus) { + _md = md; + _dispose = bus.subscribe(SparkTags.SecuritiesTag).Subscribe(dict => _live_securities = dict); + } + + SecurityImmutable? try_get_local(security_key key) { + // Try live securities first + if (_live_securities != null) { + if (_live_securities.TryGetValue(key, out SecurityMD value)) { + return value.security; + } + } + + // Then try cached securities + if (_cache.TryGetValue(key, out SecurityImmutable security)) { + return security; + } + + return null; + } + + public async Task get_security(security_key key) { + var local_sec = try_get_local(key); + if (local_sec.HasValue) return local_sec; + + // Query the server for the value + var result = await _md.get_securities_async(new SecurityKeyList() { + Keys = { key } + }); + + if (result.success) { + var security = result.value.Security.FirstOrDefault(); + if (security != null) { + _cache.TryAdd(security.Id, new SecurityImmutable(security)); + return security; + } + } + + // Give up :( + Log.Md.warn($"Failed to get security for key {key}"); + return null; + } + + public async Task> get_securities(IEnumerable keys) { + var result = new Dictionary(); + var todo = new SecurityKeyList(); + foreach (var key in keys) { + var s = try_get_local(key); + if (s.HasValue) { + result.Add(key, s.Value); + } + else { + todo.Keys.Add(key); + } + } + if (todo.Keys.Count > 0) { + var res = await _md.get_securities_async(todo); + if (res.success) { + foreach (var security in res.value.Security) { + result.Add(security.Id, new SecurityImmutable(security)); + } + } + } + return result; + } + + public void Dispose() { + Disposable.dispose(ref _dispose); + } + } +} From 1abec0ab39b706ac7ea3fa6fab334c21a5331211 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 16:32:57 -0500 Subject: [PATCH 16/54] Attempt to defer initial security publish until loading is complete so initial load is not bogged down by ticking security feed --- src/client/spark/Daemons/MDDaemon.cs | 12 +++++++----- src/client/spark_api/md/MDSource.cs | 24 ++++++++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/client/spark/Daemons/MDDaemon.cs b/src/client/spark/Daemons/MDDaemon.cs index 816faac4a..4231dd0f9 100644 --- a/src/client/spark/Daemons/MDDaemon.cs +++ b/src/client/spark/Daemons/MDDaemon.cs @@ -14,19 +14,21 @@ using SecurityKey = System.UInt32; using System.Collections.Concurrent; using System.Threading; using bts.api.user; +using bts.md; namespace bts { [Export(typeof(IInitializable))] - class MDDaemon : bts.md.MDSource, IObserver>, IInitializable, IDisposable { + class MDDaemon : md.MDSource, IObserver>, IInitializable, IDisposable { readonly Lazy _bus; readonly Lazy _token; readonly Lazy _ss; readonly Dictionary _deleted_synthetics = new Dictionary(); - static uint get_naggle() => mbts.config.Configuration.get().naggle_ms; + static uint get_naggle() => mbts.config.Configuration.get().naggle_ms; [ImportingConstructor] - public MDDaemon(Lazy bus, Lazy token, Lazy ss) : base(get_naggle(), true) { + public MDDaemon(Lazy bus, Lazy token, Lazy ss) : + base(get_naggle(), MDSourceOptions.DeferInit | MDSourceOptions.DeferPublish) { _bus = bus; _token = token; _ss = ss; @@ -44,14 +46,14 @@ namespace bts { if (!deleted_synthetic_keys.Contains(kvp.Key)) { _exchange_sec_map.TryAdd(kvp.Key, kvp.Value); _deleted_synthetics.Remove(kvp.Key); - _version += 1; + _version = DateTime.UtcNow; } } // Remove any newly deleted synthetics and store them for potential undelete foreach (var key in deleted_synthetic_keys) { if (_exchange_sec_map.TryRemove(key, out SecurityMD deleted)) { _deleted_synthetics[key] = deleted; - _version += 1; + _version = DateTime.UtcNow; } } // Post full dictionary of syntheics here diff --git a/src/client/spark_api/md/MDSource.cs b/src/client/spark_api/md/MDSource.cs index 9fdebb63b..d8559e799 100644 --- a/src/client/spark_api/md/MDSource.cs +++ b/src/client/spark_api/md/MDSource.cs @@ -15,20 +15,27 @@ using System.Reactive.Subjects; namespace bts.md { + [Flags] + enum MDSourceOptions { + DeferInit = 1, + DeferPublish = 2 + } class MDSource : IObserver>, IDisposable { protected readonly MDConsumer _consumer = new MDConsumer(); protected readonly Co2RelayControl _relay; - protected long _version = 0; + protected DateTime _version = DateTime.MinValue; protected readonly ConcurrentDictionary _exchange_sec_map = new ConcurrentDictionary(); protected List _dispose = new List(); + protected readonly MDSourceOptions _options; - public MDSource(uint naggle_ms, bool defer_init = false) - { + public MDSource(uint naggle_ms, MDSourceOptions options = 0) + { + _options = options; _relay = new Co2RelayControl(_consumer, naggle_ms); _relay.preload_securities((co2.MDRequest.Types.Flags)0xff).ignore(); _dispose.Add(_consumer.on_security.Subscribe(this)); - if(!defer_init) + if(options.HasFlag(MDSourceOptions.DeferInit)) initialize(); } @@ -57,19 +64,20 @@ namespace bts.md var smd = new SecurityMD(new SecurityImmutable(v.security), v.handle); _exchange_sec_map.AddOrUpdate(smd.security.Id, smd, (_, __) => smd); } - _version += 1; + _version = DateTime.UtcNow; } } public void Dispose() => Disposable.dispose(_dispose); async protected Task run() { - long run_version = 0; + DateTime run_version = DateTime.MinValue; + var minimum_change_delta = _options.HasFlag(MDSourceOptions.DeferPublish) ? TimeSpan.FromMilliseconds(200) : TimeSpan.Zero; while (true) { var exch = _exchange_sec_map; - long current_version = _version; - if (current_version > run_version) + var current_version = _version; + if (current_version - run_version > minimum_change_delta) { run_version = current_version; try From 2e542fb8f53c54de2dd2ee18c7f51a1213d36268 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 16:33:41 -0500 Subject: [PATCH 17/54] Update.cs all should be a more generic type --- src/client/spark_api/api/Update.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/spark_api/api/Update.cs b/src/client/spark_api/api/Update.cs index 6f9604a01..d5f1b04fd 100644 --- a/src/client/spark_api/api/Update.cs +++ b/src/client/spark_api/api/Update.cs @@ -70,9 +70,9 @@ namespace bts.api.core { } public class Update where TKey : IEquatable { - private Func _key; + readonly Func _key; internal Update( - IReadOnlyCollection all, + IEnumerable all, IEnumerable added, IEnumerable> changes, IEnumerable removed, @@ -122,7 +122,7 @@ namespace bts.api.core { ); } - public IReadOnlyCollection all { get; } + public IEnumerable all { get; } public IEnumerable added { get; } public IEnumerable> changes { get; } public IEnumerable changed => changes.Select(c => c.New); From adf4476399d68513f63f84f72e7de5283ba551ae Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 16:46:44 -0500 Subject: [PATCH 18/54] SPARK-1962 - Efficient observable join on all available securities --- src/client/spark/Security/SecurityStore.cs | 65 ++++++++++++++++++++-- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 3ef2673bf..625697ae1 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -1,14 +1,17 @@ using bts.api; +using bts.api.core; using bts.co2; using mbts; using mbts.spark; using System; -using System.Collections.Generic; -using System.ComponentModel.Composition; -using System.Threading.Tasks; -using System.Linq; -using security_key = System.UInt32; using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.ComponentModel.Composition; +using System.Linq; +using System.Reactive.Linq; +using System.Threading.Tasks; +using security_key = System.UInt32; namespace bts { [Export] @@ -42,7 +45,9 @@ namespace bts { public async Task get_security(security_key key) { var local_sec = try_get_local(key); - if (local_sec.HasValue) return local_sec; + if (local_sec.HasValue) { + return local_sec; + } // Query the server for the value var result = await _md.get_securities_async(new SecurityKeyList() { @@ -85,6 +90,54 @@ namespace bts { return result; } + IObservable> join(IObservable> source) { + return Observable.Create>(observer => { + var all = ImmutableDictionary.Empty; + return source.Subscribe(async (update) => { + var key = update.key_extractor; + var relevant_securities = await get_securities(update.added.Concat(update.changed).Select(key)); + + var added = new List<(SecurityImmutable?, T)>(); + var changed = new List>(); + var removed = new List<(SecurityImmutable?, T)>(); + + foreach (var x in update.added) { + if (relevant_securities.TryGetValue(key(x), out SecurityImmutable s)) { + added.Add((s, x)); + } + else { + added.Add((null, x)); + } + } + foreach (var x in update.removed) { + if (all.TryGetValue(key(x), out (SecurityImmutable?, T) value)) { + removed.Add(value); + } + } + foreach (var ch in update.changes) { + SecurityImmutable? si = + relevant_securities.TryGetValue(key(ch.New), out SecurityImmutable s) ? + s : (SecurityImmutable?)null; + changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New))); + } + + var builder = all.ToBuilder(); + builder.RemoveRange(update.removed.Select(key)); + builder.RemoveRange(changed.Select(ch => key(ch.Old.Item2))); + foreach (var a in added) { + builder.Add(key(a.Item2), a); + } + foreach (var ch in changed) { + builder.Add(key(ch.New.Item2), ch.New); + } + all = builder.ToImmutable(); + + observer.OnNext(new Update<(SecurityImmutable?, T), security_key>( + all.Values, added, changed, removed, i => key(i.Item2))); + }); + }); + } + public void Dispose() { Disposable.dispose(ref _dispose); } From 4991fa734c2a3b756f73e783e889e66b361c6c7c Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 17:33:31 -0500 Subject: [PATCH 19/54] SPARK-1962 - Add operator so this join can be used inline --- src/client/spark/Security/SecurityStore.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 625697ae1..1167cddcf 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -15,7 +15,7 @@ using security_key = System.UInt32; namespace bts { [Export] - class SecurityStore : IDisposable { + public class SecurityStore : IDisposable { readonly MarketDataRelay _md; IDisposable _dispose; IReadOnlyDictionary _live_securities; @@ -90,12 +90,18 @@ namespace bts { return result; } - IObservable> join(IObservable> source) { + public void Dispose() { + Disposable.dispose(ref _dispose); + } + } + + public static class SecurityStoreExtensions { + public static IObservable> join(this IObservable> source, SecurityStore store) { return Observable.Create>(observer => { var all = ImmutableDictionary.Empty; return source.Subscribe(async (update) => { var key = update.key_extractor; - var relevant_securities = await get_securities(update.added.Concat(update.changed).Select(key)); + var relevant_securities = await store.get_securities(update.added.Concat(update.changed).Select(key)); var added = new List<(SecurityImmutable?, T)>(); var changed = new List>(); @@ -137,9 +143,5 @@ namespace bts { }); }); } - - public void Dispose() { - Disposable.dispose(ref _dispose); - } } } From 12f6c58c7cce4ddbbf5229812a7970c2170c4035 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 9 Jan 2019 15:24:45 -0500 Subject: [PATCH 20/54] Reconsidering 1abec0ab --- src/client/spark/Daemons/MDDaemon.cs | 2 +- src/client/spark_api/md/MDSource.cs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/client/spark/Daemons/MDDaemon.cs b/src/client/spark/Daemons/MDDaemon.cs index 4231dd0f9..70361557b 100644 --- a/src/client/spark/Daemons/MDDaemon.cs +++ b/src/client/spark/Daemons/MDDaemon.cs @@ -28,7 +28,7 @@ namespace bts { [ImportingConstructor] public MDDaemon(Lazy bus, Lazy token, Lazy ss) : - base(get_naggle(), MDSourceOptions.DeferInit | MDSourceOptions.DeferPublish) { + base(get_naggle(), MDSourceOptions.DeferInit) { _bus = bus; _token = token; _ss = ss; diff --git a/src/client/spark_api/md/MDSource.cs b/src/client/spark_api/md/MDSource.cs index d8559e799..392fc6844 100644 --- a/src/client/spark_api/md/MDSource.cs +++ b/src/client/spark_api/md/MDSource.cs @@ -18,7 +18,6 @@ namespace bts.md [Flags] enum MDSourceOptions { DeferInit = 1, - DeferPublish = 2 } class MDSource : IObserver>, IDisposable { @@ -35,7 +34,7 @@ namespace bts.md _relay = new Co2RelayControl(_consumer, naggle_ms); _relay.preload_securities((co2.MDRequest.Types.Flags)0xff).ignore(); _dispose.Add(_consumer.on_security.Subscribe(this)); - if(options.HasFlag(MDSourceOptions.DeferInit)) + if(!options.HasFlag(MDSourceOptions.DeferInit)) initialize(); } @@ -72,12 +71,11 @@ namespace bts.md async protected Task run() { DateTime run_version = DateTime.MinValue; - var minimum_change_delta = _options.HasFlag(MDSourceOptions.DeferPublish) ? TimeSpan.FromMilliseconds(200) : TimeSpan.Zero; while (true) { var exch = _exchange_sec_map; var current_version = _version; - if (current_version - run_version > minimum_change_delta) + if (current_version > run_version) { run_version = current_version; try From e5ab062203957f5a1744c698bee9d51fe8736fd7 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 9 Jan 2019 18:32:52 -0500 Subject: [PATCH 21/54] SPARK-1962 Fix race bug identified in testing --- src/client/spark/Orders/OrdersModel.cs | 17 +- src/client/spark/Security/SecurityStore.cs | 224 +++++++++++++++------ 2 files changed, 179 insertions(+), 62 deletions(-) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index eba27fea3..910ff14e6 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -34,7 +34,8 @@ namespace mbts.spark { [ImportingConstructor] private OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, - SpreaderModel spreader, ErrorReporter error, DynamicOrderHelper dynamic_helper, AdminModel admin_model) { + SpreaderModel spreader, ErrorReporter error, DynamicOrderHelper dynamic_helper, AdminModel admin_model, + SecurityStore sstore) { _bus = bus; _factory = factory; _service = service; @@ -44,7 +45,13 @@ namespace mbts.spark { _dynamic_helper = dynamic_helper; var scheduler = ThreadPoolScheduler.Instance; var interval = TimeSpan.FromMilliseconds(100); - on_orders = _service.subscribei_orders().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); + on_orders = + _service + .subscribei_orders() + .Naggle(interval, scheduler) + .PermissionFilter(admin_model.on_admin_data) + //.join(sstore) + .Table().Cache(); on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); working_orders = on_orders.Scan( @@ -55,7 +62,11 @@ namespace mbts.spark { on_order_tuples = order_record_tuple_source.subscribe_incremental_update(); on_order_tuples_table = order_record_tuple_source.subscribe_incremental_table_update(); on_order_securities = order_record_tuple_source.subscribe_order_securities(); - on_fills = _service.subscribei_fills().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); + on_fills = + _service.subscribei_fills() + .Naggle(interval, scheduler) + .PermissionFilter(admin_model.on_admin_data) + .Table().Cache(); _activity = new OrderActivitySummarizer(bus, on_orders); on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders, on_fills); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 1167cddcf..5e424beac 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -10,6 +10,7 @@ using System.Collections.Immutable; using System.ComponentModel.Composition; using System.Linq; using System.Reactive.Linq; +using System.Reactive.Subjects; using System.Threading.Tasks; using security_key = System.UInt32; @@ -17,20 +18,27 @@ namespace bts { [Export] public class SecurityStore : IDisposable { readonly MarketDataRelay _md; + readonly IBus _bus; IDisposable _dispose; IReadOnlyDictionary _live_securities; readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); + readonly Subject> _cache_subject = new Subject>(); [ImportingConstructor] public SecurityStore(MarketDataRelay md, IBus bus) { _md = md; + _bus = bus; _dispose = bus.subscribe(SparkTags.SecuritiesTag).Subscribe(dict => _live_securities = dict); + _cache_subject.OnNext(_cache); } - SecurityImmutable? try_get_local(security_key key) { + IObservable> on_securities => _bus.subscribe(SparkTags.SecuritiesTag); + + SecurityImmutable? try_get_local(security_key key, IReadOnlyDictionary live=null) { // Try live securities first - if (_live_securities != null) { - if (_live_securities.TryGetValue(key, out SecurityMD value)) { + live = live ?? _live_securities; + if (live != null) { + if (live.TryGetValue(key, out SecurityMD value)) { return value.security; } } @@ -43,8 +51,8 @@ namespace bts { return null; } - public async Task get_security(security_key key) { - var local_sec = try_get_local(key); + async Task get_security(security_key key, IReadOnlyDictionary live) { + var local_sec = try_get_local(key, live); if (local_sec.HasValue) { return local_sec; } @@ -57,8 +65,7 @@ namespace bts { if (result.success) { var security = result.value.Security.FirstOrDefault(); if (security != null) { - _cache.TryAdd(security.Id, new SecurityImmutable(security)); - return security; + return try_cache_security(new SecurityImmutable(security)); } } @@ -67,81 +74,180 @@ namespace bts { return null; } - public async Task> get_securities(IEnumerable keys) { + async Task> get_securities(IEnumerable keys, IReadOnlyDictionary live) { var result = new Dictionary(); - var todo = new SecurityKeyList(); + var missing = new SecurityKeyList(); foreach (var key in keys) { var s = try_get_local(key); if (s.HasValue) { result.Add(key, s.Value); } else { - todo.Keys.Add(key); + missing.Keys.Add(key); } } - if (todo.Keys.Count > 0) { - var res = await _md.get_securities_async(todo); + if (missing.Keys.Count > 0) { + var res = await _md.get_securities_async(missing); if (res.success) { foreach (var security in res.value.Security) { - result.Add(security.Id, new SecurityImmutable(security)); + result.Add(security.Id, try_cache_security(new SecurityImmutable(security))); } } } return result; } + // Request any missing keys from the server but do not wait for them to arrive + IReadOnlyDictionary get_and_ensure_securities(IEnumerable keys) { + var result = new Dictionary(); + var missing = new SecurityKeyList(); + foreach (var key in keys) { + var security = try_get_local(key); + if (security.HasValue) { + result.Add(key, security.Value); + } + else { + missing.Keys.Add(key); + } + } + // Update cache in the background + Task.Run(async () => { + var r = await _md.get_securities_async(missing); + if (r.success) { + foreach (var security in r.value.Security) { + try_cache_security(new SecurityImmutable(security)); + } + } + }); + return result; + } + + SecurityImmutable try_cache_security(SecurityImmutable s) { + if (_live_securities == null || !_live_securities.TryGetValue(s.Id, out SecurityMD live)) { + _cache.TryAdd(s.Id, s); + _cache_subject.OnNext(_cache); + return s; + } + return live.security; + } + + /////////////// + // PUBLIC + /////////////// + #region public + + // Guarunteed to return a security if it has ever been seen by the system + public Task get_security(security_key key) { + return get_security(key, _live_securities); + } + // Guarunteed to return a security if it has ever been seen by the system + public Task> get_securities(IEnumerable keys) { + return get_securities(keys, _live_securities); + } public void Dispose() { Disposable.dispose(ref _dispose); } + + public IObservable> join( + IObservable> source) { + return Observable.Create>(observer => { + var state = ImmutableDictionary.Empty; + var gate = new object(); + Func keyfn = null; + + void on_securities_updated(object ambiguous) { + lock (gate) { + // If this is null we don't have any rows + if (keyfn == null) return; + var changed = new List>(); + foreach (var entry in state.Values) { + if (entry.s == null) { + var security = try_get_local(keyfn(entry.t), ambiguous as IReadOnlyDictionary); + if (security.HasValue) { + changed.Add( + new Change<(SecurityImmutable?, T)>(entry, (security, entry.t))); + } + } + } + if (changed.Count > 0) { + // update state + var builder = state.ToBuilder(); + foreach (var up in changed) { + builder[up.New.s.Value.Id] = up.New; + } + state = builder.ToImmutable(); + // fire update + observer.OnNext( + new Update<(SecurityImmutable?, T), security_key>( + state.Values, + Enumerable.Empty<(SecurityImmutable?,T)>(), + changed, + Enumerable.Empty<(SecurityImmutable?,T)>(), + i => keyfn(i.Item2))); + } + } + } + + return Disposable.Create( + // Fire update when source changes + source.Subscribe((update) => { + lock (gate) { + keyfn = update.key_extractor; + var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changed).Select(keyfn)); + + var added = new List<(SecurityImmutable?, T)>(); + var changed = new List>(); + var removed = new List<(SecurityImmutable?, T)>(); + + foreach (var x in update.added) { + if (relevant_securities.TryGetValue(keyfn(x), out SecurityImmutable s)) { + added.Add((s, x)); + } + else { + added.Add((null, x)); + } + } + foreach (var x in update.removed) { + if (state.TryGetValue(keyfn(x), out (SecurityImmutable?, T) value)) { + removed.Add(value); + } + } + foreach (var ch in update.changes) { + SecurityImmutable? si = + relevant_securities.TryGetValue(keyfn(ch.New), out SecurityImmutable s) ? + s : (SecurityImmutable?)null; + changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New))); + } + + var builder = state.ToBuilder(); + builder.RemoveRange(update.removed.Select(keyfn)); + builder.RemoveRange(changed.Select(ch => keyfn(ch.Old.Item2))); + foreach (var a in added) { + builder.Add(keyfn(a.Item2), a); + } + foreach (var ch in changed) { + builder.Add(keyfn(ch.New.Item2), ch.New); + } + state = builder.ToImmutable(); + + observer.OnNext(new Update<(SecurityImmutable?, T), security_key>( + state.Values, added, changed, removed, i => keyfn(i.Item2))); + } + }), + // Also fire update when securities change + on_securities.Subscribe(on_securities_updated), + // Also fire update when cache changes + _cache_subject.Subscribe(on_securities_updated) + ); + }); + } + #endregion } public static class SecurityStoreExtensions { - public static IObservable> join(this IObservable> source, SecurityStore store) { - return Observable.Create>(observer => { - var all = ImmutableDictionary.Empty; - return source.Subscribe(async (update) => { - var key = update.key_extractor; - var relevant_securities = await store.get_securities(update.added.Concat(update.changed).Select(key)); - - var added = new List<(SecurityImmutable?, T)>(); - var changed = new List>(); - var removed = new List<(SecurityImmutable?, T)>(); - - foreach (var x in update.added) { - if (relevant_securities.TryGetValue(key(x), out SecurityImmutable s)) { - added.Add((s, x)); - } - else { - added.Add((null, x)); - } - } - foreach (var x in update.removed) { - if (all.TryGetValue(key(x), out (SecurityImmutable?, T) value)) { - removed.Add(value); - } - } - foreach (var ch in update.changes) { - SecurityImmutable? si = - relevant_securities.TryGetValue(key(ch.New), out SecurityImmutable s) ? - s : (SecurityImmutable?)null; - changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New))); - } - - var builder = all.ToBuilder(); - builder.RemoveRange(update.removed.Select(key)); - builder.RemoveRange(changed.Select(ch => key(ch.Old.Item2))); - foreach (var a in added) { - builder.Add(key(a.Item2), a); - } - foreach (var ch in changed) { - builder.Add(key(ch.New.Item2), ch.New); - } - all = builder.ToImmutable(); - - observer.OnNext(new Update<(SecurityImmutable?, T), security_key>( - all.Values, added, changed, removed, i => key(i.Item2))); - }); - }); + public static IObservable> join( + this IObservable> source, SecurityStore store) { + return store.join(source); } } } From a5be24aa7a6557897af5ce82a6682df2a35be6ca Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 22 Jan 2019 17:25:56 -0500 Subject: [PATCH 22/54] SPARK-1731 Add Key function to SecurityStore --- src/client/spark/Security/SecurityStore.cs | 51 +++++++++++----------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 5e424beac..558c588f4 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -148,21 +148,22 @@ namespace bts { Disposable.dispose(ref _dispose); } - public IObservable> join( - IObservable> source) { - return Observable.Create>(observer => { - var state = ImmutableDictionary.Empty; + public IObservable> join( + IObservable> source, Func skeyfn) where TKey : IEquatable { + return Observable.Create>(observer => { + var state = ImmutableDictionary.Empty; var gate = new object(); - Func keyfn = null; + // We get thie function from the first source update + Func<(SecurityImmutable?,T t), TKey> rowkeyfn = null; void on_securities_updated(object ambiguous) { + if (rowkeyfn == null) return; lock (gate) { // If this is null we don't have any rows - if (keyfn == null) return; var changed = new List>(); foreach (var entry in state.Values) { if (entry.s == null) { - var security = try_get_local(keyfn(entry.t), ambiguous as IReadOnlyDictionary); + var security = try_get_local(skeyfn(entry.t), ambiguous as IReadOnlyDictionary); if (security.HasValue) { changed.Add( new Change<(SecurityImmutable?, T)>(entry, (security, entry.t))); @@ -173,17 +174,17 @@ namespace bts { // update state var builder = state.ToBuilder(); foreach (var up in changed) { - builder[up.New.s.Value.Id] = up.New; + builder[rowkeyfn(up.New)] = up.New; } state = builder.ToImmutable(); // fire update observer.OnNext( - new Update<(SecurityImmutable?, T), security_key>( + new Update<(SecurityImmutable?, T), TKey>( state.Values, Enumerable.Empty<(SecurityImmutable?,T)>(), changed, Enumerable.Empty<(SecurityImmutable?,T)>(), - i => keyfn(i.Item2))); + rowkeyfn)); } } } @@ -192,15 +193,15 @@ namespace bts { // Fire update when source changes source.Subscribe((update) => { lock (gate) { - keyfn = update.key_extractor; - var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changed).Select(keyfn)); + rowkeyfn = rowkeyfn ?? (i => update.key_extractor(i.Item2)); + var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changed).Select(skeyfn)); var added = new List<(SecurityImmutable?, T)>(); - var changed = new List>(); + var changed = new List>(); var removed = new List<(SecurityImmutable?, T)>(); foreach (var x in update.added) { - if (relevant_securities.TryGetValue(keyfn(x), out SecurityImmutable s)) { + if (relevant_securities.TryGetValue(skeyfn(x), out SecurityImmutable s)) { added.Add((s, x)); } else { @@ -208,30 +209,30 @@ namespace bts { } } foreach (var x in update.removed) { - if (state.TryGetValue(keyfn(x), out (SecurityImmutable?, T) value)) { + if (state.TryGetValue(update.key_extractor(x), out (SecurityImmutable?, T) value)) { removed.Add(value); } } foreach (var ch in update.changes) { SecurityImmutable? si = - relevant_securities.TryGetValue(keyfn(ch.New), out SecurityImmutable s) ? + relevant_securities.TryGetValue(skeyfn(ch.New), out SecurityImmutable s) ? s : (SecurityImmutable?)null; changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New))); } var builder = state.ToBuilder(); - builder.RemoveRange(update.removed.Select(keyfn)); - builder.RemoveRange(changed.Select(ch => keyfn(ch.Old.Item2))); + builder.RemoveRange(update.removed.Select(update.key_extractor)); + builder.RemoveRange(changed.Select(ch => rowkeyfn(ch.Old))); foreach (var a in added) { - builder.Add(keyfn(a.Item2), a); + builder.Add(rowkeyfn(a), a); } foreach (var ch in changed) { - builder.Add(keyfn(ch.New.Item2), ch.New); + builder.Add(rowkeyfn(ch.New), ch.New); } state = builder.ToImmutable(); - observer.OnNext(new Update<(SecurityImmutable?, T), security_key>( - state.Values, added, changed, removed, i => keyfn(i.Item2))); + observer.OnNext(new Update<(SecurityImmutable?, T t), TKey>( + state.Values, added, changed, removed, rowkeyfn)); } }), // Also fire update when securities change @@ -245,9 +246,9 @@ namespace bts { } public static class SecurityStoreExtensions { - public static IObservable> join( - this IObservable> source, SecurityStore store) { - return store.join(source); + public static IObservable> + join(this IObservable> source, SecurityStore store, Func skeyfn) where TKey : IEquatable { + return store.join(source, skeyfn); } } } From fd270a7ab57690790a319db9b93a79a3b82de03e Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 23 Jan 2019 08:59:43 -0500 Subject: [PATCH 23/54] Add the ability to *only* run sim from a replay file --- src/lib/config/protos/MarketData.proto | 5 +++-- src/server/sparkd/md/run_md.cpp | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index 90b204f74..861b1ba08 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -4,8 +4,9 @@ package bts.md; message MarketDataConfig { // Allows you to turn off in-proc market data entirely optional bool use_md = 1 [default = true]; - optional bool use_replay = 2 [default=false]; - optional bool use_sim = 3 [default=true]; + optional bool use_onixs = 2 [default=true]; + optional bool use_replay = 3 [default=false]; + optional bool use_sim = 4 [default=true]; } message PlatformMap diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index c3a9e04d4..0a12bce66 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -79,10 +79,11 @@ void run_cleanup() { std::thread* _md_thread; void run_market_data_publishers_forever() { os::set_thread_name("md publisher"); - if (config::get().use_replay()) { + const auto& conf = config::get(); + if (conf.use_replay()) { run_replay(); } - else { + else if (conf.use_onixs()) { run_onixs(); } } From be13b812835205f3d5b84309c82580cd4d85a6ea Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 23 Jan 2019 11:35:13 -0500 Subject: [PATCH 24/54] db_dump needs stoul to handle SIM securitykeys --- src/server/sparkd/md/run_md.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index 0a12bce66..45806edb6 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -227,9 +227,9 @@ void dump_db(const vector& args) { if (arg == "all") { all = true; break; } - auto key = std::stoi(arg); + auto key = std::stoul(arg); if (key > 0) { - keys.insert(key); + keys.insert((uint)key); } } std::set symbols; From 90dd34945451bb531b5ad43fa65a31cfe03955c0 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 23 Jan 2019 16:18:52 -0500 Subject: [PATCH 25/54] SPARK-1973 1/2 use join for OrdersModel.on_order --- .../spark/Orders/OrderActivitySummarizer.cs | 46 ++++++++++--------- src/client/spark/Orders/OrdersModel.cs | 41 +++++++++-------- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/client/spark/Orders/OrderActivitySummarizer.cs b/src/client/spark/Orders/OrderActivitySummarizer.cs index dd30e5962..3bcb6de13 100644 --- a/src/client/spark/Orders/OrderActivitySummarizer.cs +++ b/src/client/spark/Orders/OrderActivitySummarizer.cs @@ -11,6 +11,7 @@ using System.Linq; using System.Reactive.Linq; using System.Text; using System.Threading.Tasks; +using OrderPair = System.ValueTuple; namespace bts.spark { internal class Activity { @@ -26,7 +27,7 @@ namespace bts.spark { static readonly IEqualityComparer _id_comparer = bts.utils.EqualityComparer.Create(a => a.Key); - public OrderActivitySummarizer(IBus bus, IObservable> orders) { + public OrderActivitySummarizer(IBus bus, IObservable> orders) { _bus = bus; _dispose = Disposable.Create( bus.subscribe(SparkTags.SecuritiesTag).Subscribe(sec => _security = sec), @@ -38,11 +39,11 @@ namespace bts.spark { #region formatting - (SecurityImmutable?, ContractSpecificationImmutable?) get_security(uint security_key) { + SecurityImmutable? try_get_security(uint security_key) { if (_security.TryGetValue(security_key, out SecurityMD md)) { - return (md.security, md.security.ContractSpec); + return md.security; } - return (null, null); + return null; } public static string format_order_success(uint key, double price, uint size, Side side, ContractSpecificationImmutable? spec) { return $"({key}) {side.ToString().ToLower()} {size} @ {spec.format(price)}"; @@ -123,24 +124,23 @@ namespace bts.spark { #endregion - void orders_updated(TableUpdate update) { + void orders_updated(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) { - foreach (var order in update.added) { - if (!string.IsNullOrEmpty(order.State.LastMsg) && order.State.Created.ToDateTime().Date == DateTime.Today) { - (var security, var spec) = get_security(order.Def.SecurityKey); - if (security.HasValue) { - log_status(ActivityLogCategory.warn, security, order.State.LastMsg); + foreach (var o in update.added) { + if (!string.IsNullOrEmpty(o.order.State.LastMsg) && o.order.State.Created.ToDateTime().Date == DateTime.Today) { + if (o.sec.HasValue) { + log_status(ActivityLogCategory.warn, o.sec.Value, o.order.State.LastMsg); } } } foreach (var order in update.changes) { - var neworder = order.New; - var oldorder = order.Old; - (var security, var spec) = get_security(neworder.Def.SecurityKey); + var neworder = order.New.order; + var oldorder = order.Old.order; + var security = order.New.sec ?? order.Old.sec; // // description - var mod_desc = modified_description(oldorder, neworder, spec); + var mod_desc = modified_description(oldorder, neworder, security?.ContractSpec); if (!string.IsNullOrEmpty(mod_desc)) { log_status(ActivityLogCategory.modify, security, mod_desc); } @@ -171,28 +171,30 @@ namespace bts.spark { } internal void on_send_order(OrderDefinitionImmutable order, double price, uint size, uint? key, string error) { - (var security, var spec) = get_security(order.SecurityKey); + var security = try_get_security(order.SecurityKey); log_status(ActivityLogCategory.order, security, key.HasValue ? - format_order_success(key.Value, price, size, order.Side, spec) : - format_order_fail(price, size, order.Side, spec, error)); + format_order_success(key.Value, price, size, order.Side, security?.ContractSpec) : + format_order_fail(price, size, order.Side, security?.ContractSpec, error)); } internal void on_cancel_order(OrderRecordImmutable order, bool success, string error) { - (var security, var spec) = get_security(order.Def.SecurityKey); + var security = try_get_security(order.Def.SecurityKey); log_status(ActivityLogCategory.cancel, security, success ? - format_cancel_success(order, spec): - format_cancel_error(order, spec, error)); + format_cancel_success(order, security?.ContractSpec): + format_cancel_error(order, security?.ContractSpec, error)); } internal void update_order(OrderRecordImmutable order, double price, uint size) { - (var security, var spec) = get_security(order.Def.SecurityKey); + var security = try_get_security(order.Def.SecurityKey); + var spec = security?.ContractSpec; log_status(ActivityLogCategory.modify, security, $"({order.Key}) change from {order.State.Quantity}@{spec.format(order.State.Price)} to {size}@{spec.format(price)}"); } internal void update_order_failed(OrderRecordImmutable order, double price, uint size) { - (var security, var spec) = get_security(order.Def.SecurityKey); + var security = try_get_security(order.Def.SecurityKey); + var spec = security?.ContractSpec; log_status(ActivityLogCategory.modify, security, $"({order.Key}) failed to change from {order.State.Quantity}@{spec.format(order.State.Price)} to {size}@{spec.format(price)}"); diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 910ff14e6..2eab7f64a 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -16,6 +16,8 @@ using System.Linq; using System.Reactive.Concurrency; using System.Reactive.Linq; using System.Threading.Tasks; +using OrderPair = System.ValueTuple; +using OrderUpdate = bts.api.core.TableUpdate<(bts.api.SecurityImmutable? sec, bts.api.orderman.OrderRecordImmutable order), uint>; namespace mbts.spark { [Export] @@ -50,22 +52,24 @@ namespace mbts.spark { .subscribei_orders() .Naggle(interval, scheduler) .PermissionFilter(admin_model.on_admin_data) - //.join(sstore) + .join(sstore, order => order.security_key()) .Table().Cache(); on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); working_orders = on_orders.Scan( - ImmutableDictionary.Empty, + ImmutableDictionary.Empty, (workers, table) => _update_working_orders(workers, table)) .Replay(1).RefCount(); var order_record_tuple_source = new OrderRecordTupleSource(bus.subscribe(SparkTags.SecuritiesTag), on_orders); on_order_tuples = order_record_tuple_source.subscribe_incremental_update(); on_order_tuples_table = order_record_tuple_source.subscribe_incremental_table_update(); on_order_securities = order_record_tuple_source.subscribe_order_securities(); + on_fills = _service.subscribei_fills() .Naggle(interval, scheduler) .PermissionFilter(admin_model.on_admin_data) + //.join(sstore, f => ) .Table().Cache(); _activity = new OrderActivitySummarizer(bus, on_orders); on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders, on_fills); @@ -87,13 +91,13 @@ namespace mbts.spark { public OrderFactory factory => _factory; public SpreaderModel spreader => _spreader; public DynamicOrderHelper dynamic_helper => _dynamic_helper; - public IObservable> on_orders { get; } + public IObservable on_orders { get; } public IObservable> on_order_tuples { get; } public IObservable> on_order_tuples_table { get; } public IObservable> on_order_securities { get; } public IObservable> on_fills { get; } public IObservable> on_fill_tuples { get; } - public IObservable> working_orders { get; } + public IObservable> working_orders { get; } public IObservable> on_positions { get; } public IObservable> on_security_orders(uint security) => @@ -164,24 +168,25 @@ namespace mbts.spark { public void Dispose() => Disposable.dispose(ref _dispose); - static ImmutableDictionary _update_working_orders(ImmutableDictionary target, TableUpdate table) { + static ImmutableDictionary + _update_working_orders(ImmutableDictionary target, OrderUpdate table) { var clone = target.ToBuilder(); foreach (var o in table.removed) { - if (o.is_active()) { - clone.Remove(o.Key); + if (o.order.is_active()) { + clone.Remove(o.order.Key); } } foreach (var o in table.changed) { - if (o.is_active()) { - clone[o.Key] = o; + if (o.order.is_active()) { + clone[o.order.Key] = o; } else { - clone.Remove(o.Key); + clone.Remove(o.order.Key); } } foreach (var o in table.added) { - if (o.is_active()) { - clone.Add(o.Key, o); + if (o.order.is_active()) { + clone.Add(o.order.Key, o); } } return clone.ToImmutableDictionary(); @@ -196,8 +201,8 @@ namespace mbts.spark { public static async Task cancel_order(this OrdersModel orders_model, uint order_id) { var tu = await orders_model.on_orders.FirstAsync(); - if (tu.state.TryGetValue(order_id, out OrderRecordImmutable order)) { - return await orders_model.cancel_order(order); + if (tu.state.TryGetValue(order_id, out var o)) { + return await orders_model.cancel_order(o.order); } return false; } @@ -206,8 +211,8 @@ namespace mbts.spark { var tu = await model.on_orders.FirstAsync(); var orders = tu.state.Values - .Where(o => o.is_non_terminal() && pred(o)) - .Select(async o => { return (o, await model.cancel_order(o)); } ); + .Where(o => o.order.is_non_terminal() && pred(o.order)) + .Select(async o => { return (o.order, await model.cancel_order(o.order)); } ); return await Task.WhenAll(orders); } @@ -315,8 +320,8 @@ namespace mbts.spark { public static async Task order_from_key(this OrdersModel orders_model, uint order_key) { (var tu, var success) = await orders_model.on_orders.FirstAsyncTask(t => t.state.ContainsKey(order_key), TimeSpan.FromSeconds(3)); - if (success && tu.state.TryGetValue(order_key, out OrderRecordImmutable rec)) { - return rec; + if (success && tu.state.TryGetValue(order_key, out var rec)) { + return rec.order; } return null; } From 9bcd8bcbdbda1debdf1913b5e94dbfab5c814ef6 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 24 Jan 2019 15:38:39 -0500 Subject: [PATCH 26/54] Add tests for IObservable.Select methods --- src/client/spark_api/api/Update.cs | 38 ++++ src/client/spark_test/api/ApiUpdateTests.cs | 193 +++++++++++++++----- 2 files changed, 184 insertions(+), 47 deletions(-) diff --git a/src/client/spark_api/api/Update.cs b/src/client/spark_api/api/Update.cs index d5f1b04fd..b2585f8ad 100644 --- a/src/client/spark_api/api/Update.cs +++ b/src/client/spark_api/api/Update.cs @@ -64,6 +64,10 @@ namespace bts.api.core { public Change merge(Change newer) { return new Change(Old, newer.New); } + public void Deconstruct(out T Old, out T New) { + Old = this.Old; + New = this.New; + } public override string ToString() { return $"{{{Old}=>{New}}}"; } @@ -415,5 +419,39 @@ namespace bts.api.core { changed = diffs; removed = rems; } + + // This method does everything lazily so it's cheap until the collections are enumerated + public static IObservable> + Select(this IObservable> src, Func map, Func key) where TKey : IEquatable { + return src.Select(update => { + return new Update( + update.all.Select(map), + update.added.Select(map), + update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))), + update.removed.Select(map), + key); + }); + } + + // Constructs a new dictionary so not as cheap as Select on the Update<> type + public static IObservable> + Select(this IObservable> src, Func map, Func key) where TKey : IEquatable { + var cache = ImmutableDictionary.Empty; + return src.Select(update => { + var added = update.added.Select(map); + var removed = update.removed.Select(map); + var changes = update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))); + var builder = cache.ToBuilder(); + builder.RemoveRange(removed.Select(key)); + builder.AddRange(added.Select(t => new KeyValuePair(key(t), t))); + foreach (var (o,n) in changes) { + Debug.Assert(key(o).Equals(key(n))); + builder[key(o)] = n; + } + cache = builder.ToImmutable(); + + return new TableUpdate(cache, added, changes, removed); + }); + } } } \ No newline at end of file diff --git a/src/client/spark_test/api/ApiUpdateTests.cs b/src/client/spark_test/api/ApiUpdateTests.cs index 791d9a773..e1f6f08b7 100644 --- a/src/client/spark_test/api/ApiUpdateTests.cs +++ b/src/client/spark_test/api/ApiUpdateTests.cs @@ -1,12 +1,12 @@ using bts.api.core; -using System.Collections.Generic; using bts.test; using System; -using System.Reactive.Linq; -using System.Threading.Tasks; +using System.Collections.Generic; using System.Linq; using System.Reactive.Concurrency; +using System.Reactive.Linq; using System.Reactive.Subjects; +using System.Threading.Tasks; namespace mbts.api.Test { [TestGroup] @@ -44,7 +44,7 @@ namespace mbts.api.Test { static int id(Item i) => i.id; Item mkitem() { - return new Item(++_id, Guid.NewGuid().ToString().Substring(0,8)); + return new Item(++_id, Guid.NewGuid().ToString().Substring(0, 8)); } IObservable> validate_incremental(IObservable> source) { @@ -68,7 +68,7 @@ namespace mbts.api.Test { // This will mostly be collapsing to add() for (var count = 1; count < total; ++count) { var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last(); - var nag = await + var nag = await validate_incremental( raw .Take(count) @@ -80,7 +80,7 @@ namespace mbts.api.Test { // Do it again with TableUpdate for (var count = 1; count < total; ++count) { var orig = raw.Take(count).reconstitute(id).Table().ToEnumerable().Last(); - var nag = await + var nag = await validate_incremental( raw .Take(count) @@ -94,35 +94,32 @@ namespace mbts.api.Test { // collapse rolling window var update_list = raw.reconstitute(id).ToEnumerable().ToList(); var window = 10; - for (int x = 0; x+window < total; x += 5) { + for (int x = 0; x + window < total; x += 5) { var start = update_list[x]; var end = update_list[x + window]; - var super_update = update_list.Skip(x).Take(window+1).Aggregate((u1, u2) => u1.merge(u2)); + var super_update = update_list.Skip(x).Take(window + 1).Aggregate((u1, u2) => u1.merge(u2)); start.merge(super_update); Assert.SequenceEqual(end.all, super_update.all); } } - void validate_filter(IObservable> raw, IObservable> filtered, Func pred) - { + void validate_filter(IObservable> raw, IObservable> filtered, Func pred) { var total = raw.ToEnumerable().Count(); // Validate that update is right at each point - for (var count = 1; count < total; ++count) - { - var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last(); + for (var count = 1; count < total; ++count) { + var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last(); var next = filtered.Take(count).ToEnumerable().Last(); Assert.That(next.all.All(pred)); Assert.That(next.added.All(pred)); Assert.That(next.removed.All(pred)); - Assert.That(next.changed.All(pred)); + Assert.That(next.changed.All(pred)); } // Do it again with TableUpdate - for (var count = 1; count < total; ++count) - { + for (var count = 1; count < total; ++count) { var orig = raw.Take(count).reconstitute(id).Table().ToEnumerable().Last(); var next = filtered.Take(count).ToEnumerable().Last(); Assert.That(next.added.All(pred)); @@ -132,36 +129,32 @@ namespace mbts.api.Test { } - void validate_filter_sub(IObservable> raw, IObservable> filtered, BehaviorSubject> pred) - { - raw.reconstitute(x => x.id).CombineLatest(filtered, (r, f) => Tuple.Create(r, f)).Subscribe(tup => - { + void validate_filter_sub(IObservable> raw, IObservable> filtered, BehaviorSubject> pred) { + raw.reconstitute(x => x.id).CombineLatest(filtered, (r, f) => Tuple.Create(r, f)).Subscribe(tup => { Assert.That(tup.Item2.all.All(pred.Value)); Assert.That(tup.Item2.added.All(pred.Value)); - Assert.That(!tup.Item2.removed.Any(pred.Value)); + Assert.That(!tup.Item2.removed.Any(pred.Value)); Assert.That(tup.Item2.changed.All(pred.Value)); Dictionary rawd = tup.Item1.added.ToDictionary(x => x.id); Dictionary fd = tup.Item2.added.ToDictionary(x => x.id); - foreach (var kv in rawd) - { - if(pred.Value(kv.Value)) + foreach (var kv in rawd) { + if (pred.Value(kv.Value)) { Assert.That(fd.ContainsKey(kv.Key)); - else + } + else { Assert.That(!fd.ContainsKey(kv.Key)); + } } }); } [Test] - public void test_filter() - { - var raw = Observable.Create>(observer => - { + public void test_filter() { + var raw = Observable.Create>(observer => { RawUpdate rup = new RawUpdate(); - for (int x = 0; x < 50; ++x) - { + for (int x = 0; x < 50; ++x) { rup.Clear(); var item = _item_pool[x]; rup.added_items.Add(item); @@ -169,39 +162,35 @@ namespace mbts.api.Test { observer.OnNext(rup); } var rand = new Random(); - for (int r = 0; r < rand.Next(4, 30); r += rand.Next(1, 2)) - { + for (int r = 0; r < rand.Next(4, 30); r += rand.Next(1, 2)) { rup.Clear(); rup.removed_keys.Add(_item_pool[r].id); observer.OnNext(rup); } rup.Clear(); - for (int c = 31; c < rand.Next(32, 50); c += rand.Next(1, 2)) - { + for (int c = 31; c < rand.Next(32, 50); c += rand.Next(1, 2)) { var ch = new Item(_item_pool[c].id, _item_pool[c].name + "_change"); rup.changed_items.Add(ch); - rup.changed_keys.Add(ch.id); + rup.changed_keys.Add(ch.id); } observer.OnNext(rup); - observer.OnCompleted(); + observer.OnCompleted(); return Disposable.empty; }); Func evens = (x => x.id % 2 == 0); Func odds = (x => x.id % 2 == 1); - var predeven = Observable.Create>(observer => - { - observer.OnNext(evens); + var predeven = Observable.Create>(observer => { + observer.OnNext(evens); observer.OnCompleted(); return Disposable.empty; }); - var filtered = raw.reconstitute(x=> x.id).Filter(predeven); + var filtered = raw.reconstitute(x => x.id).Filter(predeven); validate_filter(raw, filtered, evens); - var predodd = Observable.Create>(observer => - { + var predodd = Observable.Create>(observer => { observer.OnNext(odds); observer.OnCompleted(); return Disposable.empty; @@ -212,13 +201,13 @@ namespace mbts.api.Test { - var sub= new BehaviorSubject>(evens); + var sub = new BehaviorSubject>(evens); filtered = raw.reconstitute(x => x.id).Filter(sub); validate_filter_sub(raw, filtered, sub); sub.OnNext(odds); sub.OnCompleted(); - } + } [Test] public async Task test_add() { @@ -249,7 +238,7 @@ namespace mbts.api.Test { rup.added_keys.AddRange(_item_pool.Take(20).Select(id)); observer.OnNext(rup); - for (int x = 0; x < 50; x+=2) { + for (int x = 0; x < 50; x += 2) { rup.Clear(); var item = _item_pool[x]; rup.removed_keys.Add(item.id); @@ -347,11 +336,121 @@ namespace mbts.api.Test { public void test_merge_begin_end_state() { int key(string s) => s[0]; var empty = new string[0]; - var a2 = new Update(new[] { "a2" }, empty, new[] { new Change("a1", "a2") }, empty, key); + var a2 = new Update(new[] { "a2" }, empty, new[] { new Change("a1", "a2") }, empty, key); var a3 = new Update(new[] { "a3" }, empty, new[] { new Change("a2", "a3") }, empty, key); var merged = a2.merge(a3); Assert.SequenceEqual(new[] { new Change("a1", "a3") }, merged.changes); } + T id(T t) => t; + + [Test,Ignore("SPARK-1988")] + public void test_reconstitute_twice() { + var empty = new int[0]; + RawUpdate create(int val) => new RawUpdate() { added_items = { val }, added_keys = { val } }; + var input = + Enumerable.Range(1, 10) + .Select(create) + .ToObservable() + .reconstitute(id); + + var expected = Enumerable.Range(1, 10).Select(i => Enumerable.Range(1, i)).SelectMany(id); + var a = input.ToEnumerable().SelectMany(u => u.all).ToList(); + var b = input.ToEnumerable().SelectMany(u => u.all).ToList(); + Assert.SequenceEqual(expected, a); + Assert.SequenceEqual(expected, b); + } + + [Test] + public void test_select_update() { + var empty = new int[0]; + RawUpdate create(int val) => new RawUpdate() { added_items = { val }, added_keys = { val } }; + int square(int i) => i * i; + var input = + Enumerable.Range(1, 10) + .Select(create) // IEnumerable + .ToObservable() // IObservable + .reconstitute(id); // IObservable + + var output = + input + .Select(square, id) // function under test + .ToEnumerable() + .ToList(); //REMOVE THIS WHEN SPARK-1988 is fixed. + + var expected = Enumerable.Range(1, 10).Select(square); + Assert.SequenceEqual(expected, output.SelectMany(u => u.added)); + + var expected_all = Enumerable.Range(1, 10).Select(i => Enumerable.Range(1, i).Select(square)).SelectMany(id); + var accumulated = output.SelectMany(u => u.all).ToList(); + Assert.SequenceEqual(expected_all, accumulated); + } + + [Test] + public void test_select_tableupdate() { + var empty = new int[0]; + RawUpdate create(int val) => new RawUpdate() { added_items = { val }, added_keys = { val } }; + int square(int i) => i * i; + var input = + Enumerable.Range(1, 10) + .Select(create) // IEnumerable + .ToObservable() // IObservable + .reconstitute(id); // IObservable + + var output = + input + .Table() + .Select(square, id) // function under test + .ToEnumerable() + .ToList(); //REMOVE THIS WHEN SPARK-1988 is fixed. + + var expected = Enumerable.Range(1, 10).Select(square); + Assert.SequenceEqual(expected, output.SelectMany(u => u.added)); + + var expected_all = Enumerable.Range(1, 10).Select(i => Enumerable.Range(1, i).Select(square)).SelectMany(id); + var accumulated = output.SelectMany(u => u.state.Values).ToList(); + Assert.SequenceEqual(expected_all, accumulated); + + } + + //public class Box { + // public int value; + // public override string ToString() { + // return value.ToString(); + // } + //} + + //[Test] + //public void test_scan() { + // var obs = + // Enumerable.Range(1, 5) + // .ToObservable() + // .Scan( + // new Box { value = 0 }, + // (box, i) => { + // box.value += i; + // return box; + // }); + // var a = obs.Select(box => box.value).ToEnumerable(); + // var b = obs.Select(box => box.value).ToEnumerable(); + // Assert.SequenceEqual(a, b); + //} + + //[Test] + //public void test_scan2() { + // var obs = + // Enumerable.Range(1, 5) + // .ToObservable() + // .Scan>( + // () => new Box { value = 0 }, + // (getbox, i) => { + // var box = getbox(); + // box.value += i; + // return () => box; + // }); + // var a = obs.Select(box => box().value).ToEnumerable().ToList(); + // var b = obs.Select(box => box().value).ToEnumerable().ToList(); + // Assert.SequenceEqual(a, b); + //} } } From b08bc0eb8b5f56c97deade9aae9f91bec8d4188e Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 25 Jan 2019 13:40:56 -0500 Subject: [PATCH 27/54] SPARK-1972 Join Securities to Orders in OrdersModel --- .../spark/Daemons/DeleteSyntheticDaemon.cs | 2 +- src/client/spark/Daemons/OrderTicketDaemon.cs | 20 +-- .../Trading/OrderBook/OrderBookColumns.cs | 86 +++++----- .../OrderBook/OrderBookRowCollection.cs | 24 +-- .../Trading/OrderBook/OrderBookRowVM.cs | 19 +-- .../Modules/Trading/OrderBook/OrderBookVM.cs | 17 +- .../Trading/OrderBook/OrderBookWindow.cs | 8 +- .../OrderBook/OrderBookWindowFactory.cs | 4 +- .../Notifications/INotificationService.cs | 28 ++-- .../INotificationServiceDataCache.cs | 33 ++-- .../spark/Notifications/ToastNotification.cs | 8 +- .../spark/Orders/ExistingOrderTicketVM.cs | 2 +- src/client/spark/Orders/FillSecurity.cs | 25 +++ src/client/spark/Orders/OrderRecordTuple.cs | 47 ------ .../spark/Orders/OrderRecordTupleSource.cs | 147 ------------------ src/client/spark/Orders/OrderSecurity.cs | 25 +++ src/client/spark/Orders/OrdersModel.cs | 29 ++-- .../Events/OrderCanceledEventPredicate.cs | 19 +-- .../Events/OrderCompletedEventPredicate.cs | 14 +- .../Data/Events/OrderEventPredicateHelper.cs | 24 +-- .../Data/Events/OrderFilledEventPredicate.cs | 47 +++--- .../Data/OrderRecordTuplePredicate.cs | 24 +-- src/client/spark/Security/SecurityStore.cs | 3 +- .../core/GlobalExtensions/GlobalExtensions.cs | 1 + .../core/Utility/TableUpdateTransform.cs | 39 ++++- .../DIYGrid/Settings/GridSettingsManager.cs | 3 +- .../spark/view/Spark/Core/SparkUtils.cs | 11 +- .../spark/view/Spark/Ladder/LadderVM.cs | 17 +- src/client/spark_api/api/Update.cs | 14 +- .../Extensions/DictionaryExtensions.cs | 23 +++ .../spark_core/utils/FilteredDictionary.cs | 86 ++++++++++ .../spark_core/utils/MappedDictionary.cs | 64 ++++++++ 32 files changed, 508 insertions(+), 405 deletions(-) create mode 100644 src/client/spark/Orders/FillSecurity.cs delete mode 100644 src/client/spark/Orders/OrderRecordTuple.cs delete mode 100644 src/client/spark/Orders/OrderRecordTupleSource.cs create mode 100644 src/client/spark/Orders/OrderSecurity.cs create mode 100644 src/client/spark_core/utils/FilteredDictionary.cs create mode 100644 src/client/spark_core/utils/MappedDictionary.cs diff --git a/src/client/spark/Daemons/DeleteSyntheticDaemon.cs b/src/client/spark/Daemons/DeleteSyntheticDaemon.cs index 15bc8d00c..52819f4e3 100644 --- a/src/client/spark/Daemons/DeleteSyntheticDaemon.cs +++ b/src/client/spark/Daemons/DeleteSyntheticDaemon.cs @@ -32,7 +32,7 @@ namespace bts { if (!orders_success) { _error_reporter.report_error($"Unable to delete synthetic security ({id}): unable to determine whether or not security ({id}) has working orders (deletion is not permitted while there are working orders)."); Log.Ui.warn($"Ignoring request to delete synthetic security ({id}) because the orders lookup timed out (must confirm no working orders)."); - } else if (orders.state.Values.Any(o => o.Def.SecurityKey == id && o.is_non_terminal())) { + } else if (orders.state.Values.Any(o => o.order.Def.SecurityKey == id && o.order.is_non_terminal())) { Log.Ui.warn($"Ignoring request to delete synthetic security ({id}) because non-terminal-orders were detected."); } else { ExecutionStrategyImmutable? strategy = await _orders_model.spreader.lookup_strategy_async(id); diff --git a/src/client/spark/Daemons/OrderTicketDaemon.cs b/src/client/spark/Daemons/OrderTicketDaemon.cs index f0dfc0583..5b8ac95c3 100644 --- a/src/client/spark/Daemons/OrderTicketDaemon.cs +++ b/src/client/spark/Daemons/OrderTicketDaemon.cs @@ -19,7 +19,7 @@ namespace bts.shell { readonly OrdersModel _model; readonly ErrorReporter _error_reporter; IReadOnlyDictionary _md_map; - TableUpdate _orders; + TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> _orders; IDisposable _dispose; [ImportingConstructor] @@ -57,14 +57,16 @@ namespace bts.shell { } void _show_modify_order(uint order_key) { - if (_orders.state.TryGetValue(order_key, out OrderRecordImmutable ori) && _md_map.TryGetValue(ori.Def.SecurityKey, out SecurityMD smd) - && (ori.HasDef && ori.Def.HasCustomDef && !(ori.Def.CustomDef.HasStop || ori.Def.CustomDef.HasStopLimit))) - _show_order_window(new OrderModificationDialog { - DataContext = new ExistingOrderTicketVM(_model, - Observable.Return(new OrderRecordTuple(ori, smd)), - Observable.Return(_model.factory.default_ladder_settings(smd.security).calculator_buttons) - ) - }); + if (_orders.state.TryGetValue(order_key, out var ori) && ori.sec.HasValue + && (ori.order.HasDef && ori.order.Def.HasCustomDef && !(ori.order.Def.CustomDef.HasStop || ori.order.Def.CustomDef.HasStopLimit))) { + var md = _md_map.TryGetValue(ori.order.security_key()); + _show_order_window(new OrderModificationDialog { + DataContext = new ExistingOrderTicketVM(_model, + Observable.Return((md,ori.order)), + Observable.Return(_model.factory.default_ladder_settings(ori.sec.Value).calculator_buttons) + ) + }); + } } } } \ No newline at end of file diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs index acb550f67..209416179 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs @@ -30,7 +30,7 @@ namespace mbts.spark { interface ICVOrderBookColumn : IKeyedColumn, IGridColumn { string sort_member_path { get; } ListSortDirection? SortDirection { get; set; } - IDataGridColumnFilter filter { get; } + IDataGridColumnFilter filter { get; } IComparer comparer { get; } } @@ -65,9 +65,9 @@ namespace mbts.spark { public OrderBookColumnCollection(IBus bus, OrdersModel model, IObservable<(IEnumerable added, IEnumerable removed)> order_securities) { // security Add(security_column = new SecurityColumn( - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( order_securities, - ort => ort.security_md.HasValue ? ort.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable, + ort => ort.security ?? SecurityDataHelper.Defaults.SecurityImmutable, SecurityCriteria.from_securities, SecurityCriteria.get_to_securities(bus), nameof(OrderBookRowVM.security), @@ -81,9 +81,9 @@ namespace mbts.spark { nameof(OrderBookRowVM.symbol), r => r.symbol, DefaultColumnWidths.Symbol, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( mbts.co2.UserSymbol.get_all().Select(u => u.symbol).Distinct().ToList(), - ort => ort.security_md.HasValue ? ort.security_md.Value.security.Symbol : string.Empty, + ort => ort.security?.Symbol ?? string.Empty, nameof(OrderBookRowVM.symbol) ), DataGridFilterKnownTemplates.DiscreteValues, @@ -95,7 +95,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.time_created), r => r.time_created_text, DefaultColumnWidths.Time, - new TimeFilterVM( + new TimeFilterVM( ort => ort.order.created(), nameof(OrderBookRowVM.time_created)), DataGridFilterKnownTemplates.TimeFilter, @@ -107,7 +107,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.last_change), r => r.last_change_text, DefaultColumnWidths.Time, - new TimeFilterVM( + new TimeFilterVM( ort => ort.order.last_change(), nameof(OrderBookRowVM.last_change)), DataGridFilterKnownTemplates.TimeFilter, @@ -119,7 +119,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.user), r => r.user, DefaultColumnWidths.User, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( model.factory.subscribe_users().Select(d => d.Values), ort => ort.order.user(), nameof(OrderBookRowVM.user)), @@ -132,7 +132,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.account), r => r.account, DefaultColumnWidths.Account, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( model.factory.subscribe_accounts().Select(d => d.Values), ort => ort.order.account(), vals => vals.ToArray(), @@ -147,9 +147,9 @@ namespace mbts.spark { nameof(OrderBookRowVM.exchange), r => r.exchange.ToString(), DefaultColumnWidths.Exchange, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( EnumHelper.get_values_except(Venue.Unset).ToList(), - ort => ort.security_md.HasValue ? ort.security_md.Value.security.Exchange : Venue.Unset, + ort => ort.security.exchange_or_default(), nameof(OrderBookRowVM.exchange)), DataGridFilterKnownTemplates.EnumValues, _wrap_comparer(r => r.exchange))); @@ -160,7 +160,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.time_in_force), r => r.time_in_force.ToString(), DefaultColumnWidths.TimeInForce, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( EnumHelper.get_values().ToList(), ort => ort.order.time_in_force(), nameof(OrderBookRowVM.time_in_force)), @@ -168,7 +168,7 @@ namespace mbts.spark { _wrap_comparer(r => r.time_in_force))); // order type Add(order_type_column = new OrderBookOrderTypeColumn( - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( utilities.OrderFactory.all_order_types, ort => ort.order.order_type(), nameof(OrderBookRowVM.order_type)))); @@ -179,7 +179,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.order_sub_type), r => r.order_sub_type_text, 80, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( EnumHelper.get_values().ToList(), ort => ort.order.sub_type(), nameof(OrderBookRowVM.order_sub_type)), @@ -187,16 +187,16 @@ namespace mbts.spark { _wrap_comparer(r => r.order_sub_type))); // side Add(side_column = new SideColumn( - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( new[] { Side.Buy, Side.Sell }, ort=> ort.order.side(), nameof(OrderBookRowVM.side)))); // quantity Add(qty_column = new SizeColumn( - new UIntMinMax(ort => ort.order.quantity(), nameof(OrderBookRowVM.quantity)))); + new UIntMinMax(ort => ort.order.quantity(), nameof(OrderBookRowVM.quantity)))); // filled quantity Add(filled_qty_column = new FilledColumn( - new UIntMinMax(ort => ort.order.filled(), nameof(OrderBookRowVM.filled_quantity)))); + new UIntMinMax(ort => ort.order.filled(), nameof(OrderBookRowVM.filled_quantity)))); // price Add(price_column = new SimpleTextColumn( Columns.Price, @@ -204,7 +204,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.price), r => r.price_text, DefaultColumnWidths.Price, - new DoubleMinMax( + new DoubleMinMax( ort => ort.order.price(), nameof(OrderBookRowVM.price)), DataGridFilterKnownTemplates.DoubleMinMax, @@ -216,7 +216,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.avg_price), r => r.avg_price_text, DefaultColumnWidths.Price, - new DoubleMinMax( + new DoubleMinMax( ort => ort.order.avg_price(), nameof(OrderBookRowVM.avg_price)), DataGridFilterKnownTemplates.DoubleMinMax, @@ -228,7 +228,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.last_price), r => r.last_price_text, DefaultColumnWidths.Price, - new DoubleMinMax( + new DoubleMinMax( ort => ort.order.last_price(), nameof(OrderBookRowVM.last_price)), DataGridFilterKnownTemplates.DoubleMinMax, @@ -240,7 +240,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.status), r => r.status.ToString(), 80, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( EnumHelper.get_values().ToList(), ort => ort.order.status(), nameof(OrderBookRowVM.status)), @@ -248,7 +248,7 @@ namespace mbts.spark { _wrap_comparer(r => r.status))); // last message Add(last_message_column = new OrderBookLastMessageColumn( - new RegexFilterVM( + new RegexFilterVM( ort => ort.order.last_msg(), nameof(OrderBookRowVM.last_msg)))); // security key @@ -258,7 +258,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.security_key), r => r.security_key.ToString(), DefaultColumnWidths.SecurityKey, - new RegexFilterVM( + new RegexFilterVM( ort => ort.order.security_key().ToString(), nameof(OrderBookRowVM.security_key)), DataGridFilterKnownTemplates.Regex, @@ -270,7 +270,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.order_key), r => r.order_key.ToString(), DefaultColumnWidths.OrderKey, - new RegexFilterVM( + new RegexFilterVM( ort => ort.order.Key.ToString(), nameof(OrderBookRowVM.order_key)), DataGridFilterKnownTemplates.Regex, @@ -282,7 +282,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.parent_order_key), r => r.parent_order_key.ToString(), DefaultColumnWidths.ParentKey, - new RegexFilterVM( + new RegexFilterVM( ort => ort.order.ParentKey.ToString(), nameof(OrderBookRowVM.parent_order_key)), DataGridFilterKnownTemplates.Regex, @@ -378,9 +378,9 @@ namespace mbts.spark { } } - public Func get_filter() { + public Func get_filter() { var active_filters = this.OfType().Where(c => c.filter.active.GetValueOrDefault()).Select(c => c.filter).ToArray(); - return active_filters.Length == 0 ? null : new Func(r => active_filters.All(f => f.filter(r))); + return active_filters.Length == 0 ? null : new Func(r => active_filters.All(f => f.filter(r))); } public OrderBookSortingData get_sorting_data() { @@ -470,7 +470,7 @@ namespace mbts.spark { GlyphContext _root_glyph_context, _child_glyph_context; protected OrderbookColumnBase(Columns col_key, string header, Func get_text, string sort_prop, double width, - IDataGridColumnPersistentFilter filter, string filter_template, IComparer comparer) : base(col_key) { + IDataGridColumnPersistentFilter filter, string filter_template, IComparer comparer) : base(col_key) { id = key.ToString(); Width = width; Header = header; @@ -488,10 +488,10 @@ namespace mbts.spark { } } - public IDataGridColumnPersistentFilter filter { get; } + public IDataGridColumnPersistentFilter filter { get; } public IComparer comparer { get; } - IDataGridColumnFilter ICVOrderBookColumn.filter => filter; + IDataGridColumnFilter ICVOrderBookColumn.filter => filter; IDataGridColumnFilter IFilterableColumn.filter => filter; string IFilterableColumn.filter_template => _filter_template; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } @@ -505,7 +505,7 @@ namespace mbts.spark { public class SimpleTextColumn : OrderbookColumnBase, IRenderColumn { public SimpleTextColumn(Columns key, string header, string sort_prop, Func get_text, double width, - IDataGridColumnPersistentFilter filter, string filter_template, IComparer comparer) + IDataGridColumnPersistentFilter filter, string filter_template, IComparer comparer) : base(key, header, get_text, sort_prop, width, filter, filter_template, comparer) { } public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) => @@ -515,7 +515,7 @@ namespace mbts.spark { #region Explicit Column Defs public class SideColumn : OrderbookColumnBase, IRenderColumn { - public SideColumn(IDataGridColumnPersistentFilter filter) + public SideColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Side, "B/S", r => r.side.ToString(), nameof(OrderBookRowVM.side), DefaultColumnWidths.Side, filter, DataGridFilterKnownTemplates.EnumValues, _wrap_comparer(r => r.side)) { } public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) => @@ -524,7 +524,7 @@ namespace mbts.spark { } public class SizeColumn : OrderbookColumnBase, IRenderColumn { - public SizeColumn(IDataGridColumnPersistentFilter filter) + public SizeColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Size, "Qty", r => r.quantity.ToString(), nameof(OrderBookRowVM.quantity), DefaultColumnWidths.Size, filter, DataGridFilterKnownTemplates.UIntMinMax, _wrap_comparer(r => r.quantity)) { sort_member_path = nameof(OrderBookRowVM.quantity); } @@ -535,7 +535,7 @@ namespace mbts.spark { } public class FilledColumn : OrderbookColumnBase, IRenderColumn { - public FilledColumn(IDataGridColumnPersistentFilter filter) + public FilledColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Filled, "Filled Qty", r => r.filled_quantity.ToString(), nameof(OrderBookRowVM.filled_quantity), DefaultColumnWidths.Size, filter, DataGridFilterKnownTemplates.UIntMinMax, _wrap_comparer(r => r.filled_quantity)) { sort_member_path = nameof(OrderBookRowVM.filled_quantity); } @@ -547,7 +547,7 @@ namespace mbts.spark { public class OrderBookOrderTypeColumn : OrderTypeColumn, ICVOrderBookColumn { ListSortDirection? _cached_sort_direction; - public OrderBookOrderTypeColumn(IDataGridColumnPersistentFilter filter) + public OrderBookOrderTypeColumn(IDataGridColumnPersistentFilter filter) : base(Columns.OrderType, new Binding(nameof(OrderBookRowVM.order_type)), new Binding(nameof(OrderBookRowVM.order_metadata)), filter) { id = key.ToString(); this.filter = filter; @@ -556,10 +556,10 @@ namespace mbts.spark { _cached_sort_direction = SortDirection; } - public new IDataGridColumnPersistentFilter filter { get; } + public new IDataGridColumnPersistentFilter filter { get; } public IComparer comparer => _wrap_comparer(r => r.order_type); - IDataGridColumnFilter ICVOrderBookColumn.filter => filter; + IDataGridColumnFilter ICVOrderBookColumn.filter => filter; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } protected override void OnSortDirectionChanged(ListSortDirection? new_value) { @@ -571,7 +571,7 @@ namespace mbts.spark { public class OrderBookLastMessageColumn : TextColumn, ICVOrderBookColumn { ListSortDirection? _cached_sort_direction; - public OrderBookLastMessageColumn(IDataGridColumnPersistentFilter filter) + public OrderBookLastMessageColumn(IDataGridColumnPersistentFilter filter) : base(Columns.LastMessage, new Binding(nameof(OrderBookRowVM.last_msg)), filter, DataGridFilterKnownTemplates.Regex) { id = key.ToString(); this.filter = filter; @@ -581,10 +581,10 @@ namespace mbts.spark { tooltip_binding = new Binding(nameof(OrderBookRowVM.last_msg)); } - public new IDataGridColumnPersistentFilter filter { get; } + public new IDataGridColumnPersistentFilter filter { get; } public IComparer comparer => _wrap_comparer(r => r.last_msg); - IDataGridColumnFilter ICVOrderBookColumn.filter => filter; + IDataGridColumnFilter ICVOrderBookColumn.filter => filter; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } protected override void OnSortDirectionChanged(ListSortDirection? new_value) { @@ -600,7 +600,7 @@ namespace mbts.spark { ListSortDirection? _cached_sort_direction; - public SecurityColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Security) { + public SecurityColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Security) { id = key.ToString(); this.filter = filter; Header = DefaultColumnHeaders.Security; @@ -609,10 +609,10 @@ namespace mbts.spark { _cached_sort_direction = SortDirection; } - public IDataGridColumnPersistentFilter filter { get; } + public IDataGridColumnPersistentFilter filter { get; } public IComparer comparer => _wrap_comparer(r => r.security, SecurityHelper.Comparer.InstanceImmutable.to_nullable()); - IDataGridColumnFilter ICVOrderBookColumn.filter => filter; + IDataGridColumnFilter ICVOrderBookColumn.filter => filter; IDataGridColumnFilter IFilterableColumn.filter => filter; string IFilterableColumn.filter_template => DataGridFilterKnownTemplates.DiscreteValues; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs index b572530fc..f65aeb4be 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs @@ -1,5 +1,7 @@ using bts; +using bts.api; using bts.api.core; +using bts.api.orderman; using bts.collections; using bts.utils; using mbts.utils; @@ -13,26 +15,26 @@ using System.Reactive.Linq; namespace mbts.spark { class OrderBookRowCollection : ObservableCollection { static readonly IEqualityComparer> _node_equality_comparer = EqualityComparer.Create, uint>(n => n.value.order_key); - readonly Func _row_factory; + readonly Func _row_factory; /// Critical: Any and all row mutations must be executed through . /// Mutations can change the sort-position of a row, and will be broken if its elements aren't properly sorted (it relies heavily on binary-search logic). SortedTree _tree; IComparer _comparer; - Func _filter; + Func _filter; bool _is_refresh_pending; int _deferment_count = 0; /// When processing changed-items, the method is used to update the changed row. It is not expected to fail (return false), /// but if it does, we don't want to overwhelm the logs with a bunch of useless noise in case it starts happening repeatedly, hence the int fields below. - /// , , . + /// , , . const int _max_replace_misses_logged = 10; int _n_replace_misses_logged; - public OrderBookRowCollection(Func row_factory, IComparer comparer) { + public OrderBookRowCollection(Func row_factory, IComparer comparer) { _row_factory = row_factory; _tree = _mk_tree(_comparer = comparer); } - public Func filter { + public Func filter { get => _filter; set { _filter = value; @@ -144,24 +146,24 @@ namespace mbts.spark { }); } - public void update(IIncrementalTableUpdateTransform table) { + public void update(TableUpdate<(SecurityImmutable? sec,OrderRecordImmutable order),uint> table) { IEnumerable> impacted_branch_roots; int n_removed = table.removed.Count(); if (n_removed == total_row_count) { // full refresh (most likely this is the first update or a server restart) - _tree = _mk_tree(_comparer, table.added.Select(_row_factory)); + _tree = _mk_tree(_comparer, table.added.Select(x => _row_factory(x))); impacted_branch_roots = _tree.RootNodes; } else { - foreach (OrderRecordTuple ort in table.removed) + foreach (OrderSecurity ort in table.removed) _tree.Remove(ort.order.Key); - foreach (Change change in table.changes) { + foreach (var change in table.changes) { if (!_tree.TryReplace(change.New.order.Key, r => { r.update(change.New); return r; })) { // uh oh, replace failed! Let's log this, but not a million times (this seems like the kind of error that, if it happens once, could happen a lot). if (_n_replace_misses_logged++ < _max_replace_misses_logged) - Log.Ui.error($"{nameof(OrderBookRowCollection)} failed to process Changed {nameof(OrderRecordTuple)} in update (old:{change.Old.key}, new:{change.New.key}). This is the '{_n_replace_misses_logged}' time this has happened in this OrderBook."); + Log.Ui.error($"{nameof(OrderBookRowCollection)} failed to process Changed {nameof(OrderSecurity)} in update (old:{change.Old.order.Key}, new:{change.New.order.Key}). This is the '{_n_replace_misses_logged}' time this has happened in this OrderBook."); } } - foreach (OrderRecordTuple ort in table.added) + foreach (OrderSecurity ort in table.added) _tree.Insert(_row_factory(ort)); impacted_branch_roots = table.added.Concat(table.changed).Select(ort => ort.order.Key).Select(_tree.GetRootNode) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index 5a5213efd..dc41be7ac 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -16,9 +16,10 @@ namespace mbts.spark { IRaiseCanExecuteChangedCommand _show_order_modify_cmd, _toggle_expanded_cmd; bool _is_parent, _is_expanded; int _depth_level; - SecurityMD? _security_md => order_tuple.security_md; + SecurityImmutable? _security => order_tuple.security; + - public OrderBookRowVM(OrderBookVM owner, OrderRecordTuple order_tuple) { + public OrderBookRowVM(OrderBookVM owner, OrderSecurity order_tuple) { _owner = owner; this.order_tuple = order_tuple; is_visible = !is_child; @@ -27,16 +28,16 @@ namespace mbts.spark { public event Action invalidated; // Core - public OrderRecordTuple order_tuple { get; private set; } + public OrderSecurity order_tuple { get; private set; } public OrderRecordImmutable order => order_tuple.order; public OrderMetadata order_metadata => _owner.get_metadata(this); public uint order_key => order.Key; public uint parent_order_key => order.ParentKey; public uint security_key => order.security_key(); - public SecurityImmutable? security => _security_md.security(); + public SecurityImmutable? security => _security; public string security_text => security.display_name(); - public Venue exchange => _security_md.exchange(); - public string symbol => _security_md.symbol(); + public Venue exchange => _security.exchange_or_default(); + public string symbol => _security.symbol_or_default(); // Def public CustomOrderDefinition.CustomOneofCase order_type => order.order_type(); public OrderSubType order_sub_type => order.sub_type(); @@ -84,9 +85,9 @@ namespace mbts.spark { internal void expand_internal() => Set(ref _is_expanded, true); internal void collapse_internal() => Set(ref _is_expanded, false); - public void update(OrderRecordTuple order_tuple) { + public void update(OrderSecurity order_tuple) { Log.assert(order_tuple.order.Key == this.order.Key, "OrderBookRowVM: update_order was called with an order with the wrong key."); - bool had_security = _security_md.HasValue; + bool had_security = _security.HasValue; // _show_order_modify_cmd is lazy (instantiated in the default_command getter); it might be null, don't not force instantiation bool? could_modify_execute = _show_order_modify_cmd?.CanExecute(null); OrderRecordImmutable old_order = order; @@ -103,7 +104,7 @@ namespace mbts.spark { // however, the order-type and security cells DO use databindings - if these cells have changed, the PropertyChanged event must be raised regardless of whether or not invalidated is raised if (old_order.order_type() != order.order_type()) RaisePropertyChanged(nameof(order_type), nameof(order_metadata)); - if (!had_security && _security_md.HasValue) { + if (!had_security && _security.HasValue) { RaisePropertyChanged(nameof(security), nameof(security_text)); // need to raise invalidated to refresh the price cells (the security.ContractSpec determines how they are formatted - Ticks!) invalidate = true; diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index d86ea35bf..1b4a5afbf 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -3,7 +3,6 @@ using bts.api; using bts.api.orderman; using bts.shell; using bts.spark; -using bts.utils; using mbts.persist; using mbts.ui; using mbts.ui.controls; @@ -20,9 +19,15 @@ using System.Windows.Controls; using System.Windows.Input; using static bts.shell.OrderBookWindowFactory; using static mbts.spark.OrderBookView; -using _GridSettingsManager = mbts.ui.GridSettingsManager; +using _GridSettingsManager = + mbts.ui.GridSettingsManager< + mbts.spark.OrderBookRowVM, + bts.api.orderman.OrderSecurity, + bts.shell.INotificationServiceDataCache, + mbts.OrderRecordTuplePredicate, + bts.shell.OrderBookWindowFactory.OrderBookGridSettings>; -namespace mbts.spark { +namespace mbts.spark { partial class OrderBookVM : ViewModelBase, IDisposable, IHandleRowSelection, IHandleGridSorting, IWillLetYouKnowWhenIDispose { readonly IBus _bus; readonly OrdersModel _model; @@ -46,7 +51,7 @@ namespace mbts.spark { expand_all_rows_command = new DelegateCommand(_row_collection.expand_all); grid_settings_manager = _create_grid_settings_manager(owner_title, bus, notification_service, model, rows, columns); _resources = new List { - model.on_order_tuples_table.OnDispatcher().Subscribe(_row_collection.update), + model.on_orders.OnDispatcher().Subscribe(_row_collection.update), settings_manager.subscribe_current().OnDispatcher().Subscribe(grid_settings_manager), grid_settings_manager, Disposable.Create(() => { @@ -175,13 +180,13 @@ namespace mbts.spark { return new _GridSettingsManager( owner_title, bus, notification_service, model, rows, columns, () => { - var pvm = PredicateBuilderVMWrapper.create( + var pvm = PredicateBuilderVMWrapper.create( OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); pvm.selected_predicate_type = pvm.predicate_types.FirstOrDefault(); return pvm; }, col_id => { - var pvm = PredicateBuilderVMWrapper.create( + var pvm = PredicateBuilderVMWrapper.create( OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); pvm.selected_predicate_type = _corresponding_predicate_type(EnumHelper.parse_or_default(col_id)); return pvm; diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs index 1bbf0173e..8b0efabb6 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs @@ -16,7 +16,7 @@ using System.Windows.Data; using System.Windows.Media; //using IOrderBookWSM = mbts.persist.IWindowSettingsManager>; using OrderBookGridSettings = bts.shell.OrderBookWindowFactory.OrderBookGridSettings; -using OrderBookGridSettingsVM = mbts.spark.GridSettingsVM; +using OrderBookGridSettingsVM = mbts.spark.GridSettingsVM; namespace mbts.spark { internal class OrderBookWindow : SparkWindow, ISharedWindow { @@ -51,7 +51,7 @@ namespace mbts.spark { _settings_manager = settings_manager; } - public override OrderBookWindowFactory.OrderBookStartupData SaveState => _state = _vm.create(); + public override OrderBookWindowFactory.OrderBookStartupData SaveState => _state = _vm?.create(); public override string window_type => _window_type; public override UIElement Content => _view; @@ -86,12 +86,12 @@ namespace mbts.spark { ); _cancel_buy_orders_command = SparkCommands.CancelBuyOrders.complete( - _model.working_orders.Select(os => os.Values.Where(o => o.side() == Side.Buy).Select(o => o.Key)), + _model.working_orders.Select(os => os.Values.Where(o => o.order.side() == Side.Buy).Select(o => o.order.Key)), wbids => wbids.Any() ); _cancel_sell_orders_command = SparkCommands.CancelSellOrders.complete( - _model.working_orders.Select(os => os.Values.Where(o => o.side() == Side.Sell).Select(o => o.Key)), + _model.working_orders.Select(os => os.Values.Where(o => o.order.side() == Side.Sell).Select(o => o.order.Key)), wasks => wasks.Any() ); diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs index 77f1a5b14..52c632be9 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs @@ -215,7 +215,7 @@ namespace bts.shell { } [ProtoContract] - public class OrderBookGridSettings : IGridSettingsWithNotifications, IEquatable { + public class OrderBookGridSettings : IGridSettingsWithNotifications, IEquatable { [ProtoMember(1)] public GridSettings general { get; set; } [ProtoMember(2)] @@ -230,7 +230,7 @@ namespace bts.shell { } public IReadOnlyList predicate_animations => _predicate_animations ?? (_predicate_animations = new List()); - IEnumerable> IGridSettingsWithNotifications.predicate_animations => predicate_animations; + IEnumerable> IGridSettingsWithNotifications.predicate_animations => predicate_animations; public OrderBookGridSettings with(GridSettings general) => new OrderBookGridSettings(general ?? this.general, predicate_animations); diff --git a/src/client/spark/Notifications/INotificationService.cs b/src/client/spark/Notifications/INotificationService.cs index a01dd2506..eb3823cc7 100644 --- a/src/client/spark/Notifications/INotificationService.cs +++ b/src/client/spark/Notifications/INotificationService.cs @@ -1,4 +1,6 @@ using bts.api; +using bts.api.core; +using bts.api.orderman; using bts.spark; using mbts; using mbts.collections; @@ -9,6 +11,8 @@ using System; using System.Collections.Generic; using System.ComponentModel.Composition; using System.Linq; +using System.Reactive.Linq; +using OrderTuple = System.ValueTuple; namespace bts.shell { interface INotificationService { @@ -16,13 +20,13 @@ namespace bts.shell { INotificationServiceDataCache data_cache { get; } IDisposable link_to_activity(IPredicate predicate, SoundNotification sound); IDisposable link_to_activity(INotificationPredicate toast_notification_predicate); - IDisposable link_to_order(IPredicate predicate, SoundNotification sound); - IDisposable link_to_order(INotificationPredicate toast_notification_predicate); + IDisposable link_to_order(IPredicate predicate, SoundNotification sound); + IDisposable link_to_order(INotificationPredicate toast_notification_predicate); } static class NotificationServiceExtensions { public static IDisposable link(this INotificationService notification_service, IPredicate predicate, SoundNotification sound) { - if (predicate is IPredicate orp) + if (predicate is IPredicate orp) return notification_service.link_to_order(orp, sound); if (predicate is IPredicate ap) return notification_service.link_to_activity(ap, sound); @@ -30,7 +34,7 @@ namespace bts.shell { } public static IDisposable link(this INotificationService notification_service, INotificationPredicate toast_notification_predicate) { - if (toast_notification_predicate is INotificationPredicate orp) + if (toast_notification_predicate is INotificationPredicate orp) return notification_service.link_to_order(orp); if (toast_notification_predicate is INotificationPredicate ap) return notification_service.link_to_activity(ap); @@ -41,7 +45,7 @@ namespace bts.shell { [Export(typeof(INotificationService))] partial class NotificationService : INotificationService, IDisposable { readonly NotificationProcessor _activity_notifications; - readonly NotificationProcessor _order_notifications; + readonly NotificationProcessor _order_notifications; readonly DataCache _data_cache = new DataCache(); IDisposable _dispose; @@ -50,10 +54,10 @@ namespace bts.shell { public NotificationService(IBus bus, INotifier notifier, OrdersModel orders_model) { this.notifier = notifier; _activity_notifications = new NotificationProcessor(notifier); - _order_notifications = new NotificationProcessor(notifier); + _order_notifications = new NotificationProcessor(notifier); _dispose = Disposable.Create( bus.subscribe(SparkTags.LogActivity).Subscribe(_on_activity), - orders_model.on_order_tuples.Subscribe(_on_orders) + orders_model.on_orders.Subscribe(_on_orders) ); } @@ -63,17 +67,17 @@ namespace bts.shell { public IDisposable link_to_activity(IPredicate predicate, SoundNotification sound) => _activity_notifications.link(predicate, sound); public IDisposable link_to_activity(INotificationPredicate toast_notification_predicate) => _activity_notifications.link(toast_notification_predicate); - public IDisposable link_to_order(IPredicate predicate, SoundNotification sound) => _order_notifications.link(predicate, sound); - public IDisposable link_to_order(INotificationPredicate toast_notification_predicate) => _order_notifications.link(toast_notification_predicate); + public IDisposable link_to_order(IPredicate predicate, SoundNotification sound) => _order_notifications.link(predicate, sound); + public IDisposable link_to_order(INotificationPredicate toast_notification_predicate) => _order_notifications.link(toast_notification_predicate); public void Dispose() => Disposable.dispose(ref _dispose); void _on_activity(Activity activity) => _activity_notifications.process_item(activity); - void _on_orders(IIncrementalUpdate update) { + void _on_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable),uint> update) { // must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history - _data_cache.update(update); - foreach (var o in update.added.Values) + _data_cache.update(update.as_iincremental()); + foreach (var o in update.added) _order_notifications.process_item(o); } diff --git a/src/client/spark/Notifications/INotificationServiceDataCache.cs b/src/client/spark/Notifications/INotificationServiceDataCache.cs index aed90fbdb..fac9deeed 100644 --- a/src/client/spark/Notifications/INotificationServiceDataCache.cs +++ b/src/client/spark/Notifications/INotificationServiceDataCache.cs @@ -1,4 +1,6 @@ -using mbts.spark; +using bts.api; +using bts.api.orderman; +using mbts.spark; using mbts.utils; using System; using System.Collections.Generic; @@ -13,15 +15,21 @@ namespace bts.shell { interface INotificationServiceDataCache { DateTime last_order_reset_time { get; } bool is_first_order_update { get; } - IReadOnlyDictionary>> order_history { get; } + IReadOnlyDictionary>> order_history { get; } } static class NotificationServiceDataCacheExtensions { - public static OrderRecordTuple get_previous_order_state(this INotificationServiceDataCache notification_service_data_cache, uint key) => - notification_service_data_cache.order_history.TryGetValue(key, out List> l) && l.Count > 1 ? l[l.Count - 2].data : null; + public static OrderSecurity? get_previous_order_state(this INotificationServiceDataCache notification_service_data_cache, uint key) { + if (notification_service_data_cache.order_history.TryGetValue(key, out List> l)) { + if (l.Count > 1) { + return l[l.Count - 2].data; + } + } + return null; + } } - class TimeStampedData { + struct TimeStampedData { public readonly T data; public readonly DateTime time_stamp; @@ -34,14 +42,15 @@ namespace bts.shell { partial class NotificationService : INotificationService, IDisposable { class DataCache : INotificationServiceDataCache { const int _buffer_size = 10; - readonly Dictionary _order_cache = new Dictionary(); - readonly Dictionary>> _order_history = new Dictionary>>(); + readonly Dictionary _order_cache = new Dictionary(); + readonly Dictionary>> _order_history = new Dictionary>>(); + + public IReadOnlyDictionary>> order_history => _order_history; - public IReadOnlyDictionary>> order_history => _order_history; public DateTime last_order_reset_time { get; private set; } public bool is_first_order_update { get; private set; } - public void update(IIncrementalUpdate data) { + internal void update(IIncrementalUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> data) { DateTime now = DateTime.Now; if (data.removed.Count == _order_cache.Count) { last_order_reset_time = now; @@ -51,7 +60,7 @@ namespace bts.shell { foreach (var kv in data.removed) { if (data.added.ContainsKey(kv.Key)) { _order_cache[kv.Key] = kv.Value; - _order_history[kv.Key].Add(new TimeStampedData(kv.Value, now)); + _order_history[kv.Key].Add(new TimeStampedData(kv.Value, now)); } else { _order_cache.Remove(kv.Key); // TODO: maybe these shouldn't be removed? @@ -65,9 +74,9 @@ namespace bts.shell { continue; _order_cache.Add(kv.Key, kv.Value); _order_history.AddOrUpdate(kv.Key, - k => new List> { new TimeStampedData(kv.Value, now) }, + k => new List> { new TimeStampedData(kv.Value, now) }, (k, l) => { - l.Add(new TimeStampedData(kv.Value, now)); + l.Add(new TimeStampedData(kv.Value, now)); if (l.Count >= 10) l.RemoveAt(0); }); diff --git a/src/client/spark/Notifications/ToastNotification.cs b/src/client/spark/Notifications/ToastNotification.cs index f2590e5db..892a42cb4 100644 --- a/src/client/spark/Notifications/ToastNotification.cs +++ b/src/client/spark/Notifications/ToastNotification.cs @@ -60,14 +60,14 @@ namespace bts.shell { public static IToastNotification order_filled(OrderRecordImmutable oldval, OrderRecordImmutable newval, SecurityImmutable? security) => new OrderFilledToastNotification(oldval, newval, security); - public static IToastNotification order_filled(OrderRecordTuple oldval, OrderRecordTuple newval) => - new OrderFilledToastNotification(oldval.order, newval.order, newval.security_md?.security); + public static IToastNotification order_filled(OrderSecurity oldval, OrderSecurity newval) => + new OrderFilledToastNotification(oldval.order, newval.order, newval.security); public static IToastNotification order_completed(OrderRecordImmutable order, SecurityImmutable? security) => new OrderCompletedToastNotification(order, security); - public static IToastNotification order_completed(OrderRecordTuple order) => - new OrderCompletedToastNotification(order.order, order.security_md?.security); + public static IToastNotification order_completed(OrderSecurity order) => + new OrderCompletedToastNotification(order.order, order.security); public static IToastNotification order_status_changed(uint key, OrderStatus? old_status, OrderStatus new_status, SecurityImmutable? security) => new OrderStatusChangeToastNotification(key, old_status, new_status, security); diff --git a/src/client/spark/Orders/ExistingOrderTicketVM.cs b/src/client/spark/Orders/ExistingOrderTicketVM.cs index 18583a019..ddbf8f6f4 100644 --- a/src/client/spark/Orders/ExistingOrderTicketVM.cs +++ b/src/client/spark/Orders/ExistingOrderTicketVM.cs @@ -19,7 +19,7 @@ namespace mbts.spark { readonly Property _current_order_type; readonly Property _price_dirty, _size_dirty, _account_dirty, _time_in_force_dirty, _order_type_dirty; - public ExistingOrderTicketVM(OrdersModel model, IObservable order_source, IObservable calculator_buttons) { + public ExistingOrderTicketVM(OrdersModel model, IObservable<(SecurityMD? security_md, OrderRecordImmutable order)> order_source, IObservable calculator_buttons) { uint? key = null; var security_md_source = order_source.Where(ot => ot.security_md.HasValue).Select(ot => ot.security_md.Value); // diff --git a/src/client/spark/Orders/FillSecurity.cs b/src/client/spark/Orders/FillSecurity.cs new file mode 100644 index 000000000..01d8d0a20 --- /dev/null +++ b/src/client/spark/Orders/FillSecurity.cs @@ -0,0 +1,25 @@ +using System; + +namespace bts.api.orderman { + public readonly struct FillSecurity : IEquatable { + public FillSecurity(SecurityImmutable? sec, FillImmutable fill) { + this.sec = sec; + this.fill = fill; + } + public FillSecurity((SecurityImmutable? sec, FillImmutable fill) t) { + sec = t.sec; + fill = t.fill; + } + public static implicit operator FillSecurity(ValueTuple x) => new FillSecurity(x); + public SecurityImmutable? sec { get; } + public FillImmutable fill { get; } + + public override bool Equals(object obj) => (obj is FillSecurity os) && Equals(os); + public bool Equals(FillSecurity other) => fill.FillKey == other.fill.FillKey && sec == other.sec; + public override int GetHashCode() => (sec, fill).GetHashCode(); + + public static bool operator ==(FillSecurity left, FillSecurity right) => left.Equals(right); + public static bool operator !=(FillSecurity left, FillSecurity right) => !left.Equals(right); + + } +} \ No newline at end of file diff --git a/src/client/spark/Orders/OrderRecordTuple.cs b/src/client/spark/Orders/OrderRecordTuple.cs deleted file mode 100644 index 4e6255364..000000000 --- a/src/client/spark/Orders/OrderRecordTuple.cs +++ /dev/null @@ -1,47 +0,0 @@ -using bts.api.orderman; -using bts.utils; -using mbts.utils; -using System; - -namespace mbts.spark { - struct OrderRecordTupleKey : IEquatable { - public readonly uint order_key, security_key; - - public OrderRecordTupleKey(uint order_key, uint security_key) { - this.order_key = order_key; - this.security_key = security_key; - } - - public bool Equals(OrderRecordTupleKey other) => - other.order_key == order_key && - other.security_key == security_key; - - public override bool Equals(object obj) => - obj is OrderRecordTupleKey k && Equals(k); - - public override int GetHashCode() => - EqualityComparer.aggregate_hash( - (int)order_key, - (int)security_key); - } - - class OrderRecordTuple : IEquatable { - public readonly OrderRecordImmutable order; - public readonly SecurityMD? security_md; - public readonly OrderRecordTupleKey key; - - public OrderRecordTuple(OrderRecordImmutable order, SecurityMD? security_md) { - this.order = order; - this.security_md = security_md; - key = new OrderRecordTupleKey(order.Key, security_md.HasValue ? security_md.Value.security.Id : 0u); - } - - public bool Equals(OrderRecordTuple other) => other != null - && other.order.Equals(order) - && other.security_md.HasValue == security_md.HasValue; - - public override bool Equals(object obj) => Equals(obj as OrderRecordTuple); - public override int GetHashCode() => key.GetHashCode(); - public override string ToString() => $"order:{order.Key}, sec:{security_md.display_name()}"; - } -} \ No newline at end of file diff --git a/src/client/spark/Orders/OrderRecordTupleSource.cs b/src/client/spark/Orders/OrderRecordTupleSource.cs deleted file mode 100644 index 0c8aeaccf..000000000 --- a/src/client/spark/Orders/OrderRecordTupleSource.cs +++ /dev/null @@ -1,147 +0,0 @@ -using bts.api.core; -using bts.api.orderman; -using mbts.collections; -using mbts.utils; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reactive.Linq; -using System.Reactive.Subjects; - -namespace mbts.spark { - class OrderRecordTupleSource : IDisposable { - readonly object _lock = new object(); - // subjects - readonly Subject> _subject = new Subject>(); - readonly Subject> _table_subject = new Subject>(); - readonly Subject> _order_securities_subject = new Subject>(); - // data cache - readonly Dictionary _state = new Dictionary(); - readonly SetMap _security_orders = new SetMap(); - readonly IDictionary _security_md_cache = new Dictionary(0); - IReadOnlyDictionary _orders_cache = new Dictionary(0); - IDisposable _dispose; - - public OrderRecordTupleSource(IObservable> securities, IObservable> orders) => - _dispose = Disposable.Create( - securities.Subscribe(_update_securities), - orders.Subscribe(_update_orders)); - - public IObservable> subscribe_incremental_update() => - Observable.Create>(observer => { - lock (_lock) { - observer.OnNext(new IncrementalUpdate(new Dictionary(_state), new Dictionary(0))); - return _subject.Subscribe(observer); - } - }); - - public IObservable> subscribe_incremental_table_update() => - Observable.Create>(observer => { - lock (_lock) { - observer.OnNext(IncrementalTableUpdateTranform.init(_state.Values)); - return _table_subject.Subscribe(observer); - } - }); - - public IObservable> subscribe_order_securities() => - Observable.Create>(observer => { - lock (_lock) { - var init = new Dictionary(); - foreach (uint sec_key in _security_orders.Keys) { - if (_security_md_cache.TryGetValue(sec_key, out SecurityMD smd)) - init.Add(sec_key, smd); - } - observer.OnNext(new IncrementalUpdate(init, new Dictionary(0))); - return _order_securities_subject.Subscribe(observer); - } - }); - - public void Dispose() => Disposable.dispose(ref _dispose); - - void _update_orders(TableUpdate orders) { - lock (_lock) { - Dictionary removed = new Dictionary(), added = new Dictionary(); - Dictionary added_securities = new Dictionary(), removed_securities = new Dictionary(); - foreach (OrderRecordImmutable order in orders.removed) { - uint sec_key = order.security_key(); - OrderRecordTuple old_ft = new OrderRecordTuple(order, _get_security_md(sec_key)); - removed.Add(old_ft.order.Key, old_ft); - if (_security_orders.removes_key(sec_key, order.Key) && old_ft.security_md.HasValue) - removed_securities.Add(sec_key, old_ft.security_md.Value); - } - foreach (Change ch in orders.changes) { - // orders cannot change securities; ch.New can use the same security_md as ch.Old - var sec_md = _get_security_md(ch.Old.security_key()); - removed.Add(ch.Old.Key, new OrderRecordTuple(ch.Old, sec_md)); - added.Add(ch.New.Key, new OrderRecordTuple(ch.New, sec_md)); - } - foreach (OrderRecordImmutable order in orders.added) { - uint sec_key = order.security_key(); - OrderRecordTuple new_ort = new OrderRecordTuple(order, _get_security_md(sec_key)); - added.Add(new_ort.order.Key, new_ort); - if (_security_orders.adds_key(sec_key, order.Key) && new_ort.security_md.HasValue) - added_securities.Add(sec_key, new_ort.security_md.Value); - } - _orders_cache = orders.state; - _apply_changes(added, removed, added_securities, removed_securities); - } - } - - void _update_securities(IReadOnlyDictionary securities) { - lock (_lock) { - Dictionary added = new Dictionary(), removed = new Dictionary(); - Dictionary added_securities = new Dictionary(), removed_securities = new Dictionary(); - _security_md_cache.Keys.difference(securities.Keys, out IEnumerable added_keys, out IEnumerable removed_keys); - /// Warning: must NOT be reassigned with ; the latter is always the same object. - /// This would mean that the - /// call above would literally be calculating the difference between the keys of the same dictionary! - foreach (var k in removed_keys.ToList()) { - if (_security_orders.try_lookup(k, out ISet order_keys)) { - SecurityMD old_sec_md = _security_md_cache[k]; - removed_securities.Add(k, old_sec_md); - IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); - foreach (OrderRecordImmutable order in orders) { - OrderRecordTuple old_ft = new OrderRecordTuple(order, old_sec_md); - removed.Add(old_ft.order.Key, old_ft); - OrderRecordTuple new_ft = new OrderRecordTuple(order, null); - added.Add(new_ft.order.Key, new_ft); - } - } - _security_md_cache.Remove(k); - } - foreach (var k in added_keys.ToList()) { - SecurityMD new_sec_md = securities[k]; - if (_security_orders.try_lookup(k, out ISet order_keys)) { - added_securities.Add(k, new_sec_md); - IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); - foreach (OrderRecordImmutable order in orders) { - OrderRecordTuple old_ft = new OrderRecordTuple(order, null); - removed.Add(old_ft.order.Key, old_ft); - OrderRecordTuple new_ft = new OrderRecordTuple(order, new_sec_md); - added.Add(new_ft.order.Key, new_ft); - } - } - _security_md_cache.Add(k, new_sec_md); - } - _apply_changes(added, removed, added_securities, removed_securities); - } - } - - SecurityMD? _get_security_md(uint key) => - _security_md_cache.TryGetValue(key, out SecurityMD smd) ? smd : new SecurityMD?(); - - OrderRecordImmutable? _get_order(uint key) => - _orders_cache.TryGetValue(key, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?(); - - void _apply_changes(Dictionary added, Dictionary removed, Dictionary added_securities, Dictionary removed_securities) { - foreach (var kv in removed) - _state.Remove(kv.Key); - foreach (var kv in added) - _state.Add(kv.Key, kv.Value); - var update = new IncrementalUpdate(added, removed); - _subject.OnNext(update); - _table_subject.OnNext(update.as_table()); - _order_securities_subject.OnNext(new IncrementalUpdate(added_securities, removed_securities)); - } - } -} \ No newline at end of file diff --git a/src/client/spark/Orders/OrderSecurity.cs b/src/client/spark/Orders/OrderSecurity.cs new file mode 100644 index 000000000..dd9936162 --- /dev/null +++ b/src/client/spark/Orders/OrderSecurity.cs @@ -0,0 +1,25 @@ +using System; + +namespace bts.api.orderman { + public readonly struct OrderSecurity: IEquatable { + public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order) { + this.security = sec; + this.order = order; + } + public OrderSecurity((SecurityImmutable? sec, OrderRecordImmutable order) t) { + security = t.sec; + order = t.order; + } + public static implicit operator OrderSecurity(ValueTuple x) => new OrderSecurity(x); + public SecurityImmutable? security { get; } + public OrderRecordImmutable order { get; } + + public override bool Equals(object obj) => (obj is OrderSecurity os) && Equals(os); + public bool Equals(OrderSecurity other) => order.Key == other.order.Key && security == other.security; + public override int GetHashCode() => (security, order).GetHashCode(); + + public static bool operator == (OrderSecurity left, OrderSecurity right) => left.Equals(right); + public static bool operator != (OrderSecurity left, OrderSecurity right) => !left.Equals(right); + + } +} diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 2eab7f64a..bbf337718 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -16,10 +16,10 @@ using System.Linq; using System.Reactive.Concurrency; using System.Reactive.Linq; using System.Threading.Tasks; -using OrderPair = System.ValueTuple; using OrderUpdate = bts.api.core.TableUpdate<(bts.api.SecurityImmutable? sec, bts.api.orderman.OrderRecordImmutable order), uint>; namespace mbts.spark { + [Export] class OrdersModel : IDisposable { readonly IBus _bus; @@ -57,13 +57,15 @@ namespace mbts.spark { on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); working_orders = on_orders.Scan( - ImmutableDictionary.Empty, + ImmutableDictionary.Empty, (workers, table) => _update_working_orders(workers, table)) .Replay(1).RefCount(); - var order_record_tuple_source = new OrderRecordTupleSource(bus.subscribe(SparkTags.SecuritiesTag), on_orders); - on_order_tuples = order_record_tuple_source.subscribe_incremental_update(); - on_order_tuples_table = order_record_tuple_source.subscribe_incremental_table_update(); - on_order_securities = order_record_tuple_source.subscribe_order_securities(); + + on_order_securities = + on_orders. + as_iincremental() + .filter(t => t.sec.HasValue) + .Select(t => t.sec.Value); on_fills = _service.subscribei_fills() @@ -72,13 +74,12 @@ namespace mbts.spark { //.join(sstore, f => ) .Table().Cache(); _activity = new OrderActivitySummarizer(bus, on_orders); - on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders, on_fills); + on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders.Select(i => i.order, order => order.Key), on_fills); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); _security_positions_demuxer = new TableUpdateDemuxer(o => o.RecordId, p => p.SecurityId); _dispose = Disposable.Create( - on_orders.AddRemoveOnly().Subscribe(_security_orders_demuxer), + //on_orders.AddRemoveOnly().Subscribe(_security_orders_demuxer), on_positions.AddRemoveOnly().Subscribe(_security_positions_demuxer), - order_record_tuple_source, _bus.subscribe(SparkTags.CreateOrder).Subscribe(a => send_order(a.definition, a.price, a.size).ignore()), _bus.subscribe(SparkTags.RecreateOrder).Subscribe(a => a.ForEach(x => send_order(x.definition, x.price, x.size).ignore())), _bus.subscribe(SparkTags.CancelOrders).Subscribe(x => cancel_orders(x).ignore()), @@ -92,12 +93,10 @@ namespace mbts.spark { public SpreaderModel spreader => _spreader; public DynamicOrderHelper dynamic_helper => _dynamic_helper; public IObservable on_orders { get; } - public IObservable> on_order_tuples { get; } - public IObservable> on_order_tuples_table { get; } - public IObservable> on_order_securities { get; } + public IObservable> on_order_securities { get; } public IObservable> on_fills { get; } public IObservable> on_fill_tuples { get; } - public IObservable> working_orders { get; } + public IObservable> working_orders { get; } public IObservable> on_positions { get; } public IObservable> on_security_orders(uint security) => @@ -168,8 +167,8 @@ namespace mbts.spark { public void Dispose() => Disposable.dispose(ref _dispose); - static ImmutableDictionary - _update_working_orders(ImmutableDictionary target, OrderUpdate table) { + static ImmutableDictionary + _update_working_orders(ImmutableDictionary target, OrderUpdate table) { var clone = target.ToBuilder(); foreach (var o in table.removed) { if (o.order.is_active()) { diff --git a/src/client/spark/Predicates/Data/Events/OrderCanceledEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderCanceledEventPredicate.cs index 4546dfd6d..9ed71bb6e 100644 --- a/src/client/spark/Predicates/Data/Events/OrderCanceledEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderCanceledEventPredicate.cs @@ -26,7 +26,7 @@ namespace mbts { } [ProtoContract] - class OrderCanceledEventPredicate : IPredicateBuilder, IEquatable { + class OrderCanceledEventPredicate : IPredicateBuilder, IEquatable { public enum PredicateTypeOneofCase { [Description("(any order)")] AnyOrder, @@ -45,13 +45,13 @@ namespace mbts { } } - public IPredicate create(object obj) => + public IPredicate create(object obj) => create_predicate((INotificationServiceDataCache)obj); - public IPredicate create_predicate(INotificationServiceDataCache data_cache) => + public IPredicate create_predicate(INotificationServiceDataCache data_cache) => OrderEventPredicateHelper.create_status_transition_predicate(OrderStatus.Canceled, security, data_cache); - public INotificationPredicate create_toast(INotificationServiceDataCache data_cache) => + public INotificationPredicate create_toast(INotificationServiceDataCache data_cache) => new MyToaster(security?.create(data_cache), data_cache); public bool Equals(OrderCanceledEventPredicate other) => other != null && other.security.safe_equals(other.security); @@ -61,7 +61,7 @@ namespace mbts { static bool _is_canceled(OrderRecordImmutable o) => o.HasState && o.State.Status == OrderStatus.Canceled; - class MyToaster : INotificationPredicate, IEquatable { + class MyToaster : INotificationPredicate, IEquatable { readonly IPredicate _security_predicate; readonly INotificationServiceDataCache _data_cache; @@ -70,12 +70,13 @@ namespace mbts { _data_cache = data_cache; } - public IToastNotification trigger(OrderRecordTuple order) => - OrderEventPredicateHelper.check_order_transition_trigger(order, _is_canceled, _security_predicate, _data_cache, out OrderRecordTuple oldval) ? - ToastNotifications.order_status_changed(order.order.Key, OrderEventPredicateHelper.get_nullable_status(oldval), OrderStatus.Canceled, order.security_md.HasValue ? order.security_md.Value.security : new SecurityImmutable?()) : + public IToastNotification trigger(OrderSecurity order) => + OrderEventPredicateHelper.check_order_transition_trigger(order, _is_canceled, _security_predicate, _data_cache, out var oldval) ? + ToastNotifications.order_status_changed(order.order.Key, + OrderEventPredicateHelper.get_nullable_status(oldval), OrderStatus.Canceled, order.security) : ToastNotifications.empty; - bool IPredicate.trigger(OrderRecordTuple order) => + bool IPredicate.trigger(OrderSecurity order) => OrderEventPredicateHelper.check_order_transition_predicate(order, _is_canceled, _security_predicate, _data_cache); public bool Equals(MyToaster other) => diff --git a/src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs index 881201de5..f6942d908 100644 --- a/src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs @@ -26,7 +26,7 @@ namespace mbts { } [ProtoContract] - class OrderCompletedEventPredicate : IPredicateBuilder, IEquatable { + class OrderCompletedEventPredicate : IPredicateBuilder, IEquatable { public enum PredicateTypeOneofCase { [Description("(any order)")] AnyOrder, @@ -45,13 +45,13 @@ namespace mbts { } } - public IPredicate create(object obj) => + public IPredicate create(object obj) => create_predicate((INotificationServiceDataCache)obj); - public IPredicate create_predicate(INotificationServiceDataCache data_cache) => + public IPredicate create_predicate(INotificationServiceDataCache data_cache) => OrderEventPredicateHelper.create_status_transition_predicate(OrderStatus.Completed, security, data_cache); - public INotificationPredicate create_toast(INotificationServiceDataCache data_cache) => + public INotificationPredicate create_toast(INotificationServiceDataCache data_cache) => new MyToaster(security?.create(data_cache), data_cache); public bool Equals(OrderCompletedEventPredicate other) => other != null && other.security.safe_equals(other.security); @@ -61,7 +61,7 @@ namespace mbts { static bool _is_completed(OrderRecordImmutable o) => o.HasState && o.State.Status == OrderStatus.Completed; - class MyToaster : INotificationPredicate, IEquatable { + class MyToaster : INotificationPredicate, IEquatable { readonly IPredicate _security_predicate; readonly INotificationServiceDataCache _data_cache; @@ -70,11 +70,11 @@ namespace mbts { _data_cache = data_cache; } - public IToastNotification trigger(OrderRecordTuple order) => + public IToastNotification trigger(OrderSecurity order) => OrderEventPredicateHelper.check_order_transition_predicate(order, _is_completed, _security_predicate, _data_cache) ? ToastNotifications.order_completed(order) : ToastNotifications.empty; - bool IPredicate.trigger(OrderRecordTuple order) => + bool IPredicate.trigger(OrderSecurity order) => OrderEventPredicateHelper.check_order_transition_predicate(order, _is_completed, _security_predicate, _data_cache); public bool Equals(MyToaster other) => diff --git a/src/client/spark/Predicates/Data/Events/OrderEventPredicateHelper.cs b/src/client/spark/Predicates/Data/Events/OrderEventPredicateHelper.cs index 25c019787..f9e27e62f 100644 --- a/src/client/spark/Predicates/Data/Events/OrderEventPredicateHelper.cs +++ b/src/client/spark/Predicates/Data/Events/OrderEventPredicateHelper.cs @@ -11,16 +11,16 @@ namespace mbts { /// /// Checks if the order has transitioned from 'not passing the predicate, or not existing' to 'passing the predicate.' /// - public static bool check_order_transition_predicate(OrderRecordTuple order, Func order_predicate_core, IPredicate security_predicate, INotificationServiceDataCache data_cache) => - check_order_transition_trigger(order, order_predicate_core, security_predicate, data_cache, out OrderRecordTuple _); + public static bool check_order_transition_predicate(OrderSecurity order, Func order_predicate_core, IPredicate security_predicate, INotificationServiceDataCache data_cache) => + check_order_transition_trigger(order, order_predicate_core, security_predicate, data_cache, out var _); /// /// Checks if the order has transitioned from 'not passing the predicate, or not existing' to 'passing the predicate.' Oldval will be null if the order did not previously exist, or does not currently pass the predicate. /// - public static bool check_order_transition_trigger(OrderRecordTuple order, Func order_predicate_core, IPredicate security_predicate, INotificationServiceDataCache data_cache, out OrderRecordTuple oldval) { + public static bool check_order_transition_trigger(OrderSecurity order, Func order_predicate_core, IPredicate security_predicate, INotificationServiceDataCache data_cache, out OrderSecurity? oldval) { if (check_security_predicate(order, security_predicate) && order_predicate_core(order.order)) { oldval = data_cache.get_previous_order_state(order.order.Key); - return oldval == null ? !data_cache.is_first_order_update : !order_predicate_core(oldval.order); + return oldval == null ? !data_cache.is_first_order_update : !order_predicate_core(oldval.Value.order); } oldval = null; return false; @@ -29,19 +29,19 @@ namespace mbts { /// /// Checks if the 'security predicate is null,' or 'it is not null, and the order has a non-null security which passes.' /// - public static bool check_security_predicate(OrderRecordTuple order, IPredicate security_predicate) => - security_predicate == null || (order.security_md.HasValue && security_predicate.trigger(order.security_md.Value.security)); + public static bool check_security_predicate(OrderSecurity order, IPredicate security_predicate) => + security_predicate == null || (order.security.HasValue && security_predicate.trigger(order.security.Value)); - public static OrderStatus? get_nullable_status(OrderRecordTuple order) { - if (order == null && order.order.HasState) - return order.order.State.Status; + public static OrderStatus? get_nullable_status(OrderSecurity? order) { + if (order?.order.HasState == true) + return order?.order.State.Status; return null; } - public static IPredicate create_status_transition_predicate(OrderStatus status, SecurityPredicate security_predicate, INotificationServiceDataCache data_cache) => + public static IPredicate create_status_transition_predicate(OrderStatus status, SecurityPredicate security_predicate, INotificationServiceDataCache data_cache) => new OrderStatusTransitionPredicate(status, security_predicate?.create(data_cache), data_cache); - class OrderStatusTransitionPredicate : IPredicate, IEquatable { + class OrderStatusTransitionPredicate : IPredicate, IEquatable { readonly IPredicate _security_predicate; readonly INotificationServiceDataCache _data_cache; readonly OrderStatus _status; @@ -52,7 +52,7 @@ namespace mbts { _data_cache = data_cache; } - public bool trigger(OrderRecordTuple value) => + public bool trigger(OrderSecurity value) => check_order_transition_predicate(value, o => o.HasState && o.State.Status == _status, _security_predicate, _data_cache); public bool Equals(OrderStatusTransitionPredicate other) => diff --git a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs index 9b67bd56f..b6b1f749f 100644 --- a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs @@ -1,4 +1,5 @@ using bts.api; +using bts.api.orderman; using bts.shell; using mbts.spark; using mbts.ui.settings; @@ -14,15 +15,15 @@ namespace mbts { { OrderFilledEventPredicate.PredicateTypeOneofCase.SecurityOrder, SecurityCriteriaVMFactory.create(bus, o => o.security?.is_value ?? new SecurityCriteria(), (p, s) => { if (p.security == null) - p.security = new SecurityPredicate(); - p.security.is_value = s; +{ p.security = new SecurityPredicate(); +} p.security.is_value = s; }) } }; } [ProtoContract] - class OrderFilledEventPredicate : IPredicateBuilder, IEquatable { + class OrderFilledEventPredicate : IPredicateBuilder, IEquatable { public enum PredicateTypeOneofCase { [PredicateCase("(any order)")] AnyOrder, @@ -35,19 +36,21 @@ namespace mbts { public PredicateTypeOneofCase predicate_type_case { get { - if (security != null) + if (security != null) { return PredicateTypeOneofCase.SecurityOrder; + } + return PredicateTypeOneofCase.AnyOrder; } } - public IPredicate create(object obj) => + public IPredicate create(object obj) => create_predicate((INotificationServiceDataCache)obj); - public IPredicate create_predicate(INotificationServiceDataCache notification_service_data_cache) => + public IPredicate create_predicate(INotificationServiceDataCache notification_service_data_cache) => new FillTrigger(_create_model(notification_service_data_cache)); - public INotificationPredicate create_toast(INotificationServiceDataCache notification_service_data_cache) => + public INotificationPredicate create_toast(INotificationServiceDataCache notification_service_data_cache) => new FillToastNotificationTrigger(_create_model(notification_service_data_cache)); public bool Equals(OrderFilledEventPredicate other) => other != null && other.security.safe_equals(other.security); @@ -66,35 +69,43 @@ namespace mbts { _security_predicate = security_predicate; } - public (bool trigger, OrderRecordTuple oldval, OrderRecordTuple newval) get_filled_order_args(OrderRecordTuple ort) { + // Please review carfully + public (OrderSecurity oldval, OrderSecurity newval)? get_filled_order_args(OrderSecurity ort) { if (ort.order.is_exchange_order() && OrderEventPredicateHelper.check_security_predicate(ort, _security_predicate)) { - if (_data_cache.get_previous_order_state(ort.order.Key) is OrderRecordTuple oldval) - return (ort.order.filled() > oldval.order.filled(), oldval, ort); + if (_data_cache.get_previous_order_state(ort.order.Key) is var oldval && oldval.HasValue) { + if (ort.order.filled() > oldval?.order.filled()) { + return (oldval.Value, ort); + } + } else if (!_data_cache.is_first_order_update && ort.order.filled() > 0) { // old order will be a copy of new order, except with filledQuantity set to 0 var old_state = ort.order.State.CloneToMutable(); old_state.FilledQuantity = 0; - return (true, new OrderRecordTuple(ort.order.with(state: old_state), ort.security_md), ort); + return (new OrderSecurity(ort.security, ort.order.with(state: old_state)), ort); } } - return (false, null, null); + return null; } } - class FillTrigger : IPredicate { + class FillTrigger : IPredicate { protected readonly FillTriggerModel _model; public FillTrigger(FillTriggerModel model) => _model = model; - public bool trigger(OrderRecordTuple order) => _model.get_filled_order_args(order).trigger; + public bool trigger(OrderSecurity order) => _model.get_filled_order_args(order).HasValue; } - class FillToastNotificationTrigger : FillTrigger, INotificationPredicate { + class FillToastNotificationTrigger : FillTrigger, INotificationPredicate { public FillToastNotificationTrigger(FillTriggerModel model) : base(model) { } - public new IToastNotification trigger(OrderRecordTuple value) { - (var trigger, var oldval, var newval) = _model.get_filled_order_args(value); - return trigger ? ToastNotifications.order_filled(oldval, newval) : ToastNotifications.empty; + public new IToastNotification trigger(OrderSecurity value) { + var args = _model.get_filled_order_args(value); + if (args.HasValue) { + return ToastNotifications.order_filled(args.Value.oldval, args.Value.newval); + } + return ToastNotifications.empty; + //return args ? ToastNotifications.order_filled(oldval, newval) : ToastNotifications.empty; } } } diff --git a/src/client/spark/Predicates/Data/OrderRecordTuplePredicate.cs b/src/client/spark/Predicates/Data/OrderRecordTuplePredicate.cs index 1a938f50a..e12c4d480 100644 --- a/src/client/spark/Predicates/Data/OrderRecordTuplePredicate.cs +++ b/src/client/spark/Predicates/Data/OrderRecordTuplePredicate.cs @@ -61,7 +61,7 @@ namespace mbts { } [ProtoContract] - class OrderRecordTuplePredicate : INotificationPredicateBuilder, IEquatable { + class OrderRecordTuplePredicate : INotificationPredicateBuilder, IEquatable { SecurityPredicate _security; SidePredicate _side; DoublePredicate _price, _last_price, _avg_price; @@ -206,14 +206,14 @@ namespace mbts { } } - public IPredicate create_predicate(INotificationServiceDataCache data_cache) { + public IPredicate create_predicate(INotificationServiceDataCache data_cache) { switch (predicate_type_case) { - case PTOC.Security: return wrap(_security, r => r.security_md.HasValue ? r.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable); + case PTOC.Security: return wrap(_security, r => r.security ?? SecurityDataHelper.Defaults.SecurityImmutable); case PTOC.Time: return wrap(_time_created, r => r.order.created()); case PTOC.LastChange: return wrap(_last_change, r => r.order.last_change()); case PTOC.User: return wrap(_user, r => r.order.user_id()); case PTOC.Account: return wrap(_account, r => r.order.account_id()); - case PTOC.Exchange: return wrap(_exchange, r => r.security_md.HasValue ? r.security_md.Value.security.Exchange : Venue.Unset); + case PTOC.Exchange: return wrap(_exchange, r => r.security.exchange_or_default()); case PTOC.TimeInForce: return wrap(_time_in_force, r => r.order.time_in_force()); case PTOC.OrderType: return wrap(_order_type, r => r.order.order_type()); case PTOC.SubType: return wrap(_order_sub_type, r => r.order.sub_type()); @@ -234,14 +234,14 @@ namespace mbts { default: return null; } // - IPredicate wrap(IPredicateBuilder trigger, Func select) => - trigger.create(data_cache).wrap(select); + IPredicate wrap(IPredicateBuilder trigger, Func select) => + trigger.create(data_cache).wrap(select); } - IPredicate IPredicateBuilder.create(object obj) => + IPredicate IPredicateBuilder.create(object obj) => create_predicate((INotificationServiceDataCache)obj); - public INotificationPredicate create_notification_predicate(INotificationServiceDataCache notification_service_data_cache) { + public INotificationPredicate create_notification_predicate(INotificationServiceDataCache notification_service_data_cache) { if (!is_toast_enabled) return null; switch (predicate_type_case) { @@ -252,7 +252,7 @@ namespace mbts { } } - INotificationPredicate INotificationPredicateBuilder.create_notification_predicate(object obj) => + INotificationPredicate INotificationPredicateBuilder.create_notification_predicate(object obj) => create_notification_predicate((INotificationServiceDataCache)obj); public bool Equals(OrderRecordTuplePredicate other) { @@ -325,8 +325,8 @@ namespace mbts { return EqualityComparer.phase_hash(raw_hash, (int)ptoc); } - static string _get_order_placed_toast_msg(OrderRecordTuple order) => - $"Order Sent: {order.security_md.display_name()} {order.order.short_description(order.security_md.spec_or_default())}"; + static string _get_order_placed_toast_msg(OrderSecurity order) => + $"Order Sent: {order.security.display_name()} {order.order.short_description(order.security.spec_or_default())}"; void _set(ref T field, T value) { if (value != null) { @@ -359,7 +359,7 @@ namespace mbts { } [ProtoContract] - class OrderRecordTuplePredicateAnimations : IPredicateAnimationsData { + class OrderRecordTuplePredicateAnimations : IPredicateAnimationsData { [ProtoMember(1)] OrderRecordTuplePredicate _predicate; [ProtoMember(2)] diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 558c588f4..3b0f53794 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -15,6 +15,7 @@ using System.Threading.Tasks; using security_key = System.UInt32; namespace bts { + [Export] public class SecurityStore : IDisposable { readonly MarketDataRelay _md; @@ -104,7 +105,7 @@ namespace bts { foreach (var key in keys) { var security = try_get_local(key); if (security.HasValue) { - result.Add(key, security.Value); + result.TryAdd(key, security.Value); } else { missing.Keys.Add(key); diff --git a/src/client/spark/core/GlobalExtensions/GlobalExtensions.cs b/src/client/spark/core/GlobalExtensions/GlobalExtensions.cs index 93ba83022..36c545789 100644 --- a/src/client/spark/core/GlobalExtensions/GlobalExtensions.cs +++ b/src/client/spark/core/GlobalExtensions/GlobalExtensions.cs @@ -233,6 +233,7 @@ namespace mbts return has_value; } + } } diff --git a/src/client/spark/core/Utility/TableUpdateTransform.cs b/src/client/spark/core/Utility/TableUpdateTransform.cs index 6dee3f511..fbd3da11f 100644 --- a/src/client/spark/core/Utility/TableUpdateTransform.cs +++ b/src/client/spark/core/Utility/TableUpdateTransform.cs @@ -26,9 +26,6 @@ namespace mbts.utils { } class IncrementalUpdate : IIncrementalUpdate where TKey : IEquatable { - public IncrementalUpdate() - : this(new Dictionary(0), new Dictionary(0)) { } - public IncrementalUpdate(IReadOnlyDictionary added, IReadOnlyDictionary removed) { this.added = added; this.removed = removed; @@ -305,6 +302,40 @@ namespace mbts.utils { state_clone.Add(kv.Key, kv.Value); return new PivotTableUpdateTransform(state_clone, added.Values.ToList(), removed.Values.ToList(), _key_map, _selector, _result_key, _key); } - } + } + + public static IObservable> as_iincremental(this IObservable> src) where TKey : IEquatable { + return src.Select(as_iincremental); + } + + public static IIncrementalUpdate as_iincremental(this TableUpdate update) where TKey : IEquatable { + var key = update.key_extractor; + var added = update.added.ToDictionary(key); + var removed = update.removed.ToDictionary(key); + foreach (var ch in update.changes) { + added[key(ch.New)] = ch.New; + removed[key(ch.Old)] = ch.Old; + } + return new IncrementalUpdate(added, removed); + } + + public static IObservable> + Select(this IObservable> src, Func map) where TKey : IEquatable { + return src.Select(update => { + return new IncrementalUpdate( + new MappedDictionary(update.added, map), + new MappedDictionary(update.removed, map)); + }); + } + + public static IObservable> + Where(this IObservable> src, Func where) where TKey : IEquatable { + return src.Select(update => { + return new IncrementalUpdate( + new FilteredDictionary(update.added, where), + new FilteredDictionary(update.removed, where)); + }); + } + } } \ No newline at end of file diff --git a/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs b/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs index e4eccfc55..5d5e60b52 100644 --- a/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs +++ b/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs @@ -17,7 +17,8 @@ using System.Windows.Media; using static mbts.ui.controls.DIYGrid; namespace mbts.ui { - class GridSettingsManager : ViewModelBase, IValueFactory, IObserver, IDisposable + class GridSettingsManager : + ViewModelBase, IValueFactory, IObserver, IDisposable where TRow : IRowVM where TPredicateBuilder : IPredicateBuilder, new() where TSettingsData : class, IGridSettingsWithNotifications { diff --git a/src/client/spark/view/Spark/Core/SparkUtils.cs b/src/client/spark/view/Spark/Core/SparkUtils.cs index fb620cb3a..3474ea79f 100644 --- a/src/client/spark/view/Spark/Core/SparkUtils.cs +++ b/src/client/spark/view/Spark/Core/SparkUtils.cs @@ -48,9 +48,6 @@ namespace mbts.spark { } #endregion - public static string display_name(this SecurityMD? security_md) => - (security_md?.security).display_name(); - public static string display_name(this SecurityImmutable? security) => security.HasValue ? security.Value.ToString() : "unknown security"; @@ -59,11 +56,11 @@ namespace mbts.spark { public static SecurityImmutable? security(this SecurityMD? security_md) => security_md.HasValue ? new SecurityImmutable?(security_md.Value.security) : new SecurityImmutable?(); - public static Venue exchange(this SecurityMD? security_md) => - security_md.HasValue ? security_md.Value.security.Exchange : Venue.Unset; + public static Venue exchange_or_default(this SecurityImmutable? security) => + security?.Exchange ?? Venue.Unset; - public static string symbol(this SecurityMD? security_md) => - security_md.HasValue ? security_md.Value.security.Symbol : "unknown"; + public static string symbol_or_default(this SecurityImmutable? security) => + security?.Symbol ?? "unknown"; public static async Task get_security_async(IBus bus, uint security_key) => await bus.subscribe(SparkTags.SecuritiesTag).Where(map => map.ContainsKey(security_key)).Select(map => map[security_key].security).FirstAsync(); diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index 4b7528808..034725b4a 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -134,8 +134,14 @@ namespace mbts.spark { // Here we manage N orders to 1 price slot in the ladder Dictionary> orders_used_in_this_update = new Dictionary>(_equality); - var ladders_orders = working.Values.Where(o => o.Def.SecurityKey == security.Id && (o.HasState && o.State.DisplayStatus == OrderState.Types.DisplayStatus.Visible)); - foreach (var wo in ladders_orders) { + var ladders_orders = + working.Values + .Where(o => + o.order.Def.SecurityKey == security.Id && + (o.order.HasState && o.order.State.DisplayStatus == OrderState.Types.DisplayStatus.Visible)); + + foreach (var osec in ladders_orders) { + var wo = osec.order; var slot_price_target = wo.price(); var order_type = wo.order_type(); var filled = wo.filled(); @@ -186,10 +192,9 @@ namespace mbts.spark { var traversal_candidate = only_order; bool exit_early = false; while (!exit_early && traversal_candidate.ParentKey != 0) { - var found_parent = working.Where(o => o.Key == traversal_candidate.ParentKey); - if (found_parent.Count() == 1) - traversal_candidate = found_parent.First().Value; - else + if (working.TryGetFirst(o => o.Key == traversal_candidate.ParentKey, out var first)) + traversal_candidate = first.Value.order; + else exit_early = true; } if (!exit_early) diff --git a/src/client/spark_api/api/Update.cs b/src/client/spark_api/api/Update.cs index b2585f8ad..e97fc196b 100644 --- a/src/client/spark_api/api/Update.cs +++ b/src/client/spark_api/api/Update.cs @@ -212,11 +212,13 @@ namespace bts.api.core { ImmutableDictionary init, IEnumerable added, IEnumerable> changed, - IEnumerable removed) { + IEnumerable removed, + Func maybe_key_extractor) { _state = init; this.added = added; this.changes = changed; this.removed = removed; + this.key_extractor = maybe_key_extractor; } public IReadOnlyDictionary state { get { return _state; } } @@ -233,12 +235,13 @@ namespace bts.api.core { foreach (var i in update.changed) { builder[update.key(i)] = i; } - return new TableUpdate(builder.ToImmutableDictionary(), update.added, update.changes, update.removed); + return new TableUpdate(builder.ToImmutableDictionary(), update.added, update.changes, update.removed, update.key_extractor); } internal TableUpdate as_snapshot() { - return new TableUpdate(_state, _state.Values.ToList(), new Change[0], new T[0]); + return new TableUpdate(_state, _state.Values.ToList(), new Change[0], new T[0], key_extractor); } + internal Func key_extractor { get; private set; } } internal class LegacyTableUpdate where TKey : IEquatable { @@ -287,7 +290,7 @@ namespace bts.api.core { public static IObservable> Table(this IObservable> source) where TKey : IEquatable => source.Scan( - new TableUpdate(ImmutableDictionary.Empty, new T[0], new Change[0], new T[0]), + new TableUpdate(ImmutableDictionary.Empty, new T[0], new Change[0], new T[0], null), (table, update) => TableUpdate.update(table,update)); // Fold updates and emit at max 1/interval @@ -450,8 +453,9 @@ namespace bts.api.core { } cache = builder.ToImmutable(); - return new TableUpdate(cache, added, changes, removed); + return new TableUpdate(cache, added, changes, removed, key); }); } + } } \ No newline at end of file diff --git a/src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs b/src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs index 5f7576ad8..415cb0ef3 100644 --- a/src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs +++ b/src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs @@ -9,6 +9,14 @@ namespace System.Collections.Generic { else update_value(key, value); } + public static bool TryAdd(this IDictionary dict, TKey key, TValue value) { + if (dict.ContainsKey(key)) return false; + else { + dict[key] = value; + return true; + } + } + public static void AddOrUpdate(this IDictionary dictionary, TKey key, Func add_value_factory, Func update_value) { if (!dictionary.TryGetValue(key, out TValue value)) dictionary.Add(key, add_value_factory(key)); @@ -77,5 +85,20 @@ namespace System.Collections.Generic { /// true if the dictionary contains an element with the specified key, AND returns true; otherwise, false. public static bool TryPopValueIf(this IDictionary dictionary, TKey key, Func should_pop) => dictionary.TryGetValue(key, out TValue result) && should_pop(result) && dictionary.Remove(key); + + + public static T? TryGetValue(this IDictionary src, TKey key) where T : struct { + if (src.TryGetValue(key, out T value)) { + return value; + } + return null; + } + public static T? TryGetValue(this IReadOnlyDictionary src, TKey key) where T : struct { + if (src.TryGetValue(key, out T value)) { + return value; + } + return null; + } + } } \ No newline at end of file diff --git a/src/client/spark_core/utils/FilteredDictionary.cs b/src/client/spark_core/utils/FilteredDictionary.cs new file mode 100644 index 000000000..9ca45bbc4 --- /dev/null +++ b/src/client/spark_core/utils/FilteredDictionary.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace mbts.utils { + public class FilteredDictionary : IReadOnlyDictionary { + readonly IReadOnlyDictionary _dict; + readonly Func _filter; + int? _count; + + public FilteredDictionary(IReadOnlyDictionary src, Func filter) { + _dict = src; + _filter = filter; + _count = null; + } + + public T this[TKey key] { + get { + var result = _dict[key]; + if (_filter(result)) { + return result; + } + throw new KeyNotFoundException(key.ToString()); + } + } + + public IEnumerable Keys => _dict.Where(kvp => _filter(kvp.Value)).Select(kvp => kvp.Key); + + public IEnumerable Values => _dict.Values.Where(_filter); + + public int Count { + get { + if (_count.HasValue) return _count.Value; + _count = Values.Count(); + return _count.Value; + } + } + + public bool ContainsKey(TKey key) => Keys.Contains(key); + + public IEnumerator> GetEnumerator() { + return new FilteredEnumerator>(_dict.GetEnumerator(), kvp => _filter(kvp.Value)); + } + + public bool TryGetValue(TKey key, out T value) { + if (_dict.TryGetValue(key, out var val)) { + if (_filter(val)) { + value = val; + return true; + } + } + value = default; + return false; + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + public class FilteredEnumerator : IEnumerator { + readonly Func _filter; + readonly IEnumerator _src; + + public FilteredEnumerator(IEnumerator src, Func filter) { + _filter = filter; + _src = src; + } + public T Current => _src.Current; + + object IEnumerator.Current => Current; + + public void Dispose() => _src.Dispose(); + + public bool MoveNext() { + do { + var ok = _src.MoveNext(); + if (!ok) return false; + } while (!_filter(_src.Current)); + return true; + } + + public void Reset() { _src.MoveNext(); } + } + + +} \ No newline at end of file diff --git a/src/client/spark_core/utils/MappedDictionary.cs b/src/client/spark_core/utils/MappedDictionary.cs new file mode 100644 index 000000000..904c69252 --- /dev/null +++ b/src/client/spark_core/utils/MappedDictionary.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace mbts.utils { + internal class MappedDictionary : IReadOnlyDictionary { + readonly IReadOnlyDictionary _dict; + readonly Func _map; + + public MappedDictionary(IReadOnlyDictionary src, Func map) { + _dict = src; + _map = map; + } + + public TRes this[TKey key] => _map(_dict[key]); + + public IEnumerable Keys => _dict.Keys; + + public IEnumerable Values => _dict.Values.Select(_map); + + public int Count => _dict.Count; + + public bool ContainsKey(TKey key) => _dict.ContainsKey(key); + + public IEnumerator> GetEnumerator() { + return new MappedEnumerator, KeyValuePair>( + _dict.GetEnumerator(), + kvp => new KeyValuePair(kvp.Key, _map(kvp.Value))); + } + + public bool TryGetValue(TKey key, out TRes value) { + if (_dict.TryGetValue(key, out TSrc src)) { + value = _map(src); + return true; + } + value = default; + return false; + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + public class MappedEnumerator : IEnumerator { + readonly IEnumerator _src; + readonly Func _map; + + public MappedEnumerator(IEnumerator src, Func map) { + _src = src; + _map = map; + } + + public TRes Current => _map(_src.Current); + + object IEnumerator.Current => Current; + + public void Dispose() { _src.Dispose(); } + + public bool MoveNext() => _src.MoveNext(); + + public void Reset() { _src.Reset(); } + } + +} \ No newline at end of file From c7b890eeb0fd14d832a71e3b3fdba65ff04ab115 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 28 Jan 2019 09:41:55 -0500 Subject: [PATCH 28/54] Add tests for Filtered/Mapped dictionary types --- .../Modules/Trading/OrderBook/OrderBookVM.cs | 3 +- src/client/spark/Orders/OrdersModel.cs | 4 +- .../core/Utility/TableUpdateTransform.cs | 1 + .../spark_core/TestRunner/TestAssert.cs | 20 +++++ .../spark_core/utils/FilteredDictionary.cs | 2 +- .../spark_core/utils/MappedDictionary.cs | 2 +- .../Utils/FilteredDictionaryTests.cs | 85 +++++++++++++++++++ .../spark_test/Utils/MappedDictionaryTests.cs | 71 ++++++++++++++++ 8 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 src/client/spark_test/Utils/FilteredDictionaryTests.cs create mode 100644 src/client/spark_test/Utils/MappedDictionaryTests.cs diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index 1b4a5afbf..534bf0a02 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -43,7 +43,8 @@ namespace mbts.spark { _bus = bus; _model = model; _settings_manager = settings_manager; - _columns = new OrderBookColumnCollection(bus, model, model.on_order_securities.Select(u => (u.added.Values.Select(smd => smd.security), u.removed.Values.Select(smd => smd.security)))); + _columns = new OrderBookColumnCollection(bus, model, + model.on_order_securities.Select(x => (x.added.Values, x.removed.Values))); _row_collection = new OrderBookRowCollection(ort => new OrderBookRowVM(this, ort), _columns.get_sorting_data().comparer) { filter = _columns.get_filter() }; _current_row = backing(nameof(current_row)); _columns.filter_changed += on_filter_changed; diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index bbf337718..c5892f4a4 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -62,8 +62,8 @@ namespace mbts.spark { .Replay(1).RefCount(); on_order_securities = - on_orders. - as_iincremental() + on_orders + .as_iincremental() .filter(t => t.sec.HasValue) .Select(t => t.sec.Value); diff --git a/src/client/spark/core/Utility/TableUpdateTransform.cs b/src/client/spark/core/Utility/TableUpdateTransform.cs index fbd3da11f..c733d1202 100644 --- a/src/client/spark/core/Utility/TableUpdateTransform.cs +++ b/src/client/spark/core/Utility/TableUpdateTransform.cs @@ -1,4 +1,5 @@ using bts.api.core; +using bts.collections; using mbts.collections; using System; using System.Collections.Generic; diff --git a/src/client/spark_core/TestRunner/TestAssert.cs b/src/client/spark_core/TestRunner/TestAssert.cs index 5bc09ae02..0306073cc 100644 --- a/src/client/spark_core/TestRunner/TestAssert.cs +++ b/src/client/spark_core/TestRunner/TestAssert.cs @@ -65,5 +65,25 @@ namespace bts.test { $"Expected: {item} in [{aa}]. {msg}", file, line); } } + + public static void throws(Action runme, string msg="", [CallerFilePath] string file="", [CallerLineNumber] int line=0) where T : Exception { + bool threw = false; + try { + runme(); + } + catch (T) { + threw = true; + } + catch (Exception e) { + throw new AssertFailture( + $"Expected '{typeof(T).Name}' but threw '{e.GetType().Name}'", file, line); + } + + if (!threw) { + throw new AssertFailture( + msg ?? $"Expected '{typeof(T).Name}' exception but ran successfully.", file, line); + } + + } } } \ No newline at end of file diff --git a/src/client/spark_core/utils/FilteredDictionary.cs b/src/client/spark_core/utils/FilteredDictionary.cs index 9ca45bbc4..cb0ca57f1 100644 --- a/src/client/spark_core/utils/FilteredDictionary.cs +++ b/src/client/spark_core/utils/FilteredDictionary.cs @@ -3,7 +3,7 @@ using System.Collections; using System.Collections.Generic; using System.Linq; -namespace mbts.utils { +namespace bts.collections { public class FilteredDictionary : IReadOnlyDictionary { readonly IReadOnlyDictionary _dict; readonly Func _filter; diff --git a/src/client/spark_core/utils/MappedDictionary.cs b/src/client/spark_core/utils/MappedDictionary.cs index 904c69252..d66ce775f 100644 --- a/src/client/spark_core/utils/MappedDictionary.cs +++ b/src/client/spark_core/utils/MappedDictionary.cs @@ -3,7 +3,7 @@ using System.Collections; using System.Collections.Generic; using System.Linq; -namespace mbts.utils { +namespace bts.collections { internal class MappedDictionary : IReadOnlyDictionary { readonly IReadOnlyDictionary _dict; readonly Func _map; diff --git a/src/client/spark_test/Utils/FilteredDictionaryTests.cs b/src/client/spark_test/Utils/FilteredDictionaryTests.cs new file mode 100644 index 000000000..8a99f1c7a --- /dev/null +++ b/src/client/spark_test/Utils/FilteredDictionaryTests.cs @@ -0,0 +1,85 @@ +using bts.collections; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace bts.test { + + [TestGroup] + public class FilteredDictionaryTests { + IEnumerable range = Enumerable.Range(5,6); + Dictionary source; + IReadOnlyDictionary filtered; + + public FilteredDictionaryTests() { + // Create a source dictionary + source = Enumerable.Range(1, 10).ToDictionary(x => x); + // create a filtered dictionary + filtered = new FilteredDictionary(source, x => x >= 5); + } + + void ignore(T t) { } + + [Test] + public void index() { + foreach (var i in Enumerable.Range(-10,20)) { + if (range.Contains(i)) { + Assert.AreEqual(i, filtered[i]); + } + else { + Assert.throws(() => ignore(filtered[i])); + } + } + } + [Test] + public void keys() { + Assert.SequenceEqual(range, filtered.Keys); + } + [Test] + public void values() { + Assert.SequenceEqual(range, filtered.Values); + } + [Test] + public void count() { + Assert.AreEqual(range.Count(), filtered.Count); + Assert.AreEqual(range.Count(), filtered.Count()); + } + [Test] + public void contains_key() { + foreach (var i in Enumerable.Range(-10, 20)) { + if (range.Contains(i)) { + Assert.IsTrue(filtered.ContainsKey(i)); + } + else { + Assert.IsFalse(filtered.ContainsKey(i)); + } + } + } + [Test] + public void enumerator() { + Assert.SequenceEqual( + range.Select(i => new KeyValuePair(i, i)), filtered); + } + [Test] + public void try_get_value() { + foreach (var i in Enumerable.Range(-10, 20)) { + if (range.Contains(i)) { + Assert.IsTrue(filtered.TryGetValue(i, out var _)); + } + else { + Assert.IsFalse(filtered.TryGetValue(i, out var _)); + } + } + } + + [Test] + public void FilterAndMap() { + string transform(int i) => (i * i).ToString(); + var mapped = new MappedDictionary(filtered, transform); + Assert.SequenceEqual( + range.Select(transform), + mapped.Values); + + } + } +} diff --git a/src/client/spark_test/Utils/MappedDictionaryTests.cs b/src/client/spark_test/Utils/MappedDictionaryTests.cs new file mode 100644 index 000000000..e426de542 --- /dev/null +++ b/src/client/spark_test/Utils/MappedDictionaryTests.cs @@ -0,0 +1,71 @@ +using bts.collections; +using System.Collections.Generic; +using System.Linq; + +namespace bts.test { + + [TestGroup] + public class MappedDictionaryTests { + IEnumerable range = Enumerable.Range(1, 10); + Dictionary source; + IReadOnlyDictionary mapped; + + public MappedDictionaryTests() { + // Create a source dictionary + source = range.ToDictionary(x => x); + // create a filtered dictionary + mapped = new MappedDictionary(source, x => x.ToString()); + } + + [Test] + public void index() { + foreach(var i in range) { + Assert.AreEqual(i.ToString(), mapped[i]); + } + } + [Test] + public void keys() { + Assert.SequenceEqual(range, mapped.Keys); + } + [Test] + public void values() { + Assert.SequenceEqual(range.Select(x => x.ToString()), mapped.Values); + } + [Test] + public void count() { + Assert.AreEqual(range.Count(), mapped.Count); + } + [Test] + public void contains_key() { + foreach (var key in range) { + Assert.IsTrue(mapped.ContainsKey(key)); + } + Assert.IsFalse(mapped.ContainsKey(range.Max() + 1)); + Assert.IsFalse(mapped.ContainsKey(range.Min() - 1)); + } + [Test] + public void enumerator() { + Assert.SequenceEqual(range, mapped.Select(kvp => kvp.Key)); + Assert.SequenceEqual( + range.Select(x => x.ToString()), + mapped.Select(kvp => kvp.Value)); + } + [Test] + public void try_get_value() { + foreach (var i in Enumerable.Range(-20, 20)) { + Assert.AreEqual( + range.Contains(i), + mapped.TryGetValue(i, out string _)); + } + } + [Test] + public void MapAndFilter() { + var set = new HashSet(new[] { "4", "5" }); + var filtered = new FilteredDictionary(mapped, set.Contains); + Assert.AreEqual(2, filtered.Count); + Assert.SequenceEqual( + Enumerable.Range(4, 2).Select(i => (i, i.ToString())), + filtered.Select(kvp => (kvp.Key, kvp.Value))); + } + } +} From 5cc408557727828ad09536b625dbf1f1cb6fc7ff Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 28 Jan 2019 16:50:27 -0500 Subject: [PATCH 29/54] Try to ensure Dispatcher::background is terminated --- src/lib/utils/init_spark_lib.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/utils/init_spark_lib.cpp b/src/lib/utils/init_spark_lib.cpp index d9dc6fab9..772d62c7a 100644 --- a/src/lib/utils/init_spark_lib.cpp +++ b/src/lib/utils/init_spark_lib.cpp @@ -2,6 +2,7 @@ #include "lib/logging/logging.h" #include "lib/utils/bts_assert.h" #include "lib/utils/thread_manager.h" +#include "lib/utils/Dispatcher.h" namespace bts { namespace utils { @@ -20,6 +21,7 @@ void finit_spark_lib() if (poller) { delete poller; } + utils::Dispatcher::background().terminate(); } }} From 51704be22a870565c97f6bd36bb9215592325b09 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 28 Jan 2019 17:27:42 -0500 Subject: [PATCH 30/54] Avoid creating task that does no work. --- src/client/spark/Security/SecurityStore.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 3b0f53794..731ca169f 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -112,14 +112,16 @@ namespace bts { } } // Update cache in the background - Task.Run(async () => { - var r = await _md.get_securities_async(missing); - if (r.success) { - foreach (var security in r.value.Security) { - try_cache_security(new SecurityImmutable(security)); + if (missing.Keys.Count > 0) { + Task.Run(async () => { + var r = await _md.get_securities_async(missing); + if (r.success) { + foreach (var security in r.value.Security) { + try_cache_security(new SecurityImmutable(security)); + } } - } - }); + }); + } return result; } From 6b3e2f396d4840c563330a23ff4b36a0f25aa283 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 28 Jan 2019 17:27:53 -0500 Subject: [PATCH 31/54] Update ReplayRecorder version --- src/lib/md/relay/RelayRecorder.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/md/relay/RelayRecorder.cpp b/src/lib/md/relay/RelayRecorder.cpp index a6f453e73..6b6d3f020 100644 --- a/src/lib/md/relay/RelayRecorder.cpp +++ b/src/lib/md/relay/RelayRecorder.cpp @@ -26,6 +26,7 @@ namespace bts { namespace md { const char* magic = "Co2"; +const uint32_t recorder_version = 3; template void writebuf(ofstream& out, const T& pb) { @@ -61,7 +62,7 @@ RelayRecorder::RelayRecorder(const std::string& base_path, const std::string& fi CoconutReplayHeader header; header.set_magic(magic); - header.set_version(3); + header.set_version(recorder_version); header.set_timestamp(datetime::ll_now()); header.set_host(config::get().config_host()); header.set_naggle_ms(naggle); @@ -132,7 +133,7 @@ void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) { #ifndef _WIN32 md::SecurityIDRecorder idrec {{}}; #endif - if (readbuf(in, &header) && header.magic() == magic && header.version() == 3) { + if (readbuf(in, &header) && header.magic() == magic && header.version() == recorder_version) { co2::MDReply message; while (readbuf(in, &message)) { bool has_securities = false; @@ -226,7 +227,7 @@ void unprint_recording(const std::string& src, const std::string& dest) { // Write out the header CoconutReplayHeader header; header.set_magic(magic); - header.set_version(2); + header.set_version(recorder_version); header.set_timestamp(datetime::ll_now()); header.set_host(config::get().host_name()); writebuf(out, header); From ab1fe94b88da7b1c994f8a81c6bcdb4a7b93fcd2 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 30 Jan 2019 10:40:23 -0500 Subject: [PATCH 32/54] SPARK-1955 Introduce MDClient and SecurityOracle types --- src/lib/md/MDClient.cpp | 74 +++++++++++++++++++++++++++++++++++ src/lib/md/MDClient.h | 26 ++++++++++++ src/lib/md/SecurityOracle.cpp | 31 +++++++++++++++ src/lib/md/SecurityOracle.h | 19 +++++++++ 4 files changed, 150 insertions(+) create mode 100644 src/lib/md/MDClient.cpp create mode 100644 src/lib/md/MDClient.h create mode 100644 src/lib/md/SecurityOracle.cpp create mode 100644 src/lib/md/SecurityOracle.h diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp new file mode 100644 index 000000000..0463f6810 --- /dev/null +++ b/src/lib/md/MDClient.cpp @@ -0,0 +1,74 @@ +#include "MDClient.h" +#include "get_symbols.h" +#include + +namespace bts { +namespace md { + +struct MDClientImpl{ + utils::Dispatcher& _dispatcher; + std::shared_ptr _ss; + utils::DispatcherTimer _security_timer; + utils::DispatcherTimer _recovery_timer; + MDConsumer _consumer; + utils::Event _event; + + MDClientImpl(utils::Dispatcher& d, std::shared_ptr s): + _dispatcher(d), + _ss(s), + _security_timer(d, config::get().security_scan_interval(),[this]() { scan_securities(); }), + _recovery_timer(d, config::get().recovery_interval(), [this]() { check_heartbeats(); }), + _consumer(s,true), + _event() + { + _security_timer.start(); + _recovery_timer.start(); + } + + std::set symbols() { + std::set all_symbols; + Venue venue; + bts_assert(api::Venue_Parse(config::get().exchange(), &venue)); + SecurityType type; + bts_assert(api::SecurityType_Parse(config::get().type(), &type)); + get_system_md_symbols(venue, type, all_symbols); + return all_symbols; + } + + void check_heartbeats() { + if (!_consumer.all_heartbeats_ok()) { + MDConsumer temp(symbols(), _ss); + if (temp.all_heartbeats_ok()) { + _consumer.swap(temp); + LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); + // Subscribers are responsible for keeping the old consumer reference if they need it + _event.publish(_consumer); + } + } + } + + void scan_securities() { + if (_consumer.all_heartbeats_ok()) { + _consumer.scan_securities(); + } + } +}; + + +MDClient::MDClient(utils::Dispatcher& d, std::shared_ptr s) : _self(new MDClientImpl(d,s)) { + +} +MDClient::~MDClient() { +} + +utils::EventSubscriber& MDClient::on_consumer() { + return _self->_event; +} +MDConsumer MDClient::current_consumer() const { + return _self->_consumer; +} +SecurityOracle MDClient::oracle() { + return SecurityOracle(*this); +} + +}}//namespace \ No newline at end of file diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h new file mode 100644 index 000000000..b82549ae7 --- /dev/null +++ b/src/lib/md/MDClient.h @@ -0,0 +1,26 @@ +#pragma once +#include "MDConsumer.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/Event.h" +#include "ISyntheticSecurityService.h" +#include "SecurityOracle.h" +#include + +namespace bts { +namespace md { + +struct MDClientImpl; +class MDClient { + std::unique_ptr _self; + +public: + MDClient(utils::Dispatcher&, std::shared_ptr); + ~MDClient(); + + utils::EventSubscriber& on_consumer(); + MDConsumer current_consumer() const; + SecurityOracle oracle(); +}; + + +}}//namespace \ No newline at end of file diff --git a/src/lib/md/SecurityOracle.cpp b/src/lib/md/SecurityOracle.cpp new file mode 100644 index 000000000..4d49affe6 --- /dev/null +++ b/src/lib/md/SecurityOracle.cpp @@ -0,0 +1,31 @@ +#include "SecurityOracle.h" +#include "MDClient.h" + +namespace bts { +namespace md { + +struct SecurityOracleImpl { + SecurityOracleImpl(MDClient& client): + client(client), + consumer(client.current_consumer()) + { + sub = client.on_consumer().subscribe([this](const MDConsumer& c) { + consumer = c; + }); + } + ~SecurityOracleImpl() { + client.on_consumer().unsubscribe(sub); + } + MDClient& client; + MDConsumer consumer; + int sub; + +}; + +SecurityOracle::SecurityOracle(MDClient& client): _self(new SecurityOracleImpl(client)) { } +SecurityOracle::~SecurityOracle() { } +SecurityPtr SecurityOracle::get_security(SecurityKey key) const { + return _self->consumer.get_security(key); +} + +}}//namespace \ No newline at end of file diff --git a/src/lib/md/SecurityOracle.h b/src/lib/md/SecurityOracle.h new file mode 100644 index 000000000..fc9ec3ea6 --- /dev/null +++ b/src/lib/md/SecurityOracle.h @@ -0,0 +1,19 @@ +#pragma once +#include "SecurityPtr.h" +#include + +namespace bts { +namespace md { + +class MDClient; + +struct SecurityOracleImpl; +class SecurityOracle { + std::unique_ptr _self; +public: + SecurityOracle(MDClient&); + ~SecurityOracle(); + SecurityPtr get_security(SecurityKey key) const; +}; + +}}//namespace \ No newline at end of file From 791b0fdbc1916928f7e2cb8f450f1a8c681496c4 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 31 Jan 2019 09:18:03 -0500 Subject: [PATCH 33/54] Makd SecurityCache external to MDConsumer so it live beyond md recovery --- src/lib/md/Diag/MDPeriodicDumper.cpp | 4 +- src/lib/md/MDClient.cpp | 6 ++- src/lib/md/MDConsumer.cpp | 23 ++++++------ src/lib/md/MDConsumer.h | 12 +++--- src/lib/md/MDRecoverableConsumer.cpp | 2 +- src/lib/md/MDRecoverableConsumer.h | 5 ++- src/lib/md/SecurityCache.cpp | 2 +- src/lib/md/SecurityCache.h | 2 +- src/lib/md/SecurityOracle.cpp | 2 +- src/lib/md/SecurityOracle.h | 2 +- src/lib/md/relay/QueryServer.cpp | 8 +++- src/lib/md/test/md2_tests.cpp | 37 +++++++++++++------ .../orderman/server/OrderServiceServer.cpp | 3 +- 13 files changed, 68 insertions(+), 40 deletions(-) diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index bc964116c..05cdadb06 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -1,5 +1,6 @@ #include "MDPeriodicDumper.h" #include "lib/md/MDConsumer.h" +#include "lib/md/SecurityCache.h" #include "lib/utils/TimeStamp.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/datetime.h" @@ -158,7 +159,8 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, auto thread_name = "tail-" + qualifier; threading::configure_current_thread(thread_name); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); // read all securities into a vector std::vector entries; diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index 0463f6810..a0f5313b0 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -1,5 +1,6 @@ #include "MDClient.h" #include "get_symbols.h" +#include "SecurityCache.h" #include namespace bts { @@ -8,6 +9,7 @@ namespace md { struct MDClientImpl{ utils::Dispatcher& _dispatcher; std::shared_ptr _ss; + SecurityCache _cache; utils::DispatcherTimer _security_timer; utils::DispatcherTimer _recovery_timer; MDConsumer _consumer; @@ -18,7 +20,7 @@ struct MDClientImpl{ _ss(s), _security_timer(d, config::get().security_scan_interval(),[this]() { scan_securities(); }), _recovery_timer(d, config::get().recovery_interval(), [this]() { check_heartbeats(); }), - _consumer(s,true), + _consumer(s, _cache), _event() { _security_timer.start(); @@ -37,7 +39,7 @@ struct MDClientImpl{ void check_heartbeats() { if (!_consumer.all_heartbeats_ok()) { - MDConsumer temp(symbols(), _ss); + MDConsumer temp(_ss,_cache); if (temp.all_heartbeats_ok()) { _consumer.swap(temp); LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 449bfe420..d01908032 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -537,20 +537,21 @@ class MDConsumerImpl { std::shared_ptr _sci; map> _deferred_handles; vector _callbacks; - mutable SecurityCache _cache; + SecurityCache& _cache; // co2::Platform _platform; public: MDConsumerImpl( const set& symbols, std::shared_ptr& ss, + SecurityCache& cache, bool shared, co2::Platform platform = co2::PLATFORM_UNSET) : _workmutex(), _instances(), _sci(), _deferred_handles(), _callbacks(), - _cache() + _cache(cache) { std::ostringstream msg; for (const auto& symbol : symbols) { @@ -906,27 +907,27 @@ public: // MDConsumer public interface, forwards to pimple /////////////////////////////////////////////////////////////////////////////////////////////////// -MDConsumer::MDConsumer(std::shared_ptr& synt, bool shared) : _pimpl() { +MDConsumer::MDConsumer(std::shared_ptr& synt, SecurityCache& cache, bool shared) : _pimpl() { set all_symbols; get_system_md_symbols(VENUE_UNSET, SECURITYTYPE_UNSET, all_symbols); - _pimpl.reset(new MDConsumerImpl(all_symbols, synt, shared, co2::PLATFORM_UNSET)); + _pimpl.reset(new MDConsumerImpl(all_symbols, synt, cache, shared, co2::PLATFORM_UNSET)); } -MDConsumer::MDConsumer(const UserSymbol& s, std::shared_ptr synt, bool shared) : +MDConsumer::MDConsumer(const UserSymbol& s, std::shared_ptr synt, SecurityCache& cache, bool shared) : _pimpl() { set symbols; symbols.insert(s); - _pimpl.reset(new MDConsumerImpl(symbols, synt, shared, co2::PLATFORM_UNSET)); + _pimpl.reset(new MDConsumerImpl(symbols, synt, cache, shared, co2::PLATFORM_UNSET)); } -MDConsumer::MDConsumer(set const& symbols, std::shared_ptr s, bool shared, co2::Platform platform) : - _pimpl(new MDConsumerImpl(symbols, s, shared, platform)) {} +MDConsumer::MDConsumer(set const& symbols, std::shared_ptr s, SecurityCache& cache, bool shared, co2::Platform platform) : + _pimpl(new MDConsumerImpl(symbols, s, cache, shared, platform)) {} -MDConsumer::MDConsumer(api::Venue ven, co2::Platform platform) : _pimpl() { +MDConsumer::MDConsumer(SecurityCache& cache, api::Venue ven, co2::Platform platform) : _pimpl() { set all_symbols; get_system_md_symbols(ven, SECURITYTYPE_UNSET, all_symbols); std::shared_ptr null = nullptr; - _pimpl.reset(new MDConsumerImpl(all_symbols, null, true, platform)); + _pimpl.reset(new MDConsumerImpl(all_symbols, null, cache, true, platform)); } MDConsumer::~MDConsumer() {} @@ -944,7 +945,7 @@ size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol, size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); } void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); } -SecurityPtr MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key); } +const api::Security* MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key).get(); } MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType type, Venue exchange) const { return _pimpl->get_channel(symbol, type, exchange); diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index a53b33665..dff515edc 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -25,15 +25,17 @@ using std::size_t; class MDConsumerImpl; class MDConsumerInst; class MDConsumerChannel; +class SecurityCache; + class MDConsumer { public: // Consumer for all symbols - MDConsumer(std::shared_ptr&, bool shared=true); - MDConsumer(const UserSymbol&, std::shared_ptr, bool shared=true); + MDConsumer(std::shared_ptr&, SecurityCache&, bool shared=true); + MDConsumer(const UserSymbol&, std::shared_ptr, SecurityCache&, bool shared=true); // Consumer for a given set of symbols - MDConsumer(std::set const& symbols, std::shared_ptr, bool shared=true, co2::Platform platform=co2::PLATFORM_UNSET); - MDConsumer(api::Venue, co2::Platform); + MDConsumer(std::set const& symbols, std::shared_ptr, SecurityCache&, bool shared=true, co2::Platform platform=co2::PLATFORM_UNSET); + MDConsumer(SecurityCache&, api::Venue, co2::Platform); ~MDConsumer(); bool all_heartbeats_ok() const; @@ -53,7 +55,7 @@ public: // Retrieve any security that has been seen in the past // You still need to call scan_securities() periodically to discover new securities - SecurityPtr get_security(SecurityKey key) const; + const api::Security* get_security(SecurityKey key) const; MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const; MDConsumerChannel get_channel(const UserSymbol&) const; diff --git a/src/lib/md/MDRecoverableConsumer.cpp b/src/lib/md/MDRecoverableConsumer.cpp index 9e6250388..5b5378033 100644 --- a/src/lib/md/MDRecoverableConsumer.cpp +++ b/src/lib/md/MDRecoverableConsumer.cpp @@ -45,7 +45,7 @@ MDRecoverableConsumer::MDRecoverableConsumer(std::set symbols, std:: MDRecoverableConsumer::~MDRecoverableConsumer() {} unique_ptr MDRecoverableConsumer::init(std::set& symbols, std::shared_ptr& ss) { - return std::make_unique(symbols, ss); + return std::make_unique(symbols, ss, _cache); } size_t MDRecoverableConsumer::walk_security_list(SecurityCallback callback) { diff --git a/src/lib/md/MDRecoverableConsumer.h b/src/lib/md/MDRecoverableConsumer.h index 95068587d..c773b5d4d 100644 --- a/src/lib/md/MDRecoverableConsumer.h +++ b/src/lib/md/MDRecoverableConsumer.h @@ -5,6 +5,7 @@ #include "MDConsumer.h" #include "MDConsumerHandle.h" #include "UserSymbol.h" +#include "SecurityCache.h" #include #include #include @@ -45,7 +46,8 @@ private: struct MDConsumerGroup { MDConsumerGroup( std::set& symbols, - std::shared_ptr& ss) : consumer(symbols, ss), handles() {} + std::shared_ptr& ss, + SecurityCache& cache) : consumer(symbols, ss, cache), handles() {} MDConsumer consumer; std::vector> handles; }; @@ -53,6 +55,7 @@ private: std::set _symbols; std::shared_ptr _ss; + SecurityCache _cache; std::unique_ptr _current; std::vector> _old; diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp index affe44c9e..6d5fd1b43 100644 --- a/src/lib/md/SecurityCache.cpp +++ b/src/lib/md/SecurityCache.cpp @@ -37,7 +37,7 @@ void SecurityCacheImpl::flush() { ////////////////// -SecurityCache::SecurityCache() : _me(new SecurityCacheImpl()) {} +SecurityCache::SecurityCache() : _me(std::make_shared()) {} SecurityCache::~SecurityCache() {} SecurityPtr SecurityCache::try_get_security(SecurityKey key) const { diff --git a/src/lib/md/SecurityCache.h b/src/lib/md/SecurityCache.h index 85de41b70..e4c2ef2c4 100644 --- a/src/lib/md/SecurityCache.h +++ b/src/lib/md/SecurityCache.h @@ -13,7 +13,7 @@ namespace md { // Store security+handle struct SecurityCacheImpl; class SecurityCache { - std::unique_ptr _me; + std::shared_ptr _me; public: SecurityCache(); diff --git a/src/lib/md/SecurityOracle.cpp b/src/lib/md/SecurityOracle.cpp index 4d49affe6..6a7f81ea1 100644 --- a/src/lib/md/SecurityOracle.cpp +++ b/src/lib/md/SecurityOracle.cpp @@ -24,7 +24,7 @@ struct SecurityOracleImpl { SecurityOracle::SecurityOracle(MDClient& client): _self(new SecurityOracleImpl(client)) { } SecurityOracle::~SecurityOracle() { } -SecurityPtr SecurityOracle::get_security(SecurityKey key) const { +const api::Security* SecurityOracle::get_security(SecurityKey key) const { return _self->consumer.get_security(key); } diff --git a/src/lib/md/SecurityOracle.h b/src/lib/md/SecurityOracle.h index fc9ec3ea6..da0210f18 100644 --- a/src/lib/md/SecurityOracle.h +++ b/src/lib/md/SecurityOracle.h @@ -13,7 +13,7 @@ class SecurityOracle { public: SecurityOracle(MDClient&); ~SecurityOracle(); - SecurityPtr get_security(SecurityKey key) const; + const api::Security* get_security(SecurityKey key) const; }; }}//namespace \ No newline at end of file diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 6c3e2e457..783c355e6 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -13,6 +13,7 @@ #include "lib/config/protos/system.pb.h" #include "lib/md/protos/co2.pb.h" +#include "lib/md/SecurityCache.h" #include "lib/api/apicore.pb.h" #include "lib/api/services/MarketDataRelay.server.pb.h" #include "lib/api/public/ServiceContext.h" @@ -162,6 +163,7 @@ void copy_misc(IntPrice settlement, MiscData misc, co2::MiscMessage* pmisc) { // Handles market data queries and responds class QueryServerImpl : public IQueryServer { + SecurityCache _cache; MDConsumer _consumer; const std::set _symbols; const std::shared_ptr _ss; @@ -169,7 +171,8 @@ class QueryServerImpl : public IQueryServer { public: QueryServerImpl(const std::set& symbols, std::shared_ptr& ss, metrics::MetricsContainer& stats) : - _consumer(symbols, ss), + _cache(), + _consumer(symbols, ss, _cache), _symbols(symbols), _ss(ss), _groups() @@ -211,7 +214,8 @@ public: // not-reentrant void check_heartbeats(utils::Dispatcher& dispatcher) { if (!_consumer.all_heartbeats_ok()) { - MDConsumer temp(_symbols, _ss); + SecurityCache cache; + MDConsumer temp(_symbols, _ss, cache); if (temp.all_heartbeats_ok()) { _consumer.swap(temp); LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 9d6ef027e..35c8a828e 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -3,6 +3,7 @@ #include "../SettlementRecorder.h" #include "../coconut2.h" #include "../relay/QueryServer.h" +#include "../SecurityCache.h" #include "Co2TestInstrument.h" #include "lib/config/config.h" #include "lib/os/os.h" @@ -368,7 +369,8 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); vector consumer_securities; // Poll for security list - none visible until they have keys @@ -438,7 +440,8 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); // Add the on_security callback vector securities; @@ -477,7 +480,8 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -534,7 +538,8 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -594,7 +599,8 @@ public: // Symbol list for the consumer auto syms = get_test_symbols(); - MDConsumer consumer = MDConsumer(syms, nullptr); + SecurityCache cache; + MDConsumer consumer = MDConsumer(syms, nullptr, cache); // Add the on_security callback - callback fires for existing securities vector securities; @@ -737,7 +743,8 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); set keys; consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) { keys.insert(security->id()); }); @@ -781,7 +788,8 @@ public: csymbols.insert(UserSymbol("S2", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S3", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S4", VENUE_BTS, SECURITYTYPE_SPREAD)); - MDConsumer consumer(csymbols, nullptr); + SecurityCache cache; + MDConsumer consumer(csymbols, nullptr, cache); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -827,7 +835,8 @@ public: csymbols.insert(UserSymbol("FF", VENUE_BTS, SECURITYTYPE_FUTURE)); csymbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); csymbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); - MDConsumer consumer(csymbols, nullptr); + SecurityCache cache; + MDConsumer consumer(csymbols, nullptr, cache); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -878,7 +887,8 @@ public: set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); consumer_symbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); - MDConsumer consumer(consumer_symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(consumer_symbols, nullptr, cache); size_t security_count = 0; // Indirection for check function @@ -946,7 +956,8 @@ public: auto ph2 = *add_security(o2); set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); - MDConsumer consumer(consumer_symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(consumer_symbols, nullptr, cache); auto h1 = consumer.get_handle(k1); REQUIRE(h1.is_initialized()); auto h2 = consumer.get_handle(k2); @@ -1212,6 +1223,7 @@ struct Co2IndexTests { } ; struct Co2RelayTests { + SecurityCache cache; Co2RelayTests() : _stats() {} void setup() { config::internal::initialize_for_test( @@ -1230,7 +1242,7 @@ struct Co2RelayTests { symbols.insert(future_symbol); UserSymbol option_symbol("FOOX", VENUE_BTS, SECURITYTYPE_OPTION); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + MDConsumer consumer(symbols, nullptr, cache); return consumer; } @@ -1587,7 +1599,8 @@ namespace ImpliedTests { symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); vector consumer_securities; // Poll for security list - none visible until they have keys size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); diff --git a/src/lib/orderman/server/OrderServiceServer.cpp b/src/lib/orderman/server/OrderServiceServer.cpp index 9259a2a7c..0cee13d43 100644 --- a/src/lib/orderman/server/OrderServiceServer.cpp +++ b/src/lib/orderman/server/OrderServiceServer.cpp @@ -92,7 +92,7 @@ struct OrderServiceServer::PImpl { DynamicOrderRegistry const &d, metrics::MetricsContainer &stats, trace::PerfLog &log) : service_bridge(), service_ready(false), known_orders(), known_fills(), publish_orders(), publish_fills(), - admin_service(ctx.locate()), md_consumer(ss), + admin_service(ctx.locate()), cache(), md_consumer(ss, cache), stats(stats), dyn_order_registry(d),log(log) {} ServiceBridge *service_bridge; @@ -106,6 +106,7 @@ struct OrderServiceServer::PImpl { OrderServiceServer::OrdersPublisher publish_orders; OrderServiceServer::FillsPublisher publish_fills; api::admin::AdminService admin_service; + md::SecurityCache cache; bts::md::MDConsumer md_consumer; metrics::MetricsContainer &stats; DynamicOrderRegistry const &dyn_order_registry; From bf72122edccfbb32440e700db8c7b8a28e69ea58 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 31 Jan 2019 11:06:19 -0500 Subject: [PATCH 34/54] SPARK 1731 - Simplify SecurityOracle impl --- src/lib/md/MDClient.cpp | 17 ++++++++++------- src/lib/md/MDClient.h | 6 +++--- src/lib/md/MDConsumerInterop.cpp | 6 ++++-- src/lib/md/SecurityOracle.cpp | 22 ++-------------------- src/lib/md/SecurityOracle.h | 7 ++----- 5 files changed, 21 insertions(+), 37 deletions(-) diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index a0f5313b0..dbe3eab5b 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -1,6 +1,7 @@ #include "MDClient.h" #include "get_symbols.h" #include "SecurityCache.h" +#include "SecurityOracle.h" #include namespace bts { @@ -14,14 +15,16 @@ struct MDClientImpl{ utils::DispatcherTimer _recovery_timer; MDConsumer _consumer; utils::Event _event; + SecurityOracle _oracle; - MDClientImpl(utils::Dispatcher& d, std::shared_ptr s): + MDClientImpl(utils::Dispatcher& d, std::shared_ptr s, SecurityOracle o): _dispatcher(d), _ss(s), _security_timer(d, config::get().security_scan_interval(),[this]() { scan_securities(); }), _recovery_timer(d, config::get().recovery_interval(), [this]() { check_heartbeats(); }), _consumer(s, _cache), - _event() + _event(), + _oracle(o) { _security_timer.start(); _recovery_timer.start(); @@ -57,8 +60,8 @@ struct MDClientImpl{ }; -MDClient::MDClient(utils::Dispatcher& d, std::shared_ptr s) : _self(new MDClientImpl(d,s)) { - +MDClient::MDClient(utils::Dispatcher& d, std::shared_ptr s) : + _self(new MDClientImpl(d,s, SecurityOracle(*this))) { } MDClient::~MDClient() { } @@ -66,11 +69,11 @@ MDClient::~MDClient() { utils::EventSubscriber& MDClient::on_consumer() { return _self->_event; } -MDConsumer MDClient::current_consumer() const { +MDConsumer& MDClient::current_consumer() const { return _self->_consumer; } -SecurityOracle MDClient::oracle() { - return SecurityOracle(*this); +SecurityOracle& MDClient::oracle() const { + return _self->_oracle; } }}//namespace \ No newline at end of file diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h index b82549ae7..e80bb4a1e 100644 --- a/src/lib/md/MDClient.h +++ b/src/lib/md/MDClient.h @@ -3,12 +3,12 @@ #include "lib/utils/Dispatcher.h" #include "lib/utils/Event.h" #include "ISyntheticSecurityService.h" -#include "SecurityOracle.h" #include namespace bts { namespace md { +class SecurityOracle; struct MDClientImpl; class MDClient { std::unique_ptr _self; @@ -18,8 +18,8 @@ public: ~MDClient(); utils::EventSubscriber& on_consumer(); - MDConsumer current_consumer() const; - SecurityOracle oracle(); + MDConsumer& current_consumer() const; + SecurityOracle& oracle() const; }; diff --git a/src/lib/md/MDConsumerInterop.cpp b/src/lib/md/MDConsumerInterop.cpp index 7048354c5..142270e6f 100644 --- a/src/lib/md/MDConsumerInterop.cpp +++ b/src/lib/md/MDConsumerInterop.cpp @@ -2,6 +2,7 @@ #include "common.h" #include "UserSymbol.h" #include "MDConsumer.h" +#include "SecurityCache.h" #include "get_symbols.h" #include "relay/RelayClient.h" #include "lib/utils/Dispatcher.h" @@ -27,15 +28,16 @@ struct BlitUserSymbol { CAPI MDConsumer* create_md_consumer(BlitUserSymbol* usersymbols, uint32_t count, bool shared) { std::shared_ptr pnull = nullptr; + SecurityCache cache; if (usersymbols == nullptr || count == 0) { - return new MDConsumer(pnull, shared); + return new MDConsumer(pnull, cache, shared); } else { std::set symbols; for (size_t x = 0; x < count; ++x) { symbols.insert(UserSymbol(usersymbols[x].symbol, usersymbols[x].exchange, usersymbols[x].type)); } - return new MDConsumer(symbols, pnull, shared); + return new MDConsumer(symbols, pnull, cache, shared); } } diff --git a/src/lib/md/SecurityOracle.cpp b/src/lib/md/SecurityOracle.cpp index 6a7f81ea1..7691f6e4c 100644 --- a/src/lib/md/SecurityOracle.cpp +++ b/src/lib/md/SecurityOracle.cpp @@ -4,28 +4,10 @@ namespace bts { namespace md { -struct SecurityOracleImpl { - SecurityOracleImpl(MDClient& client): - client(client), - consumer(client.current_consumer()) - { - sub = client.on_consumer().subscribe([this](const MDConsumer& c) { - consumer = c; - }); - } - ~SecurityOracleImpl() { - client.on_consumer().unsubscribe(sub); - } - MDClient& client; - MDConsumer consumer; - int sub; - -}; - -SecurityOracle::SecurityOracle(MDClient& client): _self(new SecurityOracleImpl(client)) { } +SecurityOracle::SecurityOracle(const MDClient& client): _client(client) {} SecurityOracle::~SecurityOracle() { } const api::Security* SecurityOracle::get_security(SecurityKey key) const { - return _self->consumer.get_security(key); + return _client.current_consumer().get_security(key); } }}//namespace \ No newline at end of file diff --git a/src/lib/md/SecurityOracle.h b/src/lib/md/SecurityOracle.h index da0210f18..e45ba062f 100644 --- a/src/lib/md/SecurityOracle.h +++ b/src/lib/md/SecurityOracle.h @@ -4,14 +4,11 @@ namespace bts { namespace md { - class MDClient; - -struct SecurityOracleImpl; class SecurityOracle { - std::unique_ptr _self; + const MDClient& _client; public: - SecurityOracle(MDClient&); + SecurityOracle(const MDClient&); ~SecurityOracle(); const api::Security* get_security(SecurityKey key) const; }; From 20f42749dd1a8be2f2a9955732991e8cc5ff7573 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 31 Jan 2019 11:16:25 -0500 Subject: [PATCH 35/54] SPARK-1997 Update OrderServiceServer/FillStore to use SecurityOracle --- src/lib/orderman/actors/FillStore.cpp | 115 +++++++----------- src/lib/orderman/actors/FillStore.h | 3 +- src/lib/orderman/actors/SODPosition.cpp | 10 +- src/lib/orderman/actors/SODPosition.h | 3 +- .../orderman/server/OrderServiceServer.cpp | 15 +-- src/lib/orderman/server/OrderServiceServer.h | 3 +- src/server/sparkd/cmds/Run.cpp | 7 +- 7 files changed, 68 insertions(+), 88 deletions(-) diff --git a/src/lib/orderman/actors/FillStore.cpp b/src/lib/orderman/actors/FillStore.cpp index a7d2fab18..76f1c62a0 100644 --- a/src/lib/orderman/actors/FillStore.cpp +++ b/src/lib/orderman/actors/FillStore.cpp @@ -15,100 +15,71 @@ namespace orderman { auto const fills_ns = "fills"; -enum class FillStore::State { WAIT_SEC, READY }; - struct FillStore::UnresolvedFills { std::vector fills; api::Security const *security; }; struct FillStore::PImpl { - PImpl() - : state(State::WAIT_SEC), store(new persistence::SqliteStore("fills")), - known_orders(), fills_with_unresolved_accounts(), sod_pos_map() {} + PImpl(md::SecurityOracle& oracle) + : store(new persistence::SqliteStore("fills")), + known_orders(), fills_with_unresolved_accounts(), sod_pos_map(), oracle(oracle) {} - State state; std::unique_ptr store; std::unordered_set known_fills; std::unordered_map known_orders; std::unordered_map fills_with_unresolved_accounts; mercury::Stream fill_stream; - mercury::Actor::SubHandle stored_sec_sub; SODPosition sod_pos_map; + md::SecurityOracle oracle; }; -FillStore::FillStore() : me(new PImpl()) {} +FillStore::FillStore(md::SecurityOracle& o) : me(new PImpl(o)) {} void FillStore::init() { - me->state = State::WAIT_SEC; - me->stored_sec_sub = subscribe([this]( - auto const &update) { - if (update.events) { - if (me->state == State::WAIT_SEC) { + // Subscribe to NewExecutionReportEvent for new fills + subscribe( + [this](auto const &ev) { this->on_execution_report(ev); }); - LOG(OrderMan, Info, - "Fill store received initial list of {} securities", - update.events->size()); + // Load stored fills, resolve their securities and publish the lot + api::orderman::Fill fill; + std::vector stored_batch; + LOG(persist, Info, "Loading load all fills"); + auto all = me->store->retrieve_all()[fills_ns]; + LOG(persist, Info, "Finished loading all {} fills", all.size()); + for (auto const &kvp : all) + if (utils::pb::from_string(kvp.second, + &fill)) { - // don't need securities anymore, NewExecutionReportEvent - // events will carry the security with them - unsubscribe(me->stored_sec_sub); + auto sec = me->oracle.get_security(SecurityKey(fill.security_key())); + if (sec) { + auto fk = kvp.first; + if (!me->sod_pos_map.add_fill_to_sod_pos(fill)) + stored_batch.emplace_back(fill, *sec); + me->known_fills.insert(fk); + } else + LOG(persist, Error, + "Stored fill {} contains unresolved security {}. ", + kvp.first, fill.security_key()); + } else + LOG(persist, Error, + "Could not recover fill {} from storage. " + "This should be investigated", + kvp.first); + me->fill_stream = make_stream(); + LOG(OrderMan, Info, + "Fill Store published {} fills recoved from disk out of {} " + "total fills", + stored_batch.size(), all.size()); + me->sod_pos_map.add_sod_pos_as_fill_events(stored_batch, me->oracle); + publish_batch(me->fill_stream, stored_batch); - // Subscribe to NewExecutionReportEvent for new fills - subscribe( - [this](auto const &ev) { this->on_execution_report(ev); }); - - // index securities - std::map sec_idx; - for (auto const &ev : *(update.events)) { - auto sec_ptr = ev.security; - sec_idx[SecurityKey(sec_ptr->id())] = sec_ptr; - } - - // Load stored fills, resolve their securities and publish the lot - api::orderman::Fill fill; - std::vector stored_batch; - LOG(persist, Info, "Loading load all fills"); - auto all = me->store->retrieve_all()[fills_ns]; - LOG(persist, Info, "Finished loading all {} fills", all.size()); - for (auto const &kvp : all) - if (utils::pb::from_string(kvp.second, - &fill)) { - - auto sec = sec_idx[SecurityKey(fill.security_key())]; - if (sec) { - auto fk = kvp.first; - if (!me->sod_pos_map.add_fill_to_sod_pos(fill)) - stored_batch.emplace_back(fill, *sec); - me->known_fills.insert(fk); - } else - LOG(persist, Diag, - "Stored fill {} contains unresolved security {}. ", - kvp.first, fill.security_key()); - } else - LOG(persist, Error, - "Could not recover fill {} from storage. " - "This should be investigated", - kvp.first); - me->fill_stream = make_stream(); - LOG(OrderMan, Info, - "Fill Store published {} fills recoved from disk out of {} " - "total fills", - stored_batch.size(), all.size()); - me->sod_pos_map.add_sod_pos_as_fill_events(stored_batch, sec_idx); - publish_batch(me->fill_stream, stored_batch); - - // subscribe to order updates so we can resolve - // account ids for future execution reports - subscribe( - [this](auto const &ev) { this->on_order_state_update(ev); }); - - me->state = State::READY; - } - } - }); + // subscribe to order updates so we can resolve + // account ids for future execution reports + subscribe( + [this](auto const &ev) { this->on_order_state_update(ev); }); } FillStore::~FillStore() {} diff --git a/src/lib/orderman/actors/FillStore.h b/src/lib/orderman/actors/FillStore.h index 7d1c7b909..d4309e734 100644 --- a/src/lib/orderman/actors/FillStore.h +++ b/src/lib/orderman/actors/FillStore.h @@ -2,6 +2,7 @@ #include #include "lib/mercury/actors/Actor.h" +#include "lib/md/SecurityOracle.h" namespace bts { namespace orderman { @@ -14,7 +15,7 @@ class OrderStateUpdateEvent; class FillStore : public mercury::Actor { public: - FillStore(); + FillStore(md::SecurityOracle&); ~FillStore(); void init() override; diff --git a/src/lib/orderman/actors/SODPosition.cpp b/src/lib/orderman/actors/SODPosition.cpp index ee9515be7..18f6d83c4 100644 --- a/src/lib/orderman/actors/SODPosition.cpp +++ b/src/lib/orderman/actors/SODPosition.cpp @@ -171,22 +171,22 @@ bool SODPosition::add_fill_to_sod_pos(const Fill &fill) { void SODPosition::add_sod_pos_as_fill_events( std::vector &fe, - std::map sec_map) { + md::SecurityOracle& oracle) { if (_enabled) for (auto &acct_p : _sod_map) { for (auto &sec_p : acct_p.second) { Fill &long_pos = sec_p.second.first; Fill &short_pos = sec_p.second.second; bts_assert(long_pos.security_key() == short_pos.security_key()); - auto sec_it = sec_map.find(SecurityKey(long_pos.security_key())); - if (sec_it == sec_map.end()) + auto sec = oracle.get_security(SecurityKey(long_pos.security_key())); + if (!sec) LOG(persist, Error, "Could not recover fill for key:{} from storage. " "This should be investigated", long_pos.security_key()); else { - fe.emplace_back(long_pos, *sec_it->second); - fe.emplace_back(short_pos, *sec_it->second); + fe.emplace_back(long_pos, *sec); + fe.emplace_back(short_pos, *sec); } } } diff --git a/src/lib/orderman/actors/SODPosition.h b/src/lib/orderman/actors/SODPosition.h index d31e39446..860e0cf2d 100644 --- a/src/lib/orderman/actors/SODPosition.h +++ b/src/lib/orderman/actors/SODPosition.h @@ -4,6 +4,7 @@ #include "lib/api/services/OrderService.pb.h" #include "lib/mercury/actors/Actor.h" #include "lib/orderman/events/StoredFillEvent.h" +#include "lib/md/SecurityOracle.h" using namespace bts::api::orderman; @@ -34,7 +35,7 @@ public: static timeval get_order_cutoff_time(const std::map& ordermap); bool add_fill_to_sod_pos(const Fill& fill); void add_sod_pos_as_fill_events(std::vector& fill_events, - std::map sec_map); + md::SecurityOracle&); }; }} diff --git a/src/lib/orderman/server/OrderServiceServer.cpp b/src/lib/orderman/server/OrderServiceServer.cpp index 0cee13d43..caf1c67b2 100644 --- a/src/lib/orderman/server/OrderServiceServer.cpp +++ b/src/lib/orderman/server/OrderServiceServer.cpp @@ -5,6 +5,7 @@ #include "lib/config/protos/jetfix.pb.h" #include "lib/config/protos/orderman_cfg.pb.h" #include "lib/md/MDConsumer.h" +#include "lib/md/MDClient.h" #include "lib/mercury/actors/Timer.h" #include "lib/mercury/executor/Executor.h" #include "lib/metrics/metrics.h" @@ -88,11 +89,11 @@ ActorThread *find_thread_for_actor( struct OrderServiceServer::PImpl { PImpl(bts::api::ServiceContext &ctx, - std::shared_ptr &ss, + md::MDClient& mdc, DynamicOrderRegistry const &d, metrics::MetricsContainer &stats, trace::PerfLog &log) : service_bridge(), service_ready(false), known_orders(), known_fills(), publish_orders(), publish_fills(), - admin_service(ctx.locate()), cache(), md_consumer(ss, cache), + admin_service(ctx.locate()), md_consumer(mdc.current_consumer()), stats(stats), dyn_order_registry(d),log(log) {} ServiceBridge *service_bridge; @@ -106,8 +107,7 @@ struct OrderServiceServer::PImpl { OrderServiceServer::OrdersPublisher publish_orders; OrderServiceServer::FillsPublisher publish_fills; api::admin::AdminService admin_service; - md::SecurityCache cache; - bts::md::MDConsumer md_consumer; + bts::md::MDConsumer& md_consumer; metrics::MetricsContainer &stats; DynamicOrderRegistry const &dyn_order_registry; trace::PerfLog &log; @@ -115,10 +115,10 @@ struct OrderServiceServer::PImpl { OrderServiceServer::OrderServiceServer( bts::api::ServiceContext &ctx, - std::shared_ptr &ss, + md::MDClient& mdc, DynamicOrderRegistry const &d, metrics::MetricsContainer &stats, trace::PerfLog &log) - : api::orderman::OrderServiceBase(ctx), me(new PImpl(ctx, ss, d, stats,log)) { + : api::orderman::OrderServiceBase(ctx), me(new PImpl(ctx, mdc, d, stats,log)) { me->publish_orders = make_orders_publisher(); me->publish_fills = make_fills_publisher(); @@ -138,6 +138,7 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for MDSource. See the " "OrdermanCfg key"); + thread->executor.spawn(me->md_consumer, me->stats,log); // OrderManager @@ -173,7 +174,7 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for FilllStore. See " "the OrdermanCfg key"); - thread->executor.spawn(); + thread->executor.spawn(mdc.oracle()); // SecuritySrc thread = find_thread_for_actor("SecuritySrc", me->actor_threads); diff --git a/src/lib/orderman/server/OrderServiceServer.h b/src/lib/orderman/server/OrderServiceServer.h index 4c23c13f9..75c558932 100644 --- a/src/lib/orderman/server/OrderServiceServer.h +++ b/src/lib/orderman/server/OrderServiceServer.h @@ -3,6 +3,7 @@ #include "lib/api/services/OrderService.pb.h" #include "lib/api/services/OrderService.server.pb.h" #include "lib/mercury/actors/IPollableActor.h" +#include "lib/md/MDClient.h" #include namespace bts { @@ -19,7 +20,7 @@ class DynamicOrderRegistry; class OrderServiceServer : public api::orderman::OrderServiceBase { public: OrderServiceServer(bts::api::ServiceContext &context, - std::shared_ptr &, + md::MDClient&, DynamicOrderRegistry const&, metrics::MetricsContainer &, trace::PerfLog &); diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index a73561dec..a1840cf77 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -11,6 +11,7 @@ #include "lib/utils/Dispatcher.h" #include "lib/config/protos/toggles.pb.h" +#include "lib/md/MDClient.h" #include "lib/md/SyntheticSecurityService.h" #include "lib/md/relay/QueryServer.h" #include "lib/metrics/metrics.h" @@ -56,8 +57,12 @@ int Run::execute(std::vector const &) { persistence::PersistenceServiceServer settings_saver(sc, "settings"); user::AuthServiceServer login_server(sc); md::run_market_datapublishers(); + std::shared_ptr synthetic_service = std::make_shared(sc); + md::MDClient mdclient(utils::Dispatcher::current(), synthetic_service); + + md::QueryServerPtr md_query_server( md::run_query_server(sc, synthetic_service, stats)); @@ -68,7 +73,7 @@ int Run::execute(std::vector const &) { // finished initializing order plugins orderman::OrderServiceServer order_service_server( - sc, synthetic_service, dyn_order_registry, stats, perf_log); + sc, mdclient, dyn_order_registry, stats, perf_log); support::ErrorReportServer error_report_service(sc); admin::AdminServiceServer admin_service(sc); md::auxiliary::MarketDataAuxiliaryServiceServer md_aux_server(sc); From d99d3bccabca409cb817da840d2550d0c068d89b Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 31 Jan 2019 11:52:36 -0500 Subject: [PATCH 36/54] PR feedback #1 --- src/lib/md/MDClient.cpp | 2 +- src/lib/md/MDClient.h | 2 +- src/lib/orderman/actors/FillStore.cpp | 3 +-- src/lib/orderman/actors/FillStore.h | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index dbe3eab5b..851509d20 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -72,7 +72,7 @@ utils::EventSubscriber& MDClient::on_consumer() { MDConsumer& MDClient::current_consumer() const { return _self->_consumer; } -SecurityOracle& MDClient::oracle() const { +SecurityOracle MDClient::oracle() const { return _self->_oracle; } diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h index e80bb4a1e..f9c6b8d3a 100644 --- a/src/lib/md/MDClient.h +++ b/src/lib/md/MDClient.h @@ -19,7 +19,7 @@ public: utils::EventSubscriber& on_consumer(); MDConsumer& current_consumer() const; - SecurityOracle& oracle() const; + SecurityOracle oracle() const; }; diff --git a/src/lib/orderman/actors/FillStore.cpp b/src/lib/orderman/actors/FillStore.cpp index 76f1c62a0..f5f114eb6 100644 --- a/src/lib/orderman/actors/FillStore.cpp +++ b/src/lib/orderman/actors/FillStore.cpp @@ -2,7 +2,6 @@ #include "lib/api/services/OrderService.pb.h" #include "lib/orderman/actors/SODPosition.h" #include "lib/orderman/events/NewExecutionReportEvent.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/events/OrderStateUpdateEvent.h" #include "lib/orderman/events/StoredFillEvent.h" #include "lib/persistence/SqliteStore.h" @@ -35,7 +34,7 @@ struct FillStore::PImpl { md::SecurityOracle oracle; }; -FillStore::FillStore(md::SecurityOracle& o) : me(new PImpl(o)) {} +FillStore::FillStore(md::SecurityOracle o) : me(new PImpl(o)) {} void FillStore::init() { diff --git a/src/lib/orderman/actors/FillStore.h b/src/lib/orderman/actors/FillStore.h index d4309e734..1d35e032c 100644 --- a/src/lib/orderman/actors/FillStore.h +++ b/src/lib/orderman/actors/FillStore.h @@ -15,7 +15,7 @@ class OrderStateUpdateEvent; class FillStore : public mercury::Actor { public: - FillStore(md::SecurityOracle&); + FillStore(md::SecurityOracle); ~FillStore(); void init() override; From 3541ae5a8558d9c76905b6570c53028080078d2a Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 31 Jan 2019 17:05:22 -0500 Subject: [PATCH 37/54] SPARK-1996 QueryServer and constructor refactoring --- src/lib/md/Diag/MDPeriodicDumper.cpp | 2 +- src/lib/md/MDClient.cpp | 77 +++++---- src/lib/md/MDClient.h | 13 +- src/lib/md/MDConsumer.cpp | 39 ++--- src/lib/md/MDConsumer.h | 12 +- src/lib/md/MDConsumerInterop.cpp | 14 +- src/lib/md/MDRecoverableConsumer.cpp | 157 ------------------- src/lib/md/MDRecoverableConsumer.h | 69 -------- src/lib/md/PlatformLoader.cpp | 6 + src/lib/md/SecurityCache.cpp | 1 + src/lib/md/SyntheticSecurityService.h | 6 +- src/lib/md/coconut2.h | 1 - src/lib/md/get_symbols.h | 1 + src/lib/md/relay/QueryServer.cpp | 100 ++++-------- src/lib/md/relay/QueryServer.h | 6 +- src/lib/md/test/md2_tests.cpp | 96 +++++++----- src/lib/orderman/events/NewSecuritiesEvent.h | 4 + src/lib/orderman/server/OrderServiceServer.h | 3 - src/server/sparkd/cmds/Run.cpp | 7 +- src/server/sparkd/cmds/Web.cpp | 12 +- 20 files changed, 202 insertions(+), 424 deletions(-) delete mode 100644 src/lib/md/MDRecoverableConsumer.cpp delete mode 100644 src/lib/md/MDRecoverableConsumer.h diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index 05cdadb06..46e5bfbcd 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -160,7 +160,7 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, threading::configure_current_thread(thread_name); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols); // read all securities into a vector std::vector entries; diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index 851509d20..6822c4d20 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -2,47 +2,47 @@ #include "get_symbols.h" #include "SecurityCache.h" #include "SecurityOracle.h" +#include "lib/utils/TimeStamp.h" #include namespace bts { namespace md { struct MDClientImpl{ - utils::Dispatcher& _dispatcher; - std::shared_ptr _ss; + utils::Dispatcher* _dispatcher; + api::ISyntheticSecurityService* _ss; SecurityCache _cache; - utils::DispatcherTimer _security_timer; - utils::DispatcherTimer _recovery_timer; + std::unique_ptr _security_timer; + std::unique_ptr _recovery_timer; + std::set _symbols; MDConsumer _consumer; utils::Event _event; - SecurityOracle _oracle; - MDClientImpl(utils::Dispatcher& d, std::shared_ptr s, SecurityOracle o): + MDClientImpl( + utils::Dispatcher* d, + api::ISyntheticSecurityService* s, + std::set* symbols): _dispatcher(d), _ss(s), - _security_timer(d, config::get().security_scan_interval(),[this]() { scan_securities(); }), - _recovery_timer(d, config::get().recovery_interval(), [this]() { check_heartbeats(); }), - _consumer(s, _cache), - _event(), - _oracle(o) + _security_timer(), + _recovery_timer(), + _symbols(symbols ? *symbols : get_system_md_symbols()), + _consumer(_cache, _ss, &_symbols), + _event() { - _security_timer.start(); - _recovery_timer.start(); - } - - std::set symbols() { - std::set all_symbols; - Venue venue; - bts_assert(api::Venue_Parse(config::get().exchange(), &venue)); - SecurityType type; - bts_assert(api::SecurityType_Parse(config::get().type(), &type)); - get_system_md_symbols(venue, type, all_symbols); - return all_symbols; + if (d) { + _security_timer = + std::make_unique(*d, config::get().security_scan_interval(),[this]() { scan_securities(); }), + _recovery_timer = + std::make_unique(*d, config::get().recovery_interval(), [this]() { check_heartbeats(); }), + _security_timer->start(); + _recovery_timer->start(); + } } void check_heartbeats() { if (!_consumer.all_heartbeats_ok()) { - MDConsumer temp(_ss,_cache); + MDConsumer temp(_cache,_ss); if (temp.all_heartbeats_ok()) { _consumer.swap(temp); LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); @@ -52,20 +52,34 @@ struct MDClientImpl{ } } - void scan_securities() { + size_t scan_securities() { if (_consumer.all_heartbeats_ok()) { - _consumer.scan_securities(); + TimeStamp start = TimeStamp::now(); + auto count = _consumer.scan_securities(); + auto delta = (TimeStamp::now() - start); + if (delta.milliseconds() > 10) { + LOG(md,Diag,"Scanned handles: {}ms found: {} securities. " , + delta.milliseconds(),count); + } + return count; } + return 0; } }; -MDClient::MDClient(utils::Dispatcher& d, std::shared_ptr s) : - _self(new MDClientImpl(d,s, SecurityOracle(*this))) { -} +MDClient::MDClient( + utils::Dispatcher* d, + api::ISyntheticSecurityService* s, + std::set* symbols) : + _self(new MDClientImpl(d,s, symbols)) + { } MDClient::~MDClient() { } +size_t MDClient::scan_securities() { return _self->scan_securities(); } +void MDClient::check_heartbeats() { return _self->check_heartbeats(); } + utils::EventSubscriber& MDClient::on_consumer() { return _self->_event; } @@ -73,7 +87,10 @@ MDConsumer& MDClient::current_consumer() const { return _self->_consumer; } SecurityOracle MDClient::oracle() const { - return _self->_oracle; + return SecurityOracle(*this); +} +std::set MDClient::symbols() const { + return _self->_symbols; } }}//namespace \ No newline at end of file diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h index f9c6b8d3a..6de826de1 100644 --- a/src/lib/md/MDClient.h +++ b/src/lib/md/MDClient.h @@ -3,7 +3,9 @@ #include "lib/utils/Dispatcher.h" #include "lib/utils/Event.h" #include "ISyntheticSecurityService.h" +#include "UserSymbol.h" #include +#include namespace bts { namespace md { @@ -14,12 +16,21 @@ class MDClient { std::unique_ptr _self; public: - MDClient(utils::Dispatcher&, std::shared_ptr); + // Will check for recovery + MDClient( + utils::Dispatcher* d=nullptr, + api::ISyntheticSecurityService* ss=nullptr, + std::set* s=nullptr); ~MDClient(); + // Check heartbeats and attempt recovery if they are bad + void check_heartbeats(); + size_t scan_securities(); + utils::EventSubscriber& on_consumer(); MDConsumer& current_consumer() const; SecurityOracle oracle() const; + std::set symbols() const; }; diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index d01908032..a532cdf1a 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -307,7 +307,7 @@ class SyntheticConsumerInst { // LOCK only public methods Mutex _workmutex; - shared_ptr _ss; + api::ISyntheticSecurityService* _ss; map _data; vector> _pending; set _seen; @@ -318,7 +318,7 @@ class SyntheticConsumerInst { std::vector> _old_strategies; public: - SyntheticConsumerInst(std::shared_ptr& ss, const IGetHandle& gh) : + SyntheticConsumerInst(api::ISyntheticSecurityService* ss, const IGetHandle& gh) : _workmutex(), _ss(ss), _data(), @@ -543,9 +543,9 @@ class MDConsumerImpl { public: MDConsumerImpl( const set& symbols, - std::shared_ptr& ss, SecurityCache& cache, - bool shared, co2::Platform platform = co2::PLATFORM_UNSET) : + api::ISyntheticSecurityService* ss, + bool shared, co2::Platform platform) : _workmutex(), _instances(), _sci(), @@ -907,27 +907,20 @@ public: // MDConsumer public interface, forwards to pimple /////////////////////////////////////////////////////////////////////////////////////////////////// -MDConsumer::MDConsumer(std::shared_ptr& synt, SecurityCache& cache, bool shared) : _pimpl() { - set all_symbols; - get_system_md_symbols(VENUE_UNSET, SECURITYTYPE_UNSET, all_symbols); +MDConsumer::MDConsumer( + SecurityCache& cache, + api::ISyntheticSecurityService* ss, + const std::set* symbols, + bool shared, + co2::Platform platform) : _pimpl() { - _pimpl.reset(new MDConsumerImpl(all_symbols, synt, cache, shared, co2::PLATFORM_UNSET)); -} -MDConsumer::MDConsumer(const UserSymbol& s, std::shared_ptr synt, SecurityCache& cache, bool shared) : - _pimpl() -{ - set symbols; - symbols.insert(s); - _pimpl.reset(new MDConsumerImpl(symbols, synt, cache, shared, co2::PLATFORM_UNSET)); -} -MDConsumer::MDConsumer(set const& symbols, std::shared_ptr s, SecurityCache& cache, bool shared, co2::Platform platform) : - _pimpl(new MDConsumerImpl(symbols, s, cache, shared, platform)) {} + set all_symbols; + if (symbols == nullptr || symbols->empty()) { + all_symbols = get_system_md_symbols(); + symbols = &all_symbols; + } -MDConsumer::MDConsumer(SecurityCache& cache, api::Venue ven, co2::Platform platform) : _pimpl() { - set all_symbols; - get_system_md_symbols(ven, SECURITYTYPE_UNSET, all_symbols); - std::shared_ptr null = nullptr; - _pimpl.reset(new MDConsumerImpl(all_symbols, null, cache, true, platform)); + _pimpl.reset(new MDConsumerImpl(*symbols, cache, ss, shared, platform)); } MDConsumer::~MDConsumer() {} diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index dff515edc..02c762a8f 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -30,12 +30,12 @@ class SecurityCache; class MDConsumer { public: - // Consumer for all symbols - MDConsumer(std::shared_ptr&, SecurityCache&, bool shared=true); - MDConsumer(const UserSymbol&, std::shared_ptr, SecurityCache&, bool shared=true); - // Consumer for a given set of symbols - MDConsumer(std::set const& symbols, std::shared_ptr, SecurityCache&, bool shared=true, co2::Platform platform=co2::PLATFORM_UNSET); - MDConsumer(SecurityCache&, api::Venue, co2::Platform); + MDConsumer( + SecurityCache&, + api::ISyntheticSecurityService* ss, + const std::set* sym = nullptr, + bool shared=true, + co2::Platform platform=co2::PLATFORM_UNSET); ~MDConsumer(); bool all_heartbeats_ok() const; diff --git a/src/lib/md/MDConsumerInterop.cpp b/src/lib/md/MDConsumerInterop.cpp index 142270e6f..5027f3d75 100644 --- a/src/lib/md/MDConsumerInterop.cpp +++ b/src/lib/md/MDConsumerInterop.cpp @@ -27,18 +27,12 @@ struct BlitUserSymbol { }; CAPI MDConsumer* create_md_consumer(BlitUserSymbol* usersymbols, uint32_t count, bool shared) { - std::shared_ptr pnull = nullptr; SecurityCache cache; - if (usersymbols == nullptr || count == 0) { - return new MDConsumer(pnull, cache, shared); - } - else { - std::set symbols; - for (size_t x = 0; x < count; ++x) { - symbols.insert(UserSymbol(usersymbols[x].symbol, usersymbols[x].exchange, usersymbols[x].type)); - } - return new MDConsumer(symbols, pnull, cache, shared); + std::set symbols; + for (size_t x = 0; x < count; ++x) { + symbols.insert(UserSymbol(usersymbols[x].symbol, usersymbols[x].exchange, usersymbols[x].type)); } + return new MDConsumer(cache, nullptr/*synthetics*/, &symbols, shared); } CAPI void destroy_md_consumer(MDConsumer* consumer) { diff --git a/src/lib/md/MDRecoverableConsumer.cpp b/src/lib/md/MDRecoverableConsumer.cpp deleted file mode 100644 index 5b5378033..000000000 --- a/src/lib/md/MDRecoverableConsumer.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#ifndef NO_CPP0X -#include "MDRecoverableConsumer.h" -#include "PlatformLoader.h" -#include "lib/logging/logging.h" -#include - - -using namespace std; -namespace bts { -namespace md { - -set get_symbols(Venue v=api::VENUE_UNSET); - -MDRecoverableConsumer::MDRecoverableConsumer(std::shared_ptr ss) : - _symbols(get_symbols()), - _ss(ss), - _current(init(_symbols, ss)), - _old(), - _callbacks(), - _seen(), - _lock () -{} - -MDRecoverableConsumer::MDRecoverableConsumer(Venue exchange, std::shared_ptr ss) : // (visual studio 2012) MDRecoverableConsumer(get_symbols(exchange)) - _symbols(get_symbols(exchange)), - _ss(ss), - _current(init(_symbols, ss)), - _old(), - _callbacks(), - _seen(), - _lock () -{} - -MDRecoverableConsumer::MDRecoverableConsumer(std::set symbols, std::shared_ptr ss) : - _symbols(symbols), - _ss(ss), - _current(init(symbols, ss)), - _old(), - _callbacks(), - _seen(), - _lock () -{ -} - -MDRecoverableConsumer::~MDRecoverableConsumer() {} - -unique_ptr MDRecoverableConsumer::init(std::set& symbols, std::shared_ptr& ss) { - return std::make_unique(symbols, ss, _cache); -} - -size_t MDRecoverableConsumer::walk_security_list(SecurityCallback callback) { - return _current->consumer.walk_security_list(callback); -} - -void MDRecoverableConsumer::on_security(SecurityCallback cb) { - // Add a filter to the callback to avoid calling the on_security callback with duplicates - auto filtered_cb = [this,cb](const SecurityPtr& security, MDConsumerHandle handle) { - auto key = security->id(); - if (_seen.find(key) == _seen.end()) { - _seen.insert(key); - cb(security, handle); - } - }; - _current->consumer.on_security(filtered_cb); - Mutex::scoped_lock l (_lock); - _callbacks.push_back(filtered_cb); -} - -size_t MDRecoverableConsumer::scan_securities() { - return _current->consumer.scan_securities(); -} - -std::shared_ptr MDRecoverableConsumer::get_managed_handle(security_key key) { - Mutex::scoped_lock l (_lock); - auto handle = _current->consumer.get_managed_handle(key); - _current->handles.push_back(handle); - return handle; -} - -bool MDRecoverableConsumer::all_heartbeats_ok() const { - return _current->consumer.all_heartbeats_ok(); -} - -bool MDRecoverableConsumer::try_recover() { - Mutex::scoped_lock l (_lock); - // Replace current with a new one (if the new one is better) - auto temp = init(_symbols, _ss); - if (!temp->consumer.all_heartbeats_ok()) return false; - - std::swap(temp, _current); - // push current to old - _old.push_back(std::move(temp)); - // copy old callbacks so we can reinstate them - auto old_callbacks = _callbacks; - - // Add an on_security callback to current to check the old handles - l.release(); - - // re-instante old callbacks - for (auto &cb : old_callbacks) { - _current->consumer.on_security(cb); - } - - _current->consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle newhandle) { - Mutex::scoped_lock l (_lock); - for (auto& group : _old) { - for (auto& handle : group->handles) { - if (handle->key() == security->id()) { - // update the handle! (if this is called guarunteed success) - *handle = newhandle; - bts_assert(handle->is_initialized()); - _current->handles.push_back(move(handle)); - } - } - } - cleanup(); - }); - - return true; -} - -// cleanup should only ever be called while _lock is locked -void MDRecoverableConsumer::cleanup() { - for (auto& group : _old) { - auto& handles = group->handles; - handles.erase( - remove_if( - handles.begin(), - handles.end(), - [](shared_ptr& handle) { return handle == nullptr; }), - handles.end()); - } - auto old_sz = _old.size (); - _old.erase( - remove_if( - _old.begin(), - _old.end(), - [](unique_ptr& pgroup) -> bool { return pgroup->handles.empty(); }), - _old.end()); - if (old_sz > _old.size ()) { - LOG (md,Info,"Cleaned up {} MDConsumer instances" ,old_sz - _old.size () ); - } -} - - -// -// Utility functions -set get_symbols(Venue exchange) { - set all_symbols; - get_system_md_symbols(exchange, api::SECURITYTYPE_UNSET, all_symbols); - return all_symbols; -} - - -}}//namespace -#endif - diff --git a/src/lib/md/MDRecoverableConsumer.h b/src/lib/md/MDRecoverableConsumer.h deleted file mode 100644 index c773b5d4d..000000000 --- a/src/lib/md/MDRecoverableConsumer.h +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once - -#include "lib/utils/bts_no_copy.h" -#include "common.h" -#include "MDConsumer.h" -#include "MDConsumerHandle.h" -#include "UserSymbol.h" -#include "SecurityCache.h" -#include -#include -#include - -namespace bts { -namespace md { - -class BTS_EXPORT MDRecoverableConsumer -{ -public: - MDRecoverableConsumer(std::shared_ptr ss); - // Map only a certain exchange for more granular recovery - MDRecoverableConsumer(Venue, std::shared_ptr ss); - MDRecoverableConsumer(std::set symbols, std::shared_ptr ss); - ~MDRecoverableConsumer(); - - // - // Security functions - size_t walk_security_list(SecurityCallback callback); - void on_security(SecurityCallback cb); - // Scans for added securities - updates managed handles - size_t scan_securities(); - - // - // Data Access - std::shared_ptr get_managed_handle(security_key key); - - // - // Recover from publisher failure - bool all_heartbeats_ok() const; - bool try_recover(); - -private: - BTS_NO_COPY(MDRecoverableConsumer); - - void cleanup(); - - struct MDConsumerGroup { - MDConsumerGroup( - std::set& symbols, - std::shared_ptr& ss, - SecurityCache& cache) : consumer(symbols, ss, cache), handles() {} - MDConsumer consumer; - std::vector> handles; - }; - std::unique_ptr init(std::set& symbols, std::shared_ptr& ss); - - std::set _symbols; - std::shared_ptr _ss; - SecurityCache _cache; - std::unique_ptr _current; - std::vector> _old; - - std::vector _callbacks; - std::set _seen; - typedef tbb::mutex Mutex; - Mutex _lock; -}; - -}}//namespace - diff --git a/src/lib/md/PlatformLoader.cpp b/src/lib/md/PlatformLoader.cpp index acd1ace54..717e8e88c 100644 --- a/src/lib/md/PlatformLoader.cpp +++ b/src/lib/md/PlatformLoader.cpp @@ -172,6 +172,12 @@ size_t get_system_md_symbols(const md::PlatformConfig &config, return get_system_md_symbols(v, st, out_result); } +std::set get_system_md_symbols() { + std::set all_symbols; + get_system_md_symbols(VENUE_UNSET, SECURITYTYPE_UNSET, all_symbols); + return all_symbols; +} + extern "C" BTS_EXPORT Platform platform_for_exchange(Venue v) { const auto &pmap = config::get(); co2::Platform platform; diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp index 6d5fd1b43..160633145 100644 --- a/src/lib/md/SecurityCache.cpp +++ b/src/lib/md/SecurityCache.cpp @@ -60,6 +60,7 @@ SecurityPtr SecurityCache::get_security(SecurityKey key) { lock.unlock(); auto security = db::find_security(key); if (security) { + LOG(md,Info, "Loaded security from database: {} {}", security->symbol(), security->id()); return add_db_security(security); } return security; diff --git a/src/lib/md/SyntheticSecurityService.h b/src/lib/md/SyntheticSecurityService.h index 0f07c0080..32b169192 100644 --- a/src/lib/md/SyntheticSecurityService.h +++ b/src/lib/md/SyntheticSecurityService.h @@ -20,9 +20,9 @@ class SyntheticSecurityService : struct Pimpl; std::unique_ptr _me; - public: - SyntheticSecurityService(api::ServiceContext& context); - ~SyntheticSecurityService(); +public: + SyntheticSecurityService(api::ServiceContext& context); + ~SyntheticSecurityService(); const std::unordered_map securities() override; utils::EventSubscriber& on_securities() override; diff --git a/src/lib/md/coconut2.h b/src/lib/md/coconut2.h index 79230eb4c..4195a0a40 100644 --- a/src/lib/md/coconut2.h +++ b/src/lib/md/coconut2.h @@ -9,5 +9,4 @@ #include "SettlementRecorder.h" // TODO: separate coconut2.h into consumer and producer sides #include "SecurityIndex.h" -#include "MDRecoverableConsumer.h" diff --git a/src/lib/md/get_symbols.h b/src/lib/md/get_symbols.h index 3088e1b77..d0226432d 100644 --- a/src/lib/md/get_symbols.h +++ b/src/lib/md/get_symbols.h @@ -8,6 +8,7 @@ namespace bts { namespace md { +std::set get_system_md_symbols(); size_t get_system_md_symbols(api::Venue, api::SecurityType, std::vector* result); size_t get_system_md_symbols(api::Venue, api::SecurityType, std::set& out_result); size_t get_system_md_symbols(const md::PlatformConfig& config, std::set& out_result); diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 783c355e6..5b44513d4 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -7,6 +7,7 @@ #include "lib/utils/AuthToken.h" #include "../MDConsumer.h" +#include "../MDClient.h" #include "../PlatformLoader.h" #include "lib/logging/logging.h" @@ -163,27 +164,30 @@ void copy_misc(IntPrice settlement, MiscData misc, co2::MiscMessage* pmisc) { // Handles market data queries and responds class QueryServerImpl : public IQueryServer { - SecurityCache _cache; + MDClient& _client; MDConsumer _consumer; - const std::set _symbols; - const std::shared_ptr _ss; + int _sub; boost::container::flat_map> _groups; public: - QueryServerImpl(const std::set& symbols, std::shared_ptr& ss, metrics::MetricsContainer& stats) : - _cache(), - _consumer(symbols, ss, _cache), - _symbols(symbols), - _ss(ss), + QueryServerImpl(MDClient& client, metrics::MetricsContainer& stats) : + _client(client), + _consumer(client.current_consumer()), + _sub(0), _groups() { - for (const auto& sym : symbols) { + for (const auto& sym : client.symbols()) { _groups.emplace(std::make_pair(sym, std::make_unique(sym, _consumer, stats))); } _groups.shrink_to_fit(); + _sub = client.on_consumer().subscribe([=](const MDConsumer& c) { handle_recovery(c); }); subscribe_securities(); } + QueryServerImpl(const QueryServerImpl&)=delete; + ~QueryServerImpl() { + _client.on_consumer().unsubscribe(_sub); + } void subscribe_securities() { auto id = _consumer.id(); @@ -199,32 +203,16 @@ public: } // not-reentrant - void scan_securities() { - if (_consumer.all_heartbeats_ok()) { - TimeStamp start = TimeStamp::now(); - size_t count = _consumer.scan_securities(false); - auto delta = (TimeStamp::now() - start); - if (delta.milliseconds() > 10) { - LOG(md,Diag,"Scanned handles: {}ms found: {} securities. " , - delta.milliseconds(),count); - } - } - } - - // not-reentrant - void check_heartbeats(utils::Dispatcher& dispatcher) { - if (!_consumer.all_heartbeats_ok()) { - SecurityCache cache; - MDConsumer temp(_symbols, _ss, cache); - if (temp.all_heartbeats_ok()) { - _consumer.swap(temp); - LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); - for (auto& pair : _groups) { - pair.second->set_consumer(_consumer); - } - subscribe_securities(); - dispatcher.post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000); + void handle_recovery(const MDConsumer& consumer) { + MDConsumer temp(consumer); + if (temp.all_heartbeats_ok()) { + _consumer.swap(temp); + LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); + for (auto& pair : _groups) { + pair.second->set_consumer(_consumer); } + subscribe_securities(); + utils::Dispatcher::background().post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000); } } @@ -500,53 +488,25 @@ struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { }; struct QueryServer { - QueryServer(std::set symbols, - utils::Dispatcher& threadpool, + QueryServer(utils::Dispatcher& threadpool, const bts::api::ServiceContext& ctx, - std::shared_ptr& ss, + MDClient& client, metrics::MetricsContainer& stats): - db(symbols, ss, stats), + db(client, stats), relay(db, threadpool, ctx, stats), - admin_client(ctx.locate()), - security_timer(), - health_timer() + admin_client(ctx.locate()) { } QueryServerImpl db; MDRelayServer relay; api::admin::AdminService admin_client; - std::unique_ptr security_timer; - std::unique_ptr health_timer; rxcpp::composite_subscription entitlement_sub; }; -QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared_ptr& ss, metrics::MetricsContainer& stats) { +QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, MDClient& client, metrics::MetricsContainer& stats) { auto& threadpool = utils::Dispatcher::get_named("MDQuery"); - std::set all_symbols; - Venue venue; - bts_assert(api::Venue_Parse(config::get().exchange(), &venue)); - SecurityType type; - bts_assert(api::SecurityType_Parse(config::get().type(), &type)); - get_system_md_symbols(venue, type, all_symbols); + QueryServerPtr container = std::make_shared(threadpool, ctx, client, stats); - // auto admin = ctx.locate(); - // admin.subscribe_md_entitlement_data().subscribe( - - QueryServerPtr container = std::make_shared(all_symbols, threadpool, ctx, ss, stats); - auto& db = container->db; - - // Scan for new securities every half second - container->security_timer = std::make_unique(threadpool, config::get().security_scan_interval(), [&db]() { - db.scan_securities(); - }); - container->security_timer->start(); - - // Scan for dead producers every 30 seconds - container->health_timer = std::make_unique(threadpool, config::get().recovery_interval(), [&db, &threadpool]() { - db.check_heartbeats(threadpool); - }); - container->health_timer->start(); - // Subscribe to entitlement data container->entitlement_sub = container->admin_client.subscribe_md_entitlement_data().subscribe([=](const api::admin::Md_entitlement_dataUpdate& update) { @@ -587,8 +547,8 @@ QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared return container; } -std::unique_ptr create_query_server(const std::set& symbols, std::shared_ptr& ss, metrics::MetricsContainer& stats) { - return std::make_unique(symbols, ss, stats); +std::unique_ptr create_query_server(MDClient& mdc, metrics::MetricsContainer& mets) { + return std::make_unique(mdc, mets); } }}//namespace diff --git a/src/lib/md/relay/QueryServer.h b/src/lib/md/relay/QueryServer.h index 13a8d056e..186089658 100644 --- a/src/lib/md/relay/QueryServer.h +++ b/src/lib/md/relay/QueryServer.h @@ -14,18 +14,18 @@ namespace bts { namespace api { class ServiceContext; } namespace metrics { class MetricsContainer; } namespace md { +class MDClient; typedef boost::container::flat_set Entitlements; struct IQueryServer { virtual ~IQueryServer() {} - virtual void scan_securities() = 0; virtual void query(const co2::MDRequest& request, co2::MDReply* reply, const Entitlements*, bool) const = 0; }; -std::unique_ptr create_query_server(const std::set&, std::shared_ptr&, metrics::MetricsContainer&); +std::unique_ptr create_query_server(MDClient&, metrics::MetricsContainer&); struct QueryServer; typedef std::shared_ptr QueryServerPtr; -QueryServerPtr run_query_server(const bts::api::ServiceContext&, std::shared_ptr&, metrics::MetricsContainer&); +QueryServerPtr run_query_server(const bts::api::ServiceContext&, MDClient&, metrics::MetricsContainer&); }}//namespace diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 35c8a828e..69413197a 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -1,9 +1,9 @@ - #include "catch/catch.hpp" #include "../SettlementRecorder.h" #include "../coconut2.h" #include "../relay/QueryServer.h" #include "../SecurityCache.h" +#include "../MDClient.h" #include "Co2TestInstrument.h" #include "lib/config/config.h" #include "lib/os/os.h" @@ -370,7 +370,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); vector consumer_securities; // Poll for security list - none visible until they have keys @@ -441,7 +441,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); // Add the on_security callback vector securities; @@ -481,7 +481,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -539,7 +539,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -598,9 +598,9 @@ public: assign_ids(producers.begin(), producers.end()); // Symbol list for the consumer - auto syms = get_test_symbols(); + auto symbols = get_test_symbols(); SecurityCache cache; - MDConsumer consumer = MDConsumer(syms, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); // Add the on_security callback - callback fires for existing securities vector securities; @@ -626,20 +626,23 @@ public: for (auto& pro : producers) { pro.update_bbo(TimeStamp::now(), PriceSize({1, 0}, 1), PriceSize({2, 0}, 2)); } // Symbol list for the consumer - MDRecoverableConsumer consumer(get_test_symbols(), nullptr); - map> products; + SecurityCache cache; + auto symbols = get_test_symbols(); + MDClient client(nullptr, nullptr, &symbols); + MDConsumer consumer = client.current_consumer(); + map products; consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); - products.insert(make_pair(security->id(), consumer.get_managed_handle(security->id()))); + products.insert(make_pair(security->id(), handle)); }); - consumer.scan_securities(); + client.scan_securities(); REQUIRE(products.size() == producers.size()); - // Choose a handle + // Check handles for (auto& pair : products) { auto& handle = pair.second; - REQUIRE(handle->is_initialized()); - REQUIRE(handle->is_heart_beating()); + REQUIRE(handle.is_initialized()); + REQUIRE(handle.is_heart_beating()); } // Delete the producer @@ -661,15 +664,15 @@ public: try { for (auto& pair : products) { auto& handle = pair.second; - REQUIRE(handle->is_initialized()); - REQUIRE(handle->is_heart_beating() == false); + REQUIRE(handle.is_initialized()); + REQUIRE(handle.is_heart_beating() == false); } } catch(std::exception&) { LOG(md,Error,"Failed to validate heartbeat while producer is down."); REQUIRE(false); } - // Create a new publisher and attempt to recover + // Create a new producer and attempt to recover shared_ptr producer2; try { producer2 = make_shared(TEST_PLATFORM); @@ -689,16 +692,30 @@ public: } // Attempt to recover the managed handles - REQUIRE(consumer.try_recover()); + int on_consumer_count = 0; + auto handle = client.on_consumer().subscribe([&](const MDConsumer& c) { + ++on_consumer_count; + consumer = c; + products.clear(); + }); + client.check_heartbeats(); + REQUIRE(on_consumer_count == 1); + REQUIRE(products.empty()); + // Consumer has been updated + consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() == handle.key()); + products.insert(make_pair(security->id(), handle)); + }); + client.on_consumer().unsubscribe(handle); - // Since we know that both securities exist they should have been updated to the new section + // Check that the new product handles have been updated for (auto& pair : products) { auto& handle = pair.second; - REQUIRE(handle->is_initialized()); - REQUIRE(handle->is_heart_beating()); + REQUIRE(handle.is_initialized()); + REQUIRE(handle.is_heart_beating()); TimeStamp tzero; PriceSize bid, ask; - REQUIRE(handle->get_direct_book(tzero, 1, &bid, &ask) == HandleReadResult::SUCCESS); + REQUIRE(handle.get_direct_book(tzero, 1, &bid, &ask) == HandleReadResult::SUCCESS); REQUIRE(bid.price() == 4.0); REQUIRE(bid.size() == 5); REQUIRE(ask.price() == 6.0); @@ -744,7 +761,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); set keys; consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) { keys.insert(security->id()); }); @@ -789,7 +806,7 @@ public: csymbols.insert(UserSymbol("S3", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S4", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(csymbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &csymbols, true, TEST_PLATFORM); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -836,7 +853,7 @@ public: csymbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); csymbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(csymbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &csymbols, true, TEST_PLATFORM); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -888,7 +905,7 @@ public: consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); consumer_symbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(consumer_symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &consumer_symbols, true, TEST_PLATFORM); size_t security_count = 0; // Indirection for check function @@ -957,7 +974,7 @@ public: set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); SecurityCache cache; - MDConsumer consumer(consumer_symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &consumer_symbols, true, TEST_PLATFORM); auto h1 = consumer.get_handle(k1); REQUIRE(h1.is_initialized()); auto h2 = consumer.get_handle(k2); @@ -1242,7 +1259,8 @@ struct Co2RelayTests { symbols.insert(future_symbol); UserSymbol option_symbol("FOOX", VENUE_BTS, SECURITYTYPE_OPTION); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr, cache); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &symbols); return consumer; } @@ -1257,7 +1275,13 @@ struct Co2RelayTests { } metrics::MetricsContainer _stats; - std::unique_ptr setup_query_server(MDProducer& producer, Entitlements* out_entitlements, vector* out_producers) { + + std::unique_ptr setup_query_server( + MDClient& client, + MDProducer& producer, + Entitlements* out_entitlements, + vector* out_producers) { + // Create a bunch of securities for (size_t x = 0; x < 100; ++x) { // That's right. Expirations daily. @@ -1277,10 +1301,9 @@ struct Co2RelayTests { for (const auto& s : symbols) { out_entitlements->insert(s.symbol()); } - shared_ptr empty; - auto server = create_query_server(symbols, empty, _stats); + auto server = create_query_server(client, _stats); REQUIRE(server); - server->scan_securities(); + client.scan_securities(); return server; } @@ -1293,7 +1316,8 @@ struct Co2RelayTests { // initialize data vector producers; - auto server = setup_query_server(producer, &entitlements, &producers); + MDClient client; + auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices PriceSize price35 = PriceSize_from_double<2>(3.5, 5); @@ -1423,7 +1447,8 @@ struct Co2RelayTests { // initialize data vector producers; Entitlements entitlements; - auto server = setup_query_server(producer, &entitlements, &producers); + MDClient client; + auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices to all levels auto set_direct = [&](PriceSize bid, PriceSize ask) { @@ -1600,7 +1625,7 @@ namespace ImpliedTests { UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols); vector consumer_securities; // Poll for security list - none visible until they have keys size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); @@ -2373,4 +2398,3 @@ TEST_CASE("IntPrice", "[md]") { }}} - diff --git a/src/lib/orderman/events/NewSecuritiesEvent.h b/src/lib/orderman/events/NewSecuritiesEvent.h index fddcc0e6d..be5f3c505 100644 --- a/src/lib/orderman/events/NewSecuritiesEvent.h +++ b/src/lib/orderman/events/NewSecuritiesEvent.h @@ -7,6 +7,10 @@ namespace bts { namespace orderman { + //todo: + // anywhere that uses this + // get a SecurityOracle + // which keeps a shared_ptr and uses it class NewSecuritiesEvent : public mercury::Event { diff --git a/src/lib/orderman/server/OrderServiceServer.h b/src/lib/orderman/server/OrderServiceServer.h index 75c558932..999b46790 100644 --- a/src/lib/orderman/server/OrderServiceServer.h +++ b/src/lib/orderman/server/OrderServiceServer.h @@ -8,9 +8,6 @@ namespace bts { namespace trace {class PerfLog;} -namespace api { -class ISyntheticSecurityService; -} namespace metrics { class MetricsContainer; } diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index a1840cf77..2c13a48a9 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -58,13 +58,12 @@ int Run::execute(std::vector const &) { user::AuthServiceServer login_server(sc); md::run_market_datapublishers(); - std::shared_ptr synthetic_service = - std::make_shared(sc); - md::MDClient mdclient(utils::Dispatcher::current(), synthetic_service); + api::synthetic::SyntheticSecurityService synthetic_service(sc); + md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service); md::QueryServerPtr md_query_server( - md::run_query_server(sc, synthetic_service, stats)); + md::run_query_server(sc, mdclient, stats)); // initialize order plugins orderman::DynamicOrderRegistry dyn_order_registry; diff --git a/src/server/sparkd/cmds/Web.cpp b/src/server/sparkd/cmds/Web.cpp index 8cc3ad12e..029ae34bc 100644 --- a/src/server/sparkd/cmds/Web.cpp +++ b/src/server/sparkd/cmds/Web.cpp @@ -10,6 +10,7 @@ #include "lib/metrics/system_metrics.h" #include "server/sparkd/http/WebServer.h" #include "lib/md/relay/QueryServer.h" +#include "lib/md/MDClient.h" #include "lib/md/SyntheticSecurityService.h" #include "server/sparkd/user/AdminServiceServer.h" #include "server/sparkd/user/AuthServiceServer.h" @@ -39,17 +40,14 @@ int Web::execute(const std::vector& args) { md::run_market_datapublishers(); } - std::shared_ptr synthetic_service = - std::make_shared(sc); + api::synthetic::SyntheticSecurityService synthetic_service(sc); + md::MDClient mdclient(&utils::Dispatcher::current(), &synthetic_service); md::QueryServerPtr md_query_server( - md::run_query_server(sc, synthetic_service, stats)); + md::run_query_server(sc, mdclient, stats)); admin::AdminServiceServer admin_service(sc); - - - std::this_thread::sleep_until( - std::chrono::time_point::max()); + utils::Dispatcher::current().run(); return 0; } From 8d240bff602a3c14ec041e6a75c0dbdc0bfd81a7 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 1 Feb 2019 17:14:14 -0500 Subject: [PATCH 38/54] Fix fucked up curlies --- .../Predicates/Data/Events/OrderFilledEventPredicate.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs index b6b1f749f..80767eec6 100644 --- a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs @@ -14,9 +14,10 @@ namespace mbts { { OrderFilledEventPredicate.PredicateTypeOneofCase.AnyOrder, PredicateBuilderVMWrapper.create(p => p.security = null) }, { OrderFilledEventPredicate.PredicateTypeOneofCase.SecurityOrder, SecurityCriteriaVMFactory.create(bus, o => o.security?.is_value ?? new SecurityCriteria(), (p, s) => { - if (p.security == null) -{ p.security = new SecurityPredicate(); -} p.security.is_value = s; + if (p.security == null) { + p.security = new SecurityPredicate(); + } + p.security.is_value = s; }) } }; From 48849fc631ab5ae3f71e84aba5474163aff217a3 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 1 Feb 2019 17:14:26 -0500 Subject: [PATCH 39/54] Fix md unit tests --- .vscode/launch.json | 2 +- src/lib/md/MDClient.cpp | 4 ++-- src/lib/md/MDClient.h | 2 +- src/lib/md/test/md2_tests.cpp | 27 ++++++++++++++++++++------- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index cf754974d..0fdb97700 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -209,7 +209,7 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", - "args": ["test", "[synthetic]"], + "args": ["test", "[md]"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", "environment": [], diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index 6822c4d20..1cb454971 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -21,7 +21,7 @@ struct MDClientImpl{ MDClientImpl( utils::Dispatcher* d, api::ISyntheticSecurityService* s, - std::set* symbols): + const std::set* symbols): _dispatcher(d), _ss(s), _security_timer(), @@ -71,7 +71,7 @@ struct MDClientImpl{ MDClient::MDClient( utils::Dispatcher* d, api::ISyntheticSecurityService* s, - std::set* symbols) : + const std::set* symbols) : _self(new MDClientImpl(d,s, symbols)) { } MDClient::~MDClient() { diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h index 6de826de1..47925bc88 100644 --- a/src/lib/md/MDClient.h +++ b/src/lib/md/MDClient.h @@ -20,7 +20,7 @@ public: MDClient( utils::Dispatcher* d=nullptr, api::ISyntheticSecurityService* ss=nullptr, - std::set* s=nullptr); + const std::set* s=nullptr); ~MDClient(); // Check heartbeats and attempt recovery if they are bad diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 69413197a..9f0bf2aca 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -1276,6 +1276,14 @@ struct Co2RelayTests { metrics::MetricsContainer _stats; + const std::set setup_symbols() { + std::set result; + for (const auto& symbol : { "FOO", "FOOF" }) { + result.insert({symbol, VENUE_BTS, SECURITYTYPE_FUTURE}); + } + return result; + } + std::unique_ptr setup_query_server( MDClient& client, MDProducer& producer, @@ -1310,13 +1318,16 @@ struct Co2RelayTests { void relay_nans() { MDProducer producer(TEST_PLATFORM); - UserSymbol symbol("FOO", VENUE_BTS, SECURITYTYPE_FUTURE); Entitlements entitlements; // initialize data vector producers; - MDClient client; + // + // todo: setup_query_server should use client.symbols() instead of it's own list + // + const auto symbols = setup_symbols(); + MDClient client(nullptr, nullptr, &symbols); auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices @@ -1328,14 +1339,15 @@ struct Co2RelayTests { // Request via query server co2::MDRequest request; - for (const auto& symbol : { "FOO", "FOOF" }) { + for (const auto& symbol : symbols) { auto* chan = request.add_channels(); - chan->set_type(SECURITYTYPE_FUTURE); - chan->set_symbol(symbol); - chan->set_exchange(VENUE_BTS); + chan->set_type(symbol.type()); + chan->set_symbol(symbol.symbol()); + chan->set_exchange(symbol.exchange()); chan->set_flags(co2::MDRequest::BBO | co2::MDRequest::SECURITY); chan->set_entitled(true); } + // This request should include everything co2::MDReply reply; @@ -1447,7 +1459,8 @@ struct Co2RelayTests { // initialize data vector producers; Entitlements entitlements; - MDClient client; + const auto symbols = setup_symbols(); + MDClient client(nullptr, nullptr, &symbols); auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices to all levels From 4264928cc2106bb40127115e2cefd0084cea7485 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 5 Feb 2019 15:44:40 -0500 Subject: [PATCH 40/54] Use SecurityOracle in AsyncFactory,ServiceBridge,OrderServicesData --- src/lib/md/SecurityOracle.h | 3 +- src/lib/orderman/actors/AsyncFactory.cpp | 13 ++------- src/lib/orderman/actors/AsyncFactory.h | 3 +- src/lib/orderman/actors/ServiceBridge.cpp | 29 ++++--------------- src/lib/orderman/actors/ServiceBridge.h | 7 ++--- .../actors/order_manager/OrderManager.cpp | 11 ++++--- .../actors/order_manager/OrderManager.h | 3 +- .../order_manager/OrderServicesData.cpp | 4 +-- .../actors/order_manager/OrderServicesData.h | 4 +-- src/lib/orderman/orders/NativeOrder.cpp | 1 + src/lib/orderman/securities/ISecurityDB.h | 1 - src/lib/orderman/securities/MapSecurityDB.h | 13 ++++----- .../orderman/server/OrderServiceServer.cpp | 6 ++-- 13 files changed, 36 insertions(+), 62 deletions(-) diff --git a/src/lib/md/SecurityOracle.h b/src/lib/md/SecurityOracle.h index e45ba062f..82163a1e1 100644 --- a/src/lib/md/SecurityOracle.h +++ b/src/lib/md/SecurityOracle.h @@ -1,8 +1,9 @@ #pragma once -#include "SecurityPtr.h" +#include "lib/domain_model/SecurityKey.h" #include namespace bts { +namespace api {class Security;} namespace md { class MDClient; class SecurityOracle { diff --git a/src/lib/orderman/actors/AsyncFactory.cpp b/src/lib/orderman/actors/AsyncFactory.cpp index 91b76045e..f6543a5b2 100644 --- a/src/lib/orderman/actors/AsyncFactory.cpp +++ b/src/lib/orderman/actors/AsyncFactory.cpp @@ -7,7 +7,6 @@ #include "lib/orderman/events/CreateOrderReplyEvent.h" #include "lib/orderman/events/CreateOrderRequestEvent.h" #include "lib/orderman/events/NewAdminDataEvent.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/order_factory/OrderFactory.h" #include "lib/orderman/order_router/OrderRouter.h" #include "lib/orderman/securities/MapSecurityDB.h" @@ -16,8 +15,8 @@ namespace bts { namespace orderman { struct AsyncFactory::PImpl { - PImpl(DynamicOrderRegistry const &b) - : orders_stream(), fix_buffer_stream(), db(), order_router(), + PImpl(DynamicOrderRegistry const &b, md::SecurityOracle o) + : orders_stream(), fix_buffer_stream(), db(o), order_router(), order_factory(db, order_router, b) {} mercury::Stream orders_stream; @@ -27,7 +26,7 @@ struct AsyncFactory::PImpl { OrderFactory order_factory; }; -AsyncFactory::AsyncFactory(DynamicOrderRegistry const &b) : me(new PImpl(b)) {} +AsyncFactory::AsyncFactory(DynamicOrderRegistry const &b, md::SecurityOracle o) : me(new PImpl(b, o)) {} AsyncFactory::~AsyncFactory() {} void AsyncFactory::init() { @@ -40,12 +39,6 @@ void AsyncFactory::init() { me->order_router.update(*ev.admin_data); }); - subscribe([this](auto const &u) { - if (u.events) - for (auto const &ev : *(u.events)) - me->db.securities[SecurityKey(ev.security->id())] = ev.security; - }); - subscribe([this](const auto &u) { if (u.events) for (auto const &ev : *u.events) { diff --git a/src/lib/orderman/actors/AsyncFactory.h b/src/lib/orderman/actors/AsyncFactory.h index 76214701c..d66aebab8 100644 --- a/src/lib/orderman/actors/AsyncFactory.h +++ b/src/lib/orderman/actors/AsyncFactory.h @@ -4,6 +4,7 @@ #include "lib/orderman/orderman_utils/orderman_common.h" #include "lib/mercury/actors/Actor.h" #include "lib/config/protos/jetfix.pb.h" +#include "lib/md/SecurityOracle.h" namespace bts { namespace orderman { @@ -13,7 +14,7 @@ class DynamicOrderRegistry; class AsyncFactory : public mercury::Actor { public: - AsyncFactory(DynamicOrderRegistry const&); + AsyncFactory(DynamicOrderRegistry const&, md::SecurityOracle); ~AsyncFactory(); void init() override; diff --git a/src/lib/orderman/actors/ServiceBridge.cpp b/src/lib/orderman/actors/ServiceBridge.cpp index 62a5ba336..81e96c7b8 100644 --- a/src/lib/orderman/actors/ServiceBridge.cpp +++ b/src/lib/orderman/actors/ServiceBridge.cpp @@ -6,7 +6,6 @@ #include "lib/orderman/events/CancelOrderEvent.h" #include "lib/orderman/events/CrackOrderEvent.h" #include "lib/orderman/events/NewAdminDataEvent.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/events/OrderManagerReadyEvent.h" #include "lib/orderman/events/StoredFillEvent.h" #include "lib/orderman/events/StoredOrderEvent.h" @@ -29,10 +28,10 @@ struct ServiceBridge::PImpl { PImpl(OnReady const &on_ready, OnOrders const &on_orders, OnFills const &on_fills, DynamicOrderRegistry const &dyn_order_registry, - mercury::BrokerProxy &service_call_marshaller) + mercury::BrokerProxy &service_call_marshaller, md::SecurityOracle o) : on_ready(on_ready), on_orders(on_orders), on_fills(on_fills), - have_orders(false), have_securities(false), order_manager_ready(false), - sec_db(), order_router(), + have_orders(false), order_manager_ready(false), + sec_db(o), order_router(), order_factory(sec_db, order_router, dyn_order_registry), known_orders(), service_call_marshaller(service_call_marshaller) {} @@ -40,7 +39,6 @@ struct ServiceBridge::PImpl { OnOrders on_orders; OnFills on_fills; bool have_orders; - bool have_securities; bool order_manager_ready; securities::MapSecurityDB sec_db; OrderRouter order_router; @@ -59,9 +57,9 @@ struct ServiceBridge::PImpl { ServiceBridge::ServiceBridge(OnReady const &on_ready, OnOrders const &on_orders, OnFills const &on_fills, DynamicOrderRegistry const &dyn_order_registry, - mercury::BrokerProxy &service_call_marshaller) + mercury::BrokerProxy &service_call_marshaller, md::SecurityOracle o) : me(new PImpl(on_ready, on_orders, on_fills, dyn_order_registry, - service_call_marshaller)) {} + service_call_marshaller, o)) {} ServiceBridge::~ServiceBridge() {} @@ -69,7 +67,6 @@ void ServiceBridge::block_until_ready() { me->ready.get_future().wait(); } void ServiceBridge::init() { subscribe([this](auto const &u) { this->on(u); }); - subscribe([this](auto const &u) { this->on(u); }); subscribe([this](auto const &u) { this->on(u); }); subscribe([this](auto const &u) { this->on(u); }); subscribe([this](auto const &u) { this->on(u); }); @@ -108,20 +105,6 @@ void ServiceBridge::on(mercury::Update const &update) { } } -void ServiceBridge::on(mercury::Update const &update) { - if (update.events) { - - // update our security db - for (auto ev : *(update.events)) { - auto sec_ptr = ev.security; - me->sec_db.securities[SecurityKey(sec_ptr->id())] = sec_ptr; - } - bool was_ready = is_ready(); - me->have_securities = true; - maybe_signal_ready(was_ready); - } -} - void ServiceBridge::on(mercury::Update const &update) { if (update.events) { bool was_ready = is_ready(); @@ -247,7 +230,7 @@ void ServiceBridge::update_admin_data( } bool ServiceBridge::is_ready() const { - return me->have_orders && me->have_securities && me->order_manager_ready; + return me->have_orders && me->order_manager_ready; } void ServiceBridge::maybe_signal_ready(bool old_ready) { diff --git a/src/lib/orderman/actors/ServiceBridge.h b/src/lib/orderman/actors/ServiceBridge.h index f78f0a6c6..1c949f061 100644 --- a/src/lib/orderman/actors/ServiceBridge.h +++ b/src/lib/orderman/actors/ServiceBridge.h @@ -2,6 +2,7 @@ #include "lib/config/protos/jetfix.pb.h" #include "lib/mercury/actors/Actor.h" +#include "lib/md/SecurityOracle.h" #include // fwds @@ -25,7 +26,6 @@ class Crack; namespace orderman { class StoredFillEvent; class StoredOrderEvent; -class NewSecuritiesEvent; class OrderManagerReadyEvent; class NewAdminDataEvent; class DynamicOrderRegistry; @@ -50,8 +50,8 @@ class ServiceBridge : public mercury::Actor { ServiceBridge(OnReady const &, OnOrders const &, OnFills const &, DynamicOrderRegistry const &, - - mercury::BrokerProxy &service_call_marshaller); + mercury::BrokerProxy &service_call_marshaller, + md::SecurityOracle o); void init() override; ~ServiceBridge(); @@ -74,7 +74,6 @@ class ServiceBridge : public mercury::Actor { void on(mercury::Update const &); void on(mercury::Update const &); - void on(mercury::Update const &); void on(mercury::Update const &); void on(mercury::Update const &); }; diff --git a/src/lib/orderman/actors/order_manager/OrderManager.cpp b/src/lib/orderman/actors/order_manager/OrderManager.cpp index 68a99d941..4a8ed4482 100644 --- a/src/lib/orderman/actors/order_manager/OrderManager.cpp +++ b/src/lib/orderman/actors/order_manager/OrderManager.cpp @@ -8,6 +8,7 @@ #include "lib/orderman/events/OrderManagerReadyEvent.h" #include "lib/orderman/events/StoredFillEvent.h" #include "lib/orderman/events/StoredOrderEvent.h" +#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/order_factory/OrderFactory.h" #include "lib/orderman/orders/Order.h" #include "lib/orderman/securities/ISecurityDB.h" @@ -31,8 +32,8 @@ enum class OrderManager::State { }; struct OrderManager::PImpl { - PImpl(OrderManager &owner, DynamicOrderRegistry const &b, trace::PerfLog &l) - : state(State::WAIT_ADMIN), services_data(owner, b, l) {} + PImpl(OrderManager &owner, DynamicOrderRegistry const &b, trace::PerfLog &l, md::SecurityOracle o) + : state(State::WAIT_ADMIN), services_data(owner, b, l, o) {} State state; std::unordered_map orders; @@ -42,8 +43,8 @@ struct OrderManager::PImpl { mercury::Stream pub_ready; }; -OrderManager::OrderManager(DynamicOrderRegistry const &b, trace::PerfLog &l) - : me(new PImpl(*this, b, l)) {} +OrderManager::OrderManager(DynamicOrderRegistry const &b, trace::PerfLog &l, md::SecurityOracle o) + : me(new PImpl(*this, b, l, o)) {} OrderManager::~OrderManager() {} void OrderManager::init() { @@ -85,8 +86,6 @@ void OrderManager::on_securities( for (auto const &ev : *(update.events)) { auto sec_ptr = ev.security; secs.push_back(sec_ptr); - me->services_data.sec_db.securities[SecurityKey(sec_ptr->id())] = - sec_ptr; } me->services_data.limiter.add_securities(secs); diff --git a/src/lib/orderman/actors/order_manager/OrderManager.h b/src/lib/orderman/actors/order_manager/OrderManager.h index a8ec11582..4bee86449 100644 --- a/src/lib/orderman/actors/order_manager/OrderManager.h +++ b/src/lib/orderman/actors/order_manager/OrderManager.h @@ -4,6 +4,7 @@ #include "lib/jetfix/common/OutFIXPtr.h" #include "lib/mercury/actors/Actor.h" #include "lib/mercury/actors/IPollableActor.h" +#include "lib/md/SecurityOracle.h" #include "lib/orderman/events/ActivateOrderEvent.h" #include "lib/orderman/events/CancelOrderEvent.h" #include "lib/orderman/events/CrackOrderEvent.h" @@ -60,7 +61,7 @@ namespace orderman { class OrderManager : public mercury::Actor, public mercury::IPollableActor { public: - OrderManager(DynamicOrderRegistry const &dyn_order_builder, trace::PerfLog &); + OrderManager(DynamicOrderRegistry const &dyn_order_builder, trace::PerfLog &, md::SecurityOracle); ~OrderManager(); void init() override; diff --git a/src/lib/orderman/actors/order_manager/OrderServicesData.cpp b/src/lib/orderman/actors/order_manager/OrderServicesData.cpp index 5b19f6e44..253f56317 100644 --- a/src/lib/orderman/actors/order_manager/OrderServicesData.cpp +++ b/src/lib/orderman/actors/order_manager/OrderServicesData.cpp @@ -7,8 +7,8 @@ namespace orderman { OrderServicesData::OrderServicesData(OrderManager &owner, DynamicOrderRegistry const &b, - trace::PerfLog &l) - : owner(owner), log(l), order_router(), + trace::PerfLog &l, md::SecurityOracle o) + : owner(owner), log(l), sec_db(o), order_router(), order_factory(sec_db, order_router, b), last_timer_ev( Timer::now(), diff --git a/src/lib/orderman/actors/order_manager/OrderServicesData.h b/src/lib/orderman/actors/order_manager/OrderServicesData.h index e664aec0a..21afdbc4d 100644 --- a/src/lib/orderman/actors/order_manager/OrderServicesData.h +++ b/src/lib/orderman/actors/order_manager/OrderServicesData.h @@ -8,6 +8,7 @@ #include "lib/orderman/securities/MapSecurityDB.h" #include "lib/mercury/events/TimerEvent.h" +#include "lib/md/SecurityOracle.h" #include "lib/orderman/actors/ExecutionGateway.h" #include "lib/orderman/events/ActivateOrderEvent.h" #include "lib/orderman/events/BookUpdateEvent.h" @@ -22,7 +23,6 @@ #include "lib/orderman/events/GatewayStatusChangeEvent.h" #include "lib/orderman/events/MDRequestEvent.h" #include "lib/orderman/events/NewExecutionReportEvent.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/events/OrderStateUpdateEvent.h" #include "lib/orderman/events/SettlementRequestEvent.h" #include "lib/orderman/events/SettlementUpdateEvent.h" @@ -75,7 +75,7 @@ class OrderServicesData { public: OrderServicesData(OrderManager &owner, DynamicOrderRegistry const &b, - trace::PerfLog &l); + trace::PerfLog &l, md::SecurityOracle); OrderManager &owner; trace::PerfLog &log; diff --git a/src/lib/orderman/orders/NativeOrder.cpp b/src/lib/orderman/orders/NativeOrder.cpp index 60e1a615f..9ff30477d 100644 --- a/src/lib/orderman/orders/NativeOrder.cpp +++ b/src/lib/orderman/orders/NativeOrder.cpp @@ -4,6 +4,7 @@ #include "lib/jetfix/codec/FIXFragment.h" #include "lib/jetfix/codec/FieldDefinitions.h" #include "lib/logging/logging.h" +#include "lib/domain_model/Security.h" #include "lib/orderman/order_factory/IOrderFactory.h" #include "lib/orderman/order_router/OrderRouter.h" #include "lib/orderman/orderman_utils/ExecutionReport.h" diff --git a/src/lib/orderman/securities/ISecurityDB.h b/src/lib/orderman/securities/ISecurityDB.h index eb7a23fad..427d1171b 100644 --- a/src/lib/orderman/securities/ISecurityDB.h +++ b/src/lib/orderman/securities/ISecurityDB.h @@ -14,7 +14,6 @@ public: virtual ~ISecurityDB() {}; virtual api::Security const *resolve(SecurityKey const&k) const = 0; using SecurityMap = std::unordered_map ; - virtual SecurityMap retrieve_all() const = 0; }; }} diff --git a/src/lib/orderman/securities/MapSecurityDB.h b/src/lib/orderman/securities/MapSecurityDB.h index 2c447d6f9..b2c80252c 100644 --- a/src/lib/orderman/securities/MapSecurityDB.h +++ b/src/lib/orderman/securities/MapSecurityDB.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include "lib/md/SecurityOracle.h" #include "lib/orderman/securities/ISecurityDB.h" namespace bts { @@ -9,15 +9,12 @@ namespace securities { struct MapSecurityDB : public securities::ISecurityDB { + MapSecurityDB(md::SecurityOracle oracle): _oracle(oracle) {} api::Security const *resolve(SecurityKey const&k) const override { - auto it = securities.find(k); - if (it != securities.end()) - return it->second; - else - return nullptr; + return _oracle.get_security(k); } - SecurityMap retrieve_all() const override {return securities;} - SecurityMap securities; + private: + md::SecurityOracle _oracle; }; }} diff --git a/src/lib/orderman/server/OrderServiceServer.cpp b/src/lib/orderman/server/OrderServiceServer.cpp index caf1c67b2..1782d39f7 100644 --- a/src/lib/orderman/server/OrderServiceServer.cpp +++ b/src/lib/orderman/server/OrderServiceServer.cpp @@ -146,7 +146,7 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for OrderManager. See " "the OrdermanCfg key"); - thread->executor.spawn(d,me->log); + thread->executor.spawn(d,me->log, mdc.oracle()); // Timer thread = find_thread_for_actor("Timer", me->actor_threads); @@ -160,7 +160,7 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for AsyncFactory. See " "the OrdermanCfg key"); - thread->executor.spawn(d); + thread->executor.spawn(d, mdc.oracle()); // OrderStore thread = find_thread_for_actor("OrderStore", me->actor_threads); @@ -206,7 +206,7 @@ OrderServiceServer::OrderServiceServer( [this, &ctx](auto const &fill) { ctx.post([this, fill] { this->on_fills(fill); }); }, - me->dyn_order_registry, thread->executor.make_proxy()); + me->dyn_order_registry, thread->executor.make_proxy(),mdc.oracle()); thread->executor.activate(me->service_bridge); From e08cd7bdbf1ea33cd99bce890c56b01f21b202d8 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 6 Feb 2019 14:25:07 -0500 Subject: [PATCH 41/54] Use SecurityOracle in OrderStore --- src/lib/orderman/actors/OrderStore.cpp | 118 +++++++----------- src/lib/orderman/actors/OrderStore.h | 3 +- .../orderman/server/OrderServiceServer.cpp | 2 +- 3 files changed, 48 insertions(+), 75 deletions(-) diff --git a/src/lib/orderman/actors/OrderStore.cpp b/src/lib/orderman/actors/OrderStore.cpp index db6dd64b4..4e5f43b38 100644 --- a/src/lib/orderman/actors/OrderStore.cpp +++ b/src/lib/orderman/actors/OrderStore.cpp @@ -5,7 +5,6 @@ #include "lib/config/protos/orders_fills_cleanup.pb.h" #include "lib/config/protos/toggles.pb.h" #include "lib/orderman/actors/SODPosition.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/events/OrderStateUpdateEvent.h" #include "lib/orderman/events/StoredOrderEvent.h" #include "lib/persistence/SqliteStore.h" @@ -18,99 +17,72 @@ namespace orderman { static auto const orders_ns = "orders"; -enum class OrderStore::State { WAIT_SEC, READY }; - struct OrderStore::PImpl { - PImpl() - : state(State::WAIT_SEC), known_orders(), + PImpl(md::SecurityOracle o) + : known_orders(), store(new persistence::SqliteStore("orders")), order_stream(), soft_cleanup_enabled( - config::get().enabled()) {} + config::get().enabled()), + oracle(o) {} - State state; const GatewayName gw_name; std::unordered_map known_orders; std::unique_ptr store; mercury::Stream order_stream; - mercury::Actor::SubHandle stored_sec_sub; bool soft_cleanup_enabled; + md::SecurityOracle oracle; timeval cutoff_time; }; -OrderStore::OrderStore() : me(new PImpl()) {} +OrderStore::OrderStore(md::SecurityOracle o) : me(new PImpl(o)) {} OrderStore::~OrderStore() {} void OrderStore::init() { - me->state = State::WAIT_SEC; - me->stored_sec_sub = subscribe([this]( - auto const &update) { - if (update.events) { - if (me->state == State::WAIT_SEC) { - LOG(OrderMan, Info, - "Order store received initial list of {} securities", - update.events->size()); + // subscribe to order updates so we can store them + subscribe( + [this](auto const &ev) { this->on_order_state_update(ev); }); - // don't need securities anymore, we just need the initial - // batch so we can filter out orders on expired securities - unsubscribe(me->stored_sec_sub); + // now load orders from disk, for each order check if + // it reffers to a valid security. if so publish it. + api::orderman::OrderRecord rec; + std::vector stored_batch; - // subscribe to order updates so we can store them - subscribe( - [this](auto const &ev) { this->on_order_state_update(ev); }); + LOG(persist, Info, "Loading all orders"); + auto all = me->store->retrieve_all()[orders_ns]; + LOG(persist, Info, "Finished loading all {} orders", all.size()); - // index securities - std::map sec_idx; - for (auto const &ev : *(update.events)) { - auto sec_ptr = ev.security; - sec_idx[SecurityKey(sec_ptr->id())] = sec_ptr; + me->cutoff_time = SODPosition::get_order_cutoff_time(all); + + for (auto const &kvp : all) + if (utils::pb::from_string( + kvp.second, &rec)) { + auto ok = OrderKey(std::stoi(kvp.first)); + auto sk = SecurityKey(rec.def().security_key()); + auto sec = me->oracle.get_security(sk); + + if (config::get() + .enabled() && sec == nullptr) { + LOG(OrderMan, Diag, + "Order {} refers to unresolved security {}", + ok.value(), sk.value()); + } else { + if (!_order_cutoff(rec)) + stored_batch.emplace_back(rec); + me->known_orders[ok] = rec.state(); } + } else + LOG(OrderMan, Error, + "Could not recover order {} from storage. " + "This should be investigated", + kvp.first); - // now load orders from disk, for each order check if - // it reffers to a valid security. if so publish it. - api::orderman::OrderRecord rec; - std::vector stored_batch; - - LOG(persist, Info, "Loading all orders"); - auto all = me->store->retrieve_all()[orders_ns]; - LOG(persist, Info, "Finished loading all {} orders", all.size()); - - me->cutoff_time = SODPosition::get_order_cutoff_time(all); - - for (auto const &kvp : all) - if (utils::pb::from_string( - kvp.second, &rec)) { - auto ok = OrderKey(std::stoi(kvp.first)); - auto sk = SecurityKey(rec.def().security_key()); - - if (config::get< - config::toggles::FilterOrdersWithDanglingSecurities>() - .enabled() && - sec_idx.find(sk) == sec_idx.end()) { - LOG(OrderMan, Diag, - "Order {} refers to unresolved security {}", - ok.value(), sk.value()); - } else { - if (!_order_cutoff(rec)) - stored_batch.emplace_back(rec); - me->known_orders[ok] = rec.state(); - } - } else - LOG(OrderMan, Error, - "Could not recover order {} from storage. " - "This should be investigated", - kvp.first); - - me->order_stream = make_stream(); - LOG(OrderMan, Info, - "Order Store published {} orders recoved from disk out of {} " - "total orders", - stored_batch.size(), all.size()); - publish_batch(me->order_stream, stored_batch); - me->state = State::READY; - } - } - }); + me->order_stream = make_stream(); + LOG(OrderMan, Info, + "Order Store published {} orders recoved from disk out of {} " + "total orders", + stored_batch.size(), all.size()); + publish_batch(me->order_stream, stored_batch); } void OrderStore::on_order_state_update( diff --git a/src/lib/orderman/actors/OrderStore.h b/src/lib/orderman/actors/OrderStore.h index 535a6cc24..6b46df8ce 100644 --- a/src/lib/orderman/actors/OrderStore.h +++ b/src/lib/orderman/actors/OrderStore.h @@ -2,6 +2,7 @@ #include #include "lib/mercury/actors/Actor.h" +#include "lib/md/SecurityOracle.h" #include "lib/orderman/orderman_utils/orderman_common.h" namespace bts { @@ -15,7 +16,7 @@ class OrderStateUpdateEvent; class OrderStore : public mercury::Actor { public: - OrderStore(); + OrderStore(md::SecurityOracle); ~OrderStore(); void init() override; diff --git a/src/lib/orderman/server/OrderServiceServer.cpp b/src/lib/orderman/server/OrderServiceServer.cpp index 1782d39f7..7c157ca4f 100644 --- a/src/lib/orderman/server/OrderServiceServer.cpp +++ b/src/lib/orderman/server/OrderServiceServer.cpp @@ -167,7 +167,7 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for OrderStore. See " "the OrdermanCfg key"); - thread->executor.spawn(); + thread->executor.spawn(mdc.oracle()); // FillStore thread = find_thread_for_actor("FillStore", me->actor_threads); From e05fc076e88748e087b0e083cef58d35dbc88ebc Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 7 Feb 2019 10:32:13 -0500 Subject: [PATCH 42/54] Update fills to use SecurityStore.join --- .../spark/Modules/Trading/Fills/FillsVM.cs | 5 +- src/client/spark/Orders/FillTuple.cs | 10 +- src/client/spark/Orders/FillTupleSource.cs | 99 ++++++------------- src/client/spark/Orders/OrdersModel.cs | 6 +- src/client/spark/Security/MDLookup.cs | 30 ++++++ src/client/spark/Security/SecurityStore.cs | 4 +- .../spark/view/Spark/PAndL/PAndLModel.cs | 9 +- src/client/spark/view/Spark/PAndL/PnLData.cs | 14 ++- 8 files changed, 85 insertions(+), 92 deletions(-) create mode 100644 src/client/spark/Security/MDLookup.cs diff --git a/src/client/spark/Modules/Trading/Fills/FillsVM.cs b/src/client/spark/Modules/Trading/Fills/FillsVM.cs index 595108167..f3586e228 100644 --- a/src/client/spark/Modules/Trading/Fills/FillsVM.cs +++ b/src/client/spark/Modules/Trading/Fills/FillsVM.cs @@ -39,7 +39,7 @@ namespace mbts.spark { user_filter = DiscreteValuesFilterVM.Create(_model.factory.subscribe_users().Select(d => d.Values), r => r.user, nameof(Row.user)); security_filter = DiscreteValuesFilterVM.Create( fills_source.with_respect_to( - ft => ft.security_md.HasValue ? ft.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable, + ft => ft.security ?? SecurityDataHelper.Defaults.SecurityImmutable, s => s.Id, ft => ft.key ).Select(t => (t.added.Values, t.removed.Values)), r => r.security, @@ -186,7 +186,8 @@ namespace mbts.spark { readonly FillImmutable _fill; readonly OrderRecordImmutable _order; - public Row(FillTuple fill_tuple, OrdersModel model) : base(fill_tuple.security_md.HasValue ? fill_tuple.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable) { + public Row(FillTuple fill_tuple, OrdersModel model) : + base(fill_tuple.security ?? SecurityDataHelper.Defaults.SecurityImmutable) { _fill = fill_tuple.fill; _order = fill_tuple.order ?? OrderDataHelper.Defaults.OrderRecordImmutable; _model = model; diff --git a/src/client/spark/Orders/FillTuple.cs b/src/client/spark/Orders/FillTuple.cs index 4481d2ffa..80f7eb946 100644 --- a/src/client/spark/Orders/FillTuple.cs +++ b/src/client/spark/Orders/FillTuple.cs @@ -10,14 +10,14 @@ namespace mbts.spark { class FillTuple : IEquatable { public readonly FillImmutable fill; public readonly OrderRecordImmutable? order; - public readonly SecurityMD? security_md; + public readonly SecurityImmutable? security; public readonly FillTupleKey key; - public FillTuple(FillImmutable fill, OrderRecordImmutable? order, SecurityMD? security_md) { + public FillTuple(FillImmutable fill, OrderRecordImmutable? order, SecurityImmutable? security) { this.fill = fill; this.order = order; - this.security_md = security_md; - key = new FillTupleKey(fill.FillKey, order.HasValue ? order.Value.Key : 0u, security_md.HasValue ? security_md.Value.security.Id : 0u); + this.security = security; + key = new FillTupleKey(fill.FillKey, order.HasValue ? order.Value.Key : 0u, security?.Id ?? 0); } public string account { @@ -31,7 +31,7 @@ namespace mbts.spark { public bool Equals(FillTuple other) => other != null && other.order.HasValue == order.HasValue - && other.security_md.HasValue == security_md.HasValue + && other.security.HasValue == security.HasValue && other.fill.Equals(fill); public override bool Equals(object obj) => Equals(obj as FillTuple); diff --git a/src/client/spark/Orders/FillTupleSource.cs b/src/client/spark/Orders/FillTupleSource.cs index 18629764e..dc8962b70 100644 --- a/src/client/spark/Orders/FillTupleSource.cs +++ b/src/client/spark/Orders/FillTupleSource.cs @@ -1,4 +1,5 @@ -using bts.api.core; +using bts.api; +using bts.api.core; using bts.api.orderman; using mbts.collections; using mbts.utils; @@ -14,15 +15,14 @@ namespace mbts.spark { // returns a ref-count Observable.Create-wrapper that initializes/destroys the [hot-observable] FillTupleSource. // (initializes when the first subscription is made, and destroys when the last subscription is disposed). public static IObservable> create_observable( - IObservable> securities, - IObservable> orders, - IObservable> fills) { + IObservable> orders, + IObservable> fills) { FillTupleSource source = null; int ref_count = 0; return Observable.Create>(observer => { if (Interlocked.Increment(ref ref_count) == 1) { SparkUtils.lazy_assert(source == null, () => fmt_assert_msg("source should be null, there must be a leak.")); - source = new FillTupleSource(securities, orders, fills); + source = new FillTupleSource(orders, fills); } IDisposable sub = source.Subscribe(observer); return Disposable.Create(() => { @@ -38,19 +38,16 @@ namespace mbts.spark { readonly Dictionary _state = new Dictionary(); readonly SetMap _order_fills = new SetMap(); readonly SetMap _security_fills = new SetMap(); - IReadOnlyDictionary _fills_cache = new Dictionary(0); - IReadOnlyDictionary _orders_cache = new Dictionary(0); - IReadOnlyDictionary _security_md_cache = new Dictionary(0); + IReadOnlyDictionary _fills_cache = new Dictionary(); + IReadOnlyDictionary _orders_cache = new Dictionary(); IDisposable _dispose; object _lock = new object(); FillTupleSource( - IObservable> securities, - IObservable> orders, - IObservable> fills) => + IObservable> orders, + IObservable> fills) => // _dispose = Disposable.Create( - securities.Subscribe(_update_securities), orders.Subscribe(_update_orders), fills.AddRemoveOnly().Subscribe(_update_fills)); @@ -63,19 +60,19 @@ namespace mbts.spark { public void Dispose() => Disposable.dispose(ref _dispose); - void _update_fills(LegacyTableUpdate fills) { + void _update_fills(LegacyTableUpdate<(SecurityImmutable?,FillImmutable), string> fills) { lock (_lock) { var removed = new Dictionary(); var added = new Dictionary(); - foreach (FillImmutable fill in fills.removed) { - FillTuple old_ft = new FillTuple(fill, _get_order(fill.OrderKey), _get_security_md(fill.SecurityKey)); + foreach (var (security,fill) in fills.removed) { + FillTuple old_ft = new FillTuple(fill, _get_order(fill.OrderKey), security); removed.Add(old_ft.key, old_ft); _security_fills.remove(fill.SecurityKey, fill.FillKey); _order_fills.remove(fill.OrderKey, fill.FillKey); } - foreach (FillImmutable fill in fills.added) { - FillTuple new_ft = new FillTuple(fill, _get_order(fill.OrderKey), _get_security_md(fill.SecurityKey)); + foreach (var (security,fill) in fills.added) { + FillTuple new_ft = new FillTuple(fill, _get_order(fill.OrderKey), security); added.Add(new_ft.key, new_ft); _security_fills.add(fill.SecurityKey, fill.FillKey); _order_fills.add(fill.OrderKey, fill.FillKey); @@ -85,35 +82,33 @@ namespace mbts.spark { } } - void _update_orders(TableUpdate orders) { + void _update_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable), uint> orders) { lock (_lock) { Dictionary added = new Dictionary(), removed = new Dictionary(); HashSet changed_keys = new HashSet(); - foreach (var o in orders.removed) { + foreach (var (_,o) /*winking owl*/ in orders.removed) { if (_order_fills.try_lookup(o.Key, out ISet fill_keys)) { OrderRecordImmutable? curr_order = null; - if (orders.try_get(o.Key, out OrderRecordImmutable newval)) { - curr_order = newval; + if (orders.try_get(o.Key, out var newval)) { + curr_order = newval.Item2; changed_keys.Add(o.Key); } - IEnumerable fills = fill_keys.Select(fk => _fills_cache[fk]); - foreach (FillImmutable fill in fills) { - SecurityMD? sec_md = _get_security_md(fill.SecurityKey); - FillTuple old_ft = new FillTuple(fill, o, sec_md); + var fills = fill_keys.Select(fk => _fills_cache[fk]); + foreach (var (security, fill) in fills) { + FillTuple old_ft = new FillTuple(fill, o, security); removed.Add(old_ft.key, old_ft); - FillTuple new_ft = new FillTuple(fill, curr_order, sec_md); + FillTuple new_ft = new FillTuple(fill, curr_order, security); added.Add(new_ft.key, new_ft); } } } - foreach (var o in orders.added) { + foreach (var (_,o) in orders.added) { if (!changed_keys.Contains(o.Key) && _order_fills.try_lookup(o.Key, out ISet fill_keys)) { - IEnumerable fills = fill_keys.Select(fk => _fills_cache[fk]); - foreach (FillImmutable fill in fills) { - SecurityMD? sec_md = _get_security_md(fill.SecurityKey); - FillTuple old_ft = new FillTuple(fill, null, sec_md); + var fills = fill_keys.Select(fk => _fills_cache[fk]); + foreach (var (security,fill) in fills) { + FillTuple old_ft = new FillTuple(fill, null, security); removed.Add(old_ft.key, old_ft); - FillTuple new_ft = new FillTuple(fill, o, sec_md); + FillTuple new_ft = new FillTuple(fill, o, security); added.Add(new_ft.key, new_ft); } } @@ -123,46 +118,8 @@ namespace mbts.spark { } } - void _update_securities(IReadOnlyDictionary securities) { - lock (_lock) { - Dictionary added = new Dictionary(), removed = new Dictionary(); - _security_md_cache.Keys.difference(securities.Keys, out IEnumerable added_keys, out IEnumerable removed_keys); - foreach (var k in removed_keys) { - if (_security_fills.try_lookup(k, out ISet fill_keys)) { - IEnumerable fills = fill_keys.Select(fk => _fills_cache[fk]); - SecurityMD old_sec_md = _security_md_cache[k]; - foreach (FillImmutable fill in fills) { - OrderRecordImmutable? order = _get_order(fill.OrderKey); - FillTuple old_ft = new FillTuple(fill, order, old_sec_md); - removed.Add(old_ft.key, old_ft); - FillTuple new_ft = new FillTuple(fill, order, null); - added.Add(new_ft.key, new_ft); - } - } - } - foreach (var k in added_keys) { - if (_security_fills.try_lookup(k, out ISet fill_keys)) { - IEnumerable fills = fill_keys.Select(fk => _fills_cache[fk]); - SecurityMD new_sec_md = securities[k]; - foreach (FillImmutable fill in fills) { - OrderRecordImmutable? order = _get_order(fill.OrderKey); - FillTuple old_ft = new FillTuple(fill, order, null); - removed.Add(old_ft.key, old_ft); - FillTuple new_ft = new FillTuple(fill, order, new_sec_md); - added.Add(new_ft.key, new_ft); - } - } - } - _security_md_cache = securities; - _apply_changes(added, removed); - } - } - - SecurityMD? _get_security_md(uint key) => - _security_md_cache.TryGetValue(key, out SecurityMD smd) ? smd : new SecurityMD?(); - OrderRecordImmutable? _get_order(uint key) => - _orders_cache.TryGetValue(key, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?(); + _orders_cache.TryGetValue(key, out var o) ? o.Item2 : new OrderRecordImmutable?(); void _apply_changes(Dictionary added, Dictionary removed) { foreach (var kv in removed) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index c5892f4a4..0503e569e 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -71,10 +71,10 @@ namespace mbts.spark { _service.subscribei_fills() .Naggle(interval, scheduler) .PermissionFilter(admin_model.on_admin_data) - //.join(sstore, f => ) + .join(sstore, f => f.SecurityKey) .Table().Cache(); _activity = new OrderActivitySummarizer(bus, on_orders); - on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders.Select(i => i.order, order => order.Key), on_fills); + on_fill_tuples = FillTupleSource.create_observable(on_orders, on_fills); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); _security_positions_demuxer = new TableUpdateDemuxer(o => o.RecordId, p => p.SecurityId); _dispose = Disposable.Create( @@ -94,7 +94,7 @@ namespace mbts.spark { public DynamicOrderHelper dynamic_helper => _dynamic_helper; public IObservable on_orders { get; } public IObservable> on_order_securities { get; } - public IObservable> on_fills { get; } + public IObservable> on_fills { get; } public IObservable> on_fill_tuples { get; } public IObservable> working_orders { get; } public IObservable> on_positions { get; } diff --git a/src/client/spark/Security/MDLookup.cs b/src/client/spark/Security/MDLookup.cs new file mode 100644 index 000000000..e7949dda7 --- /dev/null +++ b/src/client/spark/Security/MDLookup.cs @@ -0,0 +1,30 @@ +using mbts; +using System; +using System.Collections.Generic; +using System.ComponentModel.Composition; +using System.Reactive.Linq; + +namespace bts { + class MDLookup : IDisposable { + IReadOnlyDictionary _state; + IDisposable _dispose; + + [ImportingConstructor] + public MDLookup(IObservable> src) { + _dispose = src.Subscribe(s => _state = s); + } + + public void Dispose() { Disposable.dispose(ref _dispose); } + + public SecurityMD? get_md(uint key) { + if (_state?.TryGetValue(key, out var md) == true) { + return md; + } + return null; + } + public SecurityMD? get_md(uint? key) { + if (key.HasValue) return get_md(key.Value); + return null; + } + } +} diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 731ca169f..cf2dd306d 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -139,11 +139,11 @@ namespace bts { /////////////// #region public - // Guarunteed to return a security if it has ever been seen by the system + // Guaranteed to return a security if it has ever been seen by the system public Task get_security(security_key key) { return get_security(key, _live_securities); } - // Guarunteed to return a security if it has ever been seen by the system + // Guaranteed to return a security if it has ever been seen by the system public Task> get_securities(IEnumerable keys) { return get_securities(keys, _live_securities); } diff --git a/src/client/spark/view/Spark/PAndL/PAndLModel.cs b/src/client/spark/view/Spark/PAndL/PAndLModel.cs index 558f2305a..9ffa2d95b 100644 --- a/src/client/spark/view/Spark/PAndL/PAndLModel.cs +++ b/src/client/spark/view/Spark/PAndL/PAndLModel.cs @@ -1,4 +1,5 @@ -using bts.api; +using bts; +using bts.api; using mbts.utils; using System; using System.ComponentModel.Composition; @@ -8,18 +9,18 @@ namespace mbts.spark { class PAndLModel : IDisposable { readonly IBus _bus; readonly OrdersModel _order_model; - //readonly AccountService _account_service; + readonly MDLookup _mdlookup; IDisposable _dispose; [ImportingConstructor] private PAndLModel(IBus bus, OrdersModel order_model/*, AccountService account_service*/) { _bus = bus; _order_model = order_model; - //_account_service = account_service; + _mdlookup = new MDLookup(bus.subscribe(SparkTags.SecuritiesTag)); } public IObservable> on_pnl_data => - _order_model.on_fill_tuples.convert(ft => new PnLData(ft)).table(pd => pd.key); + _order_model.on_fill_tuples.Select(ft => new PnLData(ft, _mdlookup.get_md(ft.security?.Id))).table(pd => pd.key); //public async Task get_accounts() { // var request = await _account_service.get_accounts_async(new GetAccountsRequest()); diff --git a/src/client/spark/view/Spark/PAndL/PnLData.cs b/src/client/spark/view/Spark/PAndL/PnLData.cs index 99c441c16..5b44015a5 100644 --- a/src/client/spark/view/Spark/PAndL/PnLData.cs +++ b/src/client/spark/view/Spark/PAndL/PnLData.cs @@ -28,19 +28,23 @@ namespace mbts.spark { class PnLData { readonly FillTuple _fill_tuple; + readonly SecurityMD? _md; - public PnLData(FillTuple fill_tuple) => + public PnLData(FillTuple fill_tuple, SecurityMD? md) { + // It is entirely possible for the FillTuple to have a security but for no SecrityMD to be available. For example: expired security + _md = md; _fill_tuple = fill_tuple; + } public FillTupleKey key => _fill_tuple.key; public FillImmutable fill => _fill_tuple.fill; - public SecurityMD? sec_md => _fill_tuple.security_md; + public SecurityMD? sec_md => _md; + public SecurityImmutable? security => _fill_tuple.security; public OrderRecordImmutable? order => _fill_tuple.order; public string account => _fill_tuple.account; - public Venue? exchange => sec_md?.security.Exchange; - public string symbol => sec_md?.security.Symbol; - public SecurityImmutable? security => sec_md?.security; + public Venue? exchange => security?.Exchange; + public string symbol => security?.Symbol; public double buy_quantity => (fill.Side == Side.Buy) ? fill.LastShares : 0; public double sell_quantity => (fill.Side == Side.Sell) ? fill.LastShares : 0; public SecurityType? security_type => security?.security_type(); From 1697a1a0159f5dabba0aa37c413285415656c2b2 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 7 Feb 2019 11:25:37 -0500 Subject: [PATCH 43/54] Cease barbarism --- src/client/spark/Orders/FillTupleSource.cs | 4 ++-- src/lib/CMakeDeps_win.cmake | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/spark/Orders/FillTupleSource.cs b/src/client/spark/Orders/FillTupleSource.cs index dc8962b70..679ca6a1f 100644 --- a/src/client/spark/Orders/FillTupleSource.cs +++ b/src/client/spark/Orders/FillTupleSource.cs @@ -82,7 +82,7 @@ namespace mbts.spark { } } - void _update_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable), uint> orders) { + void _update_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable order), uint> orders) { lock (_lock) { Dictionary added = new Dictionary(), removed = new Dictionary(); HashSet changed_keys = new HashSet(); @@ -90,7 +90,7 @@ namespace mbts.spark { if (_order_fills.try_lookup(o.Key, out ISet fill_keys)) { OrderRecordImmutable? curr_order = null; if (orders.try_get(o.Key, out var newval)) { - curr_order = newval.Item2; + curr_order = newval.order; changed_keys.Add(o.Key); } var fills = fill_keys.Select(fk => _fills_cache[fk]); diff --git a/src/lib/CMakeDeps_win.cmake b/src/lib/CMakeDeps_win.cmake index afd768425..f3e9584f4 100644 --- a/src/lib/CMakeDeps_win.cmake +++ b/src/lib/CMakeDeps_win.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file From cdce7cd2422dfc87f8b745e904c353709577b1bf Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 7 Feb 2019 17:36:38 -0500 Subject: [PATCH 44/54] Bugfix: SecurityCache is shared_ptr pimpl and should be stored by value instead of by ref even though this messes with const --- src/lib/md/MDConsumer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index a532cdf1a..d1659d628 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -537,7 +537,7 @@ class MDConsumerImpl { std::shared_ptr _sci; map> _deferred_handles; vector _callbacks; - SecurityCache& _cache; + mutable SecurityCache _cache; // co2::Platform _platform; public: From 785839327ebe31cef07fc90fe2dfaf67138097c1 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 7 Feb 2019 17:36:56 -0500 Subject: [PATCH 45/54] Add extra safety check that a given security is not serialized more than once --- src/client/spark/Security/SecurityStore.cs | 12 ++++++++---- src/lib/md/relay/QueryServer.cpp | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index cf2dd306d..ed848ef2b 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -101,21 +101,25 @@ namespace bts { // Request any missing keys from the server but do not wait for them to arrive IReadOnlyDictionary get_and_ensure_securities(IEnumerable keys) { var result = new Dictionary(); - var missing = new SecurityKeyList(); + var missing = new HashSet(); foreach (var key in keys) { var security = try_get_local(key); if (security.HasValue) { result.TryAdd(key, security.Value); } else { - missing.Keys.Add(key); + missing.Add(key); } } // Update cache in the background - if (missing.Keys.Count > 0) { + if (missing.Count > 0) { Task.Run(async () => { - var r = await _md.get_securities_async(missing); + var skl = new SecurityKeyList(); + skl.Keys.AddRange(missing); + var r = await _md.get_securities_async(skl); if (r.success) { + var retrieved = r.value.Security.Select(s => s.Id); + Log.Md.info($"Loaded securities from cold storage [{string.Join(",", retrieved)}]"); foreach (var security in r.value.Security) { try_cache_security(new SecurityImmutable(security)); } diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 5b44513d4..e8331308b 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -382,7 +382,9 @@ public: void get_securities(google::protobuf::RepeatedField keys, api::SecurityList* result) const { bts_assert(result); - for (auto k : keys) { + // Ensure keys are unique + std::set unique_keys(keys.begin(), keys.end()); + for (auto k : unique_keys) { auto ptr = _consumer.get_security(SecurityKey(k)); if (ptr) { result->add_security()->CopyFrom(*ptr); From 10e174c5b4519d9b564204b786587d35fc9394c7 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 11 Feb 2019 09:11:55 -0500 Subject: [PATCH 46/54] Updated release notes. --- doc/user/docs/release_notes.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index d3b1f2a87..5621f6bca 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -1,11 +1,16 @@ # Release Notes ## 1.5.0 -### Bug Fixes +### User Visible Changes +#### Bug Fixes - Fixed the "Since Open" time filter to use the exchange open time rather than 8:30 AM -### SDK +#### SDK - Improved SDK documentation and usability - +### Internal Changes + - Improved robustness of limit system test + - Extensive refactoring to the market data subsystem to allow for + updates of security definitions + ## 1.4.11 ### New Features - Floating Order Book From 37f595ed9a19863bbef697cc14c179e843abdd99 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 11 Feb 2019 09:55:18 -0500 Subject: [PATCH 47/54] Add startup delay for security discovery --- src/lib/config/defaults/MarketData.conf | 13 +++++++++++++ src/lib/config/protos/MarketData.proto | 11 +++++++++++ src/lib/utils/datetime.cpp | 21 +++++++++++++++++++++ src/lib/utils/datetime.h | 4 ++++ src/server/sparkd/md/run_md.cpp | 24 ++++++++++++++++++++++-- 5 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/lib/config/defaults/MarketData.conf b/src/lib/config/defaults/MarketData.conf index 17d800d6b..0a11b62c6 100644 --- a/src/lib/config/defaults/MarketData.conf +++ b/src/lib/config/defaults/MarketData.conf @@ -35,3 +35,16 @@ CMEOptions { } } +MDBootDelayWindow { + value { + delay_seconds: 120 + standard { + start: "16:30" + end: "16:40" + } + sunday { + start: "14:00" + end: "14:10" + } + } +} \ No newline at end of file diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index 08b397958..aba6237c6 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -15,6 +15,17 @@ message PlatformMap optional string glbx_platform = 1 [default = "PLATFORM_SIM"]; } +// LOCAL TIME +message MDBootDelayWindow { + message TimeSpan { + optional string start = 1 [default = "16:30"]; + optional string end = 2 [default = "16:40"]; + } + optional uint32 delay_seconds = 1 [default = 120]; + optional TimeSpan standard = 2; + optional TimeSpan sunday = 3; +} + // // Options that apply to market data server diff --git a/src/lib/utils/datetime.cpp b/src/lib/utils/datetime.cpp index 5e776e159..4cd7fe092 100755 --- a/src/lib/utils/datetime.cpp +++ b/src/lib/utils/datetime.cpp @@ -1,5 +1,6 @@ #include "datetime.h" + #include "bts_assert.h" #include "TimeStamp.h" @@ -317,6 +318,26 @@ Date ll_to_date(LLDateTime time) { return date; } + +bool is_local_time_sunday() { + auto now = local_now(); + return now.date().day_of_week() == boost::date_time::Sunday; +} +bool is_local_time_between(int hh_start, int mm_start, int hh_end, int mm_end) { + auto now = local_now(); + auto hh_now = now.time_of_day().hours(); + auto mm_now = now.time_of_day().minutes(); + + if (hh_now > hh_start && hh_now < hh_end) return true; + if (hh_now == hh_start || hh_now == hh_end) { + if (mm_now >= mm_start && mm_now <= mm_end) { + return true; + } + } + return false; +} + + struct UtcTimeStamp { int hour; int min; diff --git a/src/lib/utils/datetime.h b/src/lib/utils/datetime.h index e6095ad82..133555edc 100755 --- a/src/lib/utils/datetime.h +++ b/src/lib/utils/datetime.h @@ -76,11 +76,15 @@ BTS_EXPORT bool is_at_least_days_old (PTime const &time, uint64_t days); BTS_EXPORT LLDateTime ll_from_unix_epoch(long); BTS_EXPORT LLDateTime ll_from_unix_epoch_ms(long long); + extern "C" BTS_EXPORT void ll_to_parts(LLDateTime, int& year, int& month, int& day, int& hour, int& minute, int& second, int& millisecond); BTS_EXPORT Date ll_to_date(LLDateTime); extern "C" BTS_EXPORT LLDateTime ll_from_parts(int year, int month, int day, int hour, int minute, int second, int millisecond); BTS_EXPORT LLDateTime ll_from_date(const Date& date); +bool is_local_time_sunday(); +bool is_local_time_between(int hh_start, int mm_start, int hh_end, int mm_end); + #ifndef NO_CPP0X extern "C" BTS_EXPORT LLDateTime ll_today(); BTS_EXPORT Date today(); diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index 6bb2a2ef5..56e2c2762 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -90,6 +90,26 @@ void run_market_data_publishers_forever() { } } +void boot_sleep() { + auto conf = config::get(); + const auto* schedule = &conf.standard(); + if (datetime::is_local_time_sunday()) { + schedule = &conf.sunday(); + } + bts_assert(schedule); + int hh_start = 0, mm_start = 0, hh_end = 0, mm_end = 0; + sscanf(schedule->start().c_str(), "%i:%i", &hh_start, &mm_start); + sscanf(schedule->end().c_str(), "%i:%i", &hh_end, &mm_end); + if (datetime::is_local_time_between(hh_start, mm_start, hh_end, mm_end)) { + std::this_thread::sleep_for( + std::chrono::seconds( + conf.delay_seconds())); + } + else { + std::this_thread::sleep_for(2s); + } +} + void run_market_datapublishers() { bts_assert(_md_thread == nullptr); run_cleanup(); @@ -103,10 +123,10 @@ void run_market_datapublishers() { threading::make_thread("ExchangeSim", []() { sim::start_exchange_sim(); }) .detach(); } - - std::this_thread::sleep_for(2s); + boot_sleep(); } + void run_record(const std::string& arg){ boost::filesystem::path p(arg); record_everything(p.parent_path().string(), p.filename().string()); From 883373f7e515770486e91d7b048515ee31d1e204 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 11 Feb 2019 09:58:30 -0500 Subject: [PATCH 48/54] Bugfix: SyntheticStrategy.strategy is now null initially --- src/lib/md/MDConsumer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index d1659d628..e296fc1d6 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -483,7 +483,7 @@ private: if (legs.size() == (size_t)strategy.security->synthetic().legs_size()) { auto s = make_shared(); s->security.CopyFrom(*strategy.security); - s->strategy->CopyFrom(strategy.strategy); + s->strategy = std::make_unique(strategy.strategy); s->legs = legs; s->updated = TimeStamp::now(); result.insert( From 2925cf50927aded02ad76cdc4f34d221306ba1f9 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 11 Feb 2019 16:10:51 -0500 Subject: [PATCH 49/54] CR Feedback --- src/lib/config/defaults/MarketData.conf | 7 +++++++ src/lib/utils/datetime.cpp | 15 ++++----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/lib/config/defaults/MarketData.conf b/src/lib/config/defaults/MarketData.conf index 0a11b62c6..08603497a 100644 --- a/src/lib/config/defaults/MarketData.conf +++ b/src/lib/config/defaults/MarketData.conf @@ -35,6 +35,13 @@ CMEOptions { } } +MDBootDelayWindow { + rule: "DeploymentType.type == 'dev'" + value { + delay_seconds: 0 + } +} + MDBootDelayWindow { value { delay_seconds: 120 diff --git a/src/lib/utils/datetime.cpp b/src/lib/utils/datetime.cpp index 4cd7fe092..8c33965e4 100755 --- a/src/lib/utils/datetime.cpp +++ b/src/lib/utils/datetime.cpp @@ -324,20 +324,13 @@ bool is_local_time_sunday() { return now.date().day_of_week() == boost::date_time::Sunday; } bool is_local_time_between(int hh_start, int mm_start, int hh_end, int mm_end) { + using namespace boost::posix_time; auto now = local_now(); - auto hh_now = now.time_of_day().hours(); - auto mm_now = now.time_of_day().minutes(); - - if (hh_now > hh_start && hh_now < hh_end) return true; - if (hh_now == hh_start || hh_now == hh_end) { - if (mm_now >= mm_start && mm_now <= mm_end) { - return true; - } - } - return false; + auto start = ptime(now.date(), hours(hh_start) + minutes(mm_start)); + auto end = ptime(now.date(), hours(hh_end) + minutes(mm_end)); + return (now >= start && now <= end); } - struct UtcTimeStamp { int hour; int min; From f6f9605a83a1df4fd94131bbf952018b386303a2 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 12 Feb 2019 12:26:11 -0500 Subject: [PATCH 50/54] Revert "Try to ensure Dispatcher::background is terminated" This reverts commit 5cc408557727828ad09536b625dbf1f1cb6fc7ff. --- src/lib/utils/init_spark_lib.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/utils/init_spark_lib.cpp b/src/lib/utils/init_spark_lib.cpp index 772d62c7a..d9dc6fab9 100644 --- a/src/lib/utils/init_spark_lib.cpp +++ b/src/lib/utils/init_spark_lib.cpp @@ -2,7 +2,6 @@ #include "lib/logging/logging.h" #include "lib/utils/bts_assert.h" #include "lib/utils/thread_manager.h" -#include "lib/utils/Dispatcher.h" namespace bts { namespace utils { @@ -21,7 +20,6 @@ void finit_spark_lib() if (poller) { delete poller; } - utils::Dispatcher::background().terminate(); } }} From 15750383652b6c960fe0edbab2ed0604755084b3 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 13 Feb 2019 08:27:28 -0500 Subject: [PATCH 51/54] Fix equality on OrderSecurity type --- src/client/spark/Orders/OrderSecurity.cs | 2 +- src/client/spark/Orders/OrdersModel.cs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/client/spark/Orders/OrderSecurity.cs b/src/client/spark/Orders/OrderSecurity.cs index dd9936162..a703e4c2b 100644 --- a/src/client/spark/Orders/OrderSecurity.cs +++ b/src/client/spark/Orders/OrderSecurity.cs @@ -15,7 +15,7 @@ namespace bts.api.orderman { public OrderRecordImmutable order { get; } public override bool Equals(object obj) => (obj is OrderSecurity os) && Equals(os); - public bool Equals(OrderSecurity other) => order.Key == other.order.Key && security == other.security; + public bool Equals(OrderSecurity other) => order.Equals(other.order) && security == other.security; public override int GetHashCode() => (security, order).GetHashCode(); public static bool operator == (OrderSecurity left, OrderSecurity right) => left.Equals(right); diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 0503e569e..b41edc81f 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -175,12 +175,12 @@ namespace mbts.spark { clone.Remove(o.order.Key); } } - foreach (var o in table.changed) { - if (o.order.is_active()) { - clone[o.order.Key] = o; + foreach (var o in table.changes) { + if (o.Old.order.is_active()) { + clone.Remove(o.Old.order.Key); } - else { - clone.Remove(o.order.Key); + if (o.New.order.is_active()) { + clone.Add(o.New.order.Key, o.New); } } foreach (var o in table.added) { From 2cb485a883e55670a2fc9853404e6deba5a8de09 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 13 Feb 2019 13:18:36 -0500 Subject: [PATCH 52/54] Add note for ui_perf manual test --- testing/manual/ladder/ui_performance.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/manual/ladder/ui_performance.txt b/testing/manual/ladder/ui_performance.txt index 9fb54cc18..ee27662c8 100644 --- a/testing/manual/ladder/ui_performance.txt +++ b/testing/manual/ladder/ui_performance.txt @@ -14,3 +14,4 @@ This test needs to run against production market data or high quality recorded d 4. Repeat 2. +5. If there is *any* doubt that the UI is snappy bring up the previous release and compare. At minimum we need to not regress UI performance. From 0f8b996c8ca8d9a6a0b0c1b54ea20ae5d58af7d5 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 13 Feb 2019 13:44:07 -0500 Subject: [PATCH 53/54] Reduce config connection timeout to 6 seconds from 30 --- src/lib/config/ConfigDownloader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/config/ConfigDownloader.cpp b/src/lib/config/ConfigDownloader.cpp index 2c6e94af8..da23232eb 100755 --- a/src/lib/config/ConfigDownloader.cpp +++ b/src/lib/config/ConfigDownloader.cpp @@ -53,7 +53,7 @@ ConfigDownloader::Result ConfigDownloader::get(const string& url, stringstream& curl_easy_setopt(_curl, CURLOPT_WRITEDATA, &out_stream); curl_easy_setopt(_curl, CURLOPT_VERBOSE, 0); curl_easy_setopt(_curl, CURLOPT_FAILONERROR, 1); - curl_easy_setopt(_curl, CURLOPT_CONNECTTIMEOUT, 30); + curl_easy_setopt(_curl, CURLOPT_CONNECTTIMEOUT, 6); curl_easy_setopt(_curl, CURLOPT_SOCKOPTFUNCTION, no_linger); CURLcode res = curl_easy_perform(_curl); From 7212049c0cb890330c6665fc2b48b4b634cb952c Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 14 Feb 2019 12:37:34 -0500 Subject: [PATCH 54/54] ignore *.code-workspace files --- .gitignore | 1 + remote-sparkvm.code-workspace | 27 --------------------------- 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 remote-sparkvm.code-workspace diff --git a/.gitignore b/.gitignore index d87a92c1b..a8d93ca09 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ src/server/sparkd/orderman/perf_analysis/logs/ src/server/sparkd/orderman/perf_analysis/trace_data/ .fake *.fsx.lock +*.code-workspace src/lib/orderman/perf_analysis/logs/latency_data/ trace_data/ /src/lib/orderman/perf_analysis/last_pretty_log.csv diff --git a/remote-sparkvm.code-workspace b/remote-sparkvm.code-workspace deleted file mode 100644 index 03c875df7..000000000 --- a/remote-sparkvm.code-workspace +++ /dev/null @@ -1,27 +0,0 @@ -{ - "folders": [ - { - "path": "." - }, - { - "uri": "sftp://bts:BT5u53r@sparkvm:22/home/bts/", - "name": "sparkvm" - } - ], - "settings": { - "files.associations": { - "utility": "cpp", - "initializer_list": "cpp", - "list": "cpp", - "type_traits": "cpp", - "vector": "cpp", - "xhash": "cpp", - "xstring": "cpp", - "system_error": "cpp", - "array": "cpp", - "string_view": "cpp", - "xutility": "cpp", - "*.ipp": "cpp" - } - } -} \ No newline at end of file