From 2fc4c8bd6061c0f13393afe57039ca784c91574a Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 14 Nov 2018 19:08:57 -0500 Subject: [PATCH 1/6] Move sqlite_utils.h out of utils:: namespace --- src/lib/md/SecurityDB.cpp | 23 ++++++++++++----------- src/lib/md/SettlementRecorder.cpp | 15 ++++++++------- src/lib/persistence/SqliteStore.cpp | 27 ++++++++++++++------------- src/lib/utils/Sqlite_utils.cpp | 17 +++++++++++------ src/lib/utils/Sqlite_utils.h | 15 +++++++-------- 5 files changed, 52 insertions(+), 45 deletions(-) diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index d84fbe4d4..64b33cf11 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -34,16 +34,17 @@ sqlite3_ptr open_db() { } // Create table if it does not exist - auto dbptr = utils::open_db(db_path.c_str()); + auto dbptr = sqlite::open_db(db_path.c_str()); + bts_assert(dbptr); auto db = dbptr.get(); sqlite3_busy_timeout(db, 9999); - sqlite3_exec_nofail(db, "PRAGMA journal_mode=OFF"); - sqlite3_exec_nofail(db, "PRAGMA syncronous=0"); + sqlite::exec_nofail(db, "PRAGMA journal_mode=OFF"); + sqlite::exec_nofail(db, "PRAGMA syncronous=0"); - sqlite3_exec_nofail(db, + sqlite::exec_nofail(db, "CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);"); - sqlite3_exec_nofail(db, + sqlite::exec_nofail(db, "CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); return dbptr; @@ -67,7 +68,7 @@ size_t load_securities( // Load all existing securities and call the callback auto db = dbptr.get(); - auto select_stmt = prepare_query(db, "select key,symbol,venue,type,security_blob from securities where expiration > ?"); + auto select_stmt = sqlite::prepare_query(db, "select key,symbol,venue,type,security_blob from securities where expiration > ?"); auto select = select_stmt.get(); sqlite3_bind_int(select, 1, sec_expiration_cut_off().to_short()); auto result = sqlite3_step(select); @@ -112,8 +113,8 @@ size_t load_securities( void store_if_new(const std::vector& securities) { auto dbptr = open_db(); - auto insert = utils::prepare_query(dbptr, "insert or ignore into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); - sqlite3_exec_nofail(dbptr, "BEGIN TRANSACTION"); + auto insert = sqlite::prepare_query(dbptr, "insert or ignore into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); + sqlite::exec_nofail(dbptr, "BEGIN TRANSACTION"); std::string buffer; for (const api::Security* psecurity : securities) { if (psecurity && psecurity->SerializeToString(&buffer)) { @@ -127,7 +128,7 @@ void store_if_new(const std::vector& securities) { sqlite3_reset(insert.get()); } } - sqlite3_exec_nofail(dbptr, "COMMIT TRANSACTION"); + sqlite::exec_nofail(dbptr, "COMMIT TRANSACTION"); } @@ -135,7 +136,7 @@ std::vector load_metadata() { std::vector values; auto dbptr = open_db(); auto db = dbptr.get(); - auto select_stmt = prepare_query(db, "select key,data from metadata"); + auto select_stmt = sqlite::prepare_query(db, "select key,data from metadata"); auto select = select_stmt.get(); auto result = sqlite3_step(select); while(result == SQLITE_ROW) { @@ -156,7 +157,7 @@ std::vector load_metadata() { } bool store_metadata(const api::synthetic::Metadata& data) { auto dbptr = open_db(); - auto insert = utils::prepare_query(dbptr, "insert or replace into metadata (key, data) values (?,?)"); + auto insert = sqlite::prepare_query(dbptr, "insert or replace into metadata (key, data) values (?,?)"); SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, data.security_key()), dbptr); std::string buffer = data.SerializeAsString(); diff --git a/src/lib/md/SettlementRecorder.cpp b/src/lib/md/SettlementRecorder.cpp index dfa1eb549..0d7680bbd 100644 --- a/src/lib/md/SettlementRecorder.cpp +++ b/src/lib/md/SettlementRecorder.cpp @@ -94,13 +94,14 @@ SettlementRecorder::SettlementRecorder(Platform platform) : } // initialize database - _self->db = open_db(db_path.c_str()); + _self->db = sqlite::open_db(db_path.c_str()); + bts_assert_msg(_self->db, db_path.c_str()); sqlite3* db = _self->db.get(); sqlite3_busy_timeout(db, 9999); - sqlite3_exec_nofail(db, "PRAGMA journal_mode=OFF"); - sqlite3_exec_nofail(db, "PRAGMA syncronous=0"); + sqlite::exec_nofail(db, "PRAGMA journal_mode=OFF"); + sqlite::exec_nofail(db, "PRAGMA syncronous=0"); // key=(security_key+date) - sqlite3_exec_nofail(db, + sqlite::exec_nofail(db, "CREATE TABLE IF NOT EXISTS settlements2 (key INTEGER NOT NULL, date INTEGER NOT NULL, mantissa INTEGER NOT NULL, exponent INTEGER NOT NULL, PRIMARY KEY (key,date));"); // Delete any records > 10 days old @@ -108,11 +109,11 @@ SettlementRecorder::SettlementRecorder(Platform platform) : auto encoded = datetime::ll_to_date(datetime::ptime_to_ll(cutoff)).to_short(); char query[256]; sprintf(query, "delete from settlements2 where date < %i", encoded); - sqlite3_exec_nofail(db, query); + sqlite::exec_nofail(db, query); // statements - _self->select_stmt = prepare_query(db, "select mantissa,exponent,date from settlements2 where key=? order by date desc limit 2"); - _self->insert_stmt = prepare_query(db, "insert or replace into settlements2(key, date, mantissa,exponent) values(?,?,?,?)"); + _self->select_stmt = sqlite::prepare_query(db, "select mantissa,exponent,date from settlements2 where key=? order by date desc limit 2"); + _self->insert_stmt = sqlite::prepare_query(db, "insert or replace into settlements2(key, date, mantissa,exponent) values(?,?,?,?)"); } SettlementRecorder::~SettlementRecorder() { } diff --git a/src/lib/persistence/SqliteStore.cpp b/src/lib/persistence/SqliteStore.cpp index 814ae95f1..34c91ecd2 100644 --- a/src/lib/persistence/SqliteStore.cpp +++ b/src/lib/persistence/SqliteStore.cpp @@ -57,42 +57,43 @@ void SqliteStore::init_db() { LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", me->db, path.string()); - me->conn = utils::open_db(path.c_str()); + me->conn = sqlite::open_db(path.c_str()); + bts_assert_msg(me->conn, path.c_str()); const char *create_schema_sql = "create table store ( ns text, key text, value text)"; - utils::sqlite3_exec_nofail(me->conn, create_schema_sql); + sqlite::exec_nofail(me->conn, create_schema_sql); const char *key_index_sql = "create unique index key_index on store(ns,key)"; - utils::sqlite3_exec_nofail(me->conn, key_index_sql); + sqlite::exec_nofail(me->conn, key_index_sql); const char *ns_index_sql = "create index ns_index on store(ns)"; - utils::sqlite3_exec_nofail(me->conn, ns_index_sql); + sqlite::exec_nofail(me->conn, ns_index_sql); } else { - me->conn = utils::open_db(path.c_str()); + me->conn = sqlite::open_db(path.c_str()); } // Asynchronous for speed - utils::sqlite3_exec_nofail(me->conn, "PRAGMA synchronous = 0;"); - utils::sqlite3_exec_nofail(me->conn, "PRAGMA journal_mode=WAL"); + sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;"); + sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL"); // prepare insert statement const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)"; - me->insert = utils::prepare_query(me->conn, insert_sql); + me->insert = sqlite::prepare_query(me->conn, insert_sql); // prepare update statement const char *update_sql = "update store set value = ? where ns = ? and key = ?"; - me->update = utils::prepare_query(me->conn, update_sql); + me->update = sqlite::prepare_query(me->conn, update_sql); // prepare remove statement const char *remove_all_sql = "delete from store where ns = ?"; - me->remove_all = utils::prepare_query(me->conn, remove_all_sql); + me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql); // prepare remove statement const char *remove_sql = "delete from store where ns = ? and key = ?"; - me->remove = utils::prepare_query(me->conn, remove_sql); + me->remove = sqlite::prepare_query(me->conn, remove_sql); } SqliteStore::~SqliteStore() {} @@ -100,7 +101,7 @@ SqliteStore::~SqliteStore() {} SqliteStore::Data SqliteStore::retrieve_all() { SqliteStore::Data result; sqlite3_stmt_ptr query = - utils::prepare_query(me->conn, "select ns,key,value from store"); + sqlite::prepare_query(me->conn, "select ns,key,value from store"); int sqlite_result; do { @@ -125,7 +126,7 @@ std::pair SqliteStore::retrieve_one(std::string const &ns, std::string const &k) { auto query_str = fmt::format( "select ns,key,value from store where ns='{}' and key='{}'", ns, k); - sqlite3_stmt_ptr query = utils::prepare_query(me->conn, query_str.c_str()); + sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str()); std::pair result{false, ""}; diff --git a/src/lib/utils/Sqlite_utils.cpp b/src/lib/utils/Sqlite_utils.cpp index 4d31926a3..f8cc0a013 100644 --- a/src/lib/utils/Sqlite_utils.cpp +++ b/src/lib/utils/Sqlite_utils.cpp @@ -3,22 +3,27 @@ #include "bts_assert.h" #include namespace bts { -namespace utils { +namespace sqlite { -void sqlite3_exec_nofail(sqlite3* db, const char* sql) { +void exec_nofail(sqlite3* db, const char* sql) { char* error_message = nullptr; int result = sqlite3_exec(db, sql, nullptr, nullptr, &error_message); bts_assert_msg(result == SQLITE_OK, error_message); } -void sqlite3_exec_nofail(sqlite3_ptr & db, const char * sql) { - return sqlite3_exec_nofail(db.get(), sql); +void exec_nofail(sqlite3_ptr & db, const char * sql) { + return exec_nofail(db.get(), sql); } sqlite3_ptr open_db(const char* path) { sqlite3* temp; - bts_assert(sqlite3_open(path, &temp) == SQLITE_OK); - return sqlite3_ptr(temp, sqlite3_close); + if (sqlite3_open(path, &temp) == SQLITE_OK) { + return sqlite3_ptr(temp, sqlite3_close); + } + else { + LOG(Gen, Error, "Failed to open database {}", std::string(path)); + return null_db(); + } } sqlite3_stmt_ptr prepare_query(sqlite3 * db, const char * query) { diff --git a/src/lib/utils/Sqlite_utils.h b/src/lib/utils/Sqlite_utils.h index 4cb8e6ae3..1bc3cdbb4 100644 --- a/src/lib/utils/Sqlite_utils.h +++ b/src/lib/utils/Sqlite_utils.h @@ -1,12 +1,9 @@ #pragma once #ifndef WIN32 -#include #include -#include "lib/utils/bts_export.h" +#include #include "lib/utils/bts_assert.h" -typedef struct sqlite3 sqlite3; -typedef struct sqlite3_stmt sqlite3_stmt; typedef std::unique_ptr sqlite3_ptr; typedef std::unique_ptr sqlite3_stmt_ptr; @@ -18,13 +15,15 @@ typedef std::unique_ptr sqlite3_stmt_ptr; } \ } while(0) -namespace bts { -namespace utils { -void sqlite3_exec_nofail(sqlite3* db, const char* sql); -void sqlite3_exec_nofail(sqlite3_ptr& db, const char* sql); +namespace bts { +namespace sqlite { + +void exec_nofail(sqlite3* db, const char* sql); +void exec_nofail(sqlite3_ptr& db, const char* sql); sqlite3_ptr open_db(const char* path); +inline sqlite3_ptr null_db() { return sqlite3_ptr(nullptr, sqlite3_close); } sqlite3_stmt_ptr prepare_query(sqlite3* db, const char* query); sqlite3_stmt_ptr prepare_query(sqlite3_ptr& db, const char* query); }}//namespace From 4b3cdf9ca4b640e07fed52d9b4f37182dec18e01 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 14 Nov 2018 19:58:31 -0500 Subject: [PATCH 2/6] Add tests for SecurityIDRecorder += SpreadResolver --- src/lib/md/SecurityIDRecorder.cpp | 44 ++++++----- src/lib/md/SecurityIDRecorder.h | 18 +++-- src/lib/md/Shm/FixedSecurity.cpp | 14 +++- src/lib/md/Shm/FixedSecurity.h | 2 + src/lib/md/SyntheticSecurityService.cpp | 2 +- src/lib/md/relay/RelayRecorder.cpp | 2 +- src/lib/md/test/md2_tests.cpp | 78 +++++++++++++++++-- .../sparkd/md/platforms/celoxica/CXMain.cpp | 2 +- 8 files changed, 127 insertions(+), 35 deletions(-) diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 192ca68e2..d98fc948e 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -7,7 +7,7 @@ #include "lib/logging/logging.h" #include "lib/md/SecurityDB.h" #include -#include // for reservation +#include #include "lib/config/protos/MarketData.pb.h" #include "lib/config/protos/system.pb.h" @@ -22,6 +22,8 @@ using std::size_t; namespace bts { namespace md { +uint64_t exchange_id(const api::Security& security); + struct SecurityIDEntry { api::Security security; SecurityIDRecorder::Callback callback; @@ -58,17 +60,16 @@ struct SecurityIDRecorderState { security_key insert(const std::string& descriptor, SecurityType); }; -SecurityIDRecorder::SecurityIDRecorder(): +SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params): _self(new SecurityIDRecorderState()) { + if (params.db) { + _self->db = std::move(params.db); + } init(); -} - -SecurityIDRecorder::SecurityIDRecorder(utils::Dispatcher& dispatcher): - _self(new SecurityIDRecorderState()) -{ - init(); - _self->run(dispatcher); + if (params.dispatcher) { + _self->run(*params.dispatcher); + } } SecurityIDRecorder::~SecurityIDRecorder() { } @@ -81,13 +82,16 @@ void SecurityIDRecorder::init() { boost::filesystem::create_directories(db_dir); } - _self->db = open_db(db_path.c_str()); + if (!_self->db) { + _self->db = sqlite::open_db(db_path.c_str()); + bts_assert(_self->db); + } // Set a long busy timeout sqlite3_busy_timeout(_self->db.get(), 9999); - sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); - sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0"); - sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);"); + sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); + sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0"); + sqlite::exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);"); } void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { @@ -196,7 +200,7 @@ void SecurityIDRecorderState::work_db() { if (!new_entries.empty()) { - sqlite3_exec_nofail(db.get(), "BEGIN TRANSACTION"); + sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION"); for (size_t x = 0; x < new_entries.size(); ++x) { Security& security = new_entries[x]->security; @@ -205,7 +209,7 @@ void SecurityIDRecorderState::work_db() { security.set_id(key); } - sqlite3_exec_nofail(db.get(), "COMMIT TRANSACTION"); + sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } // record the securities @@ -231,10 +235,10 @@ void SecurityIDRecorderState::work_db() { void SecurityIDRecorder::bulk_insert(const std::vector& data, const char* prefix) { if (data.size() == 0) return; - sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); + sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); std::string descriptor; { - auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)"); + auto insert_stmt = sqlite::prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)"); for (const api::Security* sec : data) { descriptor = prefix + security_descriptor(*sec); auto type = securitytype_of(*sec); @@ -246,11 +250,11 @@ void SecurityIDRecorder::bulk_insert(const std::vector& da sqlite3_reset(insert_stmt.get()); } } - sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); } security_key SecurityIDRecorderState::search(const std::string& descriptor) { - auto select_stmt = prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); + auto select_stmt = sqlite::prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); sqlite3_bind_text(select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); int result = sqlite3_step(select_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { @@ -260,7 +264,7 @@ security_key SecurityIDRecorderState::search(const std::string& descriptor) { } security_key SecurityIDRecorderState::insert(const std::string& descriptor, SecurityType type) { - auto insert_stmt = prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); + auto insert_stmt = sqlite::prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); sqlite3_bind_text(insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); sqlite3_bind_int(insert_stmt.get(), 2, (int)type); int result = sqlite3_step(insert_stmt.get()); diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index f63731ff1..659f5a74a 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -3,11 +3,13 @@ #include "common.h" #include "MDProducerHandle.h" +#include "lib/utils/Sqlite_utils.h" #include #include #include #include -#include + +typedef struct sqlite3 sqlite3; namespace bts { namespace md { @@ -17,13 +19,19 @@ namespace md { * Provides a callback when ids are safely persisted. */ +struct SecurityIDRecorderParams { + SecurityIDRecorderParams() : dispatcher(), db(sqlite::null_db()) {} + SecurityIDRecorderParams(utils::Dispatcher& d): dispatcher(&d), db(sqlite::null_db()) {} + SecurityIDRecorderParams(sqlite3_ptr& db): dispatcher(), db(std::move(db)) {} + utils::Dispatcher* dispatcher; + sqlite3_ptr db; +}; struct SecurityIDRecorderState; + class SecurityIDRecorder { public: - SecurityIDRecorder(); - // Automatically call do_work on this dispatcher - SecurityIDRecorder(utils::Dispatcher&); + SecurityIDRecorder(SecurityIDRecorderParams); ~SecurityIDRecorder(); typedef std::function Callback; @@ -32,7 +40,7 @@ public: // Slow (copies security) - for use by services void assign_key(const api::Security& security, const Callback& key_assigned); - // Someone must call work periodically to do lookups and flush the reserved ids to disk. + // If a dispatcher is not provided you must call work periodically to do lookups and flush the reserved ids to disk. void do_work(); void bulk_insert(const std::vector&, const char* prefix); diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index cc8d304ea..a0405bdfe 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -250,6 +250,16 @@ size_t num_legs(const Security& sec) { } return 0; } +uint64_t exchange_id(const Security& security) { + switch(security.ExchangeData_case()) { + case Security::ExchangeDataCase::kCboe: return security.cboe().exchange_id(); + case Security::ExchangeDataCase::kGlbx: return security.glbx().exchange_id(); + case Security::ExchangeDataCase::kCfe: return security.cfe().exchange_id(); + case Security::ExchangeDataCase::kIce: bts_assert(false); + case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: bts_assert(false); + } + return -1; +} void convert_security(const Security& src, FixedSecurity& dest, bool set_security_key) { dest.key = set_security_key ? src.id() : invalid_security_key; @@ -272,7 +282,9 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit case Security::ExchangeDataCase::kCfe: dest.cboe_class_key = src.cfe().class_key(); break; - default: + case Security::ExchangeDataCase::kIce: + bts_assert_d(false); + case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: break; } dest.price_break = src.contract_spec().price_break() / src.contract_spec().price_multiplier(); diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index b8fd82d55..80d44b287 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -180,6 +180,8 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit SecurityType securitytype_of_contractcase(Security::ContractCase cc); SecurityType securitytype_of(const Security&); size_t num_legs(const Security&); +uint64_t exchange_id(const Security& security); + // Stack allocates a security of the given type template diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp index 8d6ca60aa..eeea9168b 100644 --- a/src/lib/md/SyntheticSecurityService.cpp +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -25,7 +25,7 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher MetadataPublishe struct SyntheticSecurityService::Pimpl { Pimpl(const MetadataStatefulPublisher& fn) : - recorder(utils::Dispatcher::background()), + recorder({utils::Dispatcher::background()}), synthetics(), on_security(), meta(), on_metadata(), metadata_publisher(fn) { diff --git a/src/lib/md/relay/RelayRecorder.cpp b/src/lib/md/relay/RelayRecorder.cpp index 55ca64edb..a6f453e73 100644 --- a/src/lib/md/relay/RelayRecorder.cpp +++ b/src/lib/md/relay/RelayRecorder.cpp @@ -130,7 +130,7 @@ void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) { CoconutReplayHeader header; size_t count = 0; #ifndef _WIN32 - md::SecurityIDRecorder idrec; + md::SecurityIDRecorder idrec {{}}; #endif if (readbuf(in, &header) && header.magic() == magic && header.version() == 3) { co2::MDReply message; diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 9a3301311..ba81b8413 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -13,6 +13,7 @@ #include "lib/metrics/metrics.h" #include "lib/config/protos/MarketData.pb.h" +#include "lib/utils/Sqlite_utils.h" #include #include @@ -27,7 +28,6 @@ #include #include - using namespace std; namespace bts { @@ -151,6 +151,7 @@ void assign_ids(vector::iterator begin, vector(); + auto hot_header = std::make_shared(); + + // Create a spread with 2 legs + auto f1 = mkfuture("FO1", invalid_security_key, 2018, 1, 17); + f1.mutable_glbx()->set_exchange_id(100); + REQUIRE(f1.id() == invalid_security_key); + auto f1handle = std::make_unique (f1, cold_header, hot_header); + + auto f2 = mkfuture("FO2", invalid_security_key, 2018, 1, 20); + f2.mutable_glbx()->set_exchange_id(200); + auto f2handle = std::make_unique (f2, cold_header, hot_header); + + auto s1 = mkspread("FOO", invalid_security_key, { f1, f2 }); + auto s1handle = std::make_unique (s1, cold_header, hot_header); + + // Add spread to SecurityIDRecorder and work it + auto db = sqlite::open_db(":memory:"); + REQUIRE(db); + SecurityIDRecorder idsvc{{db}}; + auto workwork = [&]() { idsvc.do_work(); idsvc.do_work(); }; + // auto workwork = [&]() { for(int i=5;--i;) { idsvc.do_work(); } }; + idsvc.ensure_key(s1handle->producer()); + workwork(); + + // Expect no key + REQUIRE(s1handle->consumer().key() == invalid_security_key); + + // Add leg spreads + idsvc.ensure_key(f1handle->producer()); + idsvc.ensure_key(f2handle->producer()); + workwork(); + + // Expect legs to have keys + REQUIRE(f1handle->consumer().key() != invalid_security_key); + REQUIRE(f2handle->consumer().key() != invalid_security_key); + + // Expect spread to have keys + REQUIRE(s1handle->consumer().key() != invalid_security_key); } #endif @@ -1002,7 +1064,11 @@ TEST_CASE("CO2Tests","[md]") { #ifndef WIN32 SECTION("test_security_id_recorder") { - co2_tests->test_security_id_recorder(); + co2_tests->test_security_id_recorder(":memory:"); + co2_tests->test_security_id_recorder("file::memory:?cache=shared"); + } + SECTION("test_security_id_recorder_spreads") { + co2_tests->test_security_id_recorder_spreads(); } #endif SECTION("test_price_size") { diff --git a/src/server/sparkd/md/platforms/celoxica/CXMain.cpp b/src/server/sparkd/md/platforms/celoxica/CXMain.cpp index bfb96e727..1c39dcd72 100644 --- a/src/server/sparkd/md/platforms/celoxica/CXMain.cpp +++ b/src/server/sparkd/md/platforms/celoxica/CXMain.cpp @@ -43,7 +43,7 @@ void run_cx_server(UserSymbol* psyms, size_t symcount) CXMain::CXMain() : _producer(co2::PLATFORM_CELOXICA), - _security_recorder(), + _security_recorder({}), _settlement_recorder(co2::PLATFORM_CELOXICA), _listeners() { From 25f749bb0c2b3561a20bebe85678825b66c730a6 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 15 Nov 2018 12:48:07 -0500 Subject: [PATCH 3/6] Simplify md security key generation for spreads. --- .vscode/launch.json | 3 +- src/lib/md/MDProducerHandle.cpp | 12 --- src/lib/md/MDProducerHandle.h | 3 +- src/lib/md/SecurityIDRecorder.cpp | 76 ++++++++++++++++-- src/lib/md/SecurityIDRecorder.h | 12 ++- src/lib/md/SecurityIndex.h | 54 +------------ src/lib/md/Shm/FixedSecurity.cpp | 7 +- src/lib/md/Shm/FixedSecurity.h | 5 ++ src/lib/md/SpreadResolver.h | 79 ------------------- src/lib/md/test/md2_tests.cpp | 57 ++----------- .../md/platforms/mdp3_onixs/MDP3Listener.h | 6 +- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 8 +- .../sparkd/md/platforms/mdp3_onixs/MDP3Main.h | 1 + 13 files changed, 108 insertions(+), 215 deletions(-) delete mode 100644 src/lib/md/SpreadResolver.h diff --git a/.vscode/launch.json b/.vscode/launch.json index 0eaae7c1f..163cad3eb 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -209,8 +209,7 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", - // "args": ["test","Co2RelayTests"], - "args": ["md", "run", "10"], + "args": ["test","Co2Tests"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", "environment": [], diff --git a/src/lib/md/MDProducerHandle.cpp b/src/lib/md/MDProducerHandle.cpp index 4e3cb476d..cd22abe83 100644 --- a/src/lib/md/MDProducerHandle.cpp +++ b/src/lib/md/MDProducerHandle.cpp @@ -61,18 +61,6 @@ void MDProducerHandle::set_key(security_key key) { } } -void MDProducerHandle::set_legs(const std::vector& legs) { - bts_assert(cdata().security.type == api::SECURITYTYPE_SPREAD); - FixedSpreadSecurityL* fss = (FixedSpreadSecurityL*)&cdata().security; - bts_assert(fss->leg_count() == legs.size()); - for (size_t x = 0; x < legs.size(); ++x) { - security_key key = legs[x]; - fss->legs[x].key = key; - // attempt to validate that this FixedSpreadLeg is valid - bts_assert_d(fss->legs[x].leg_quantity > 0); - } -} - void MDProducerHandle::update_bbo(TimeStamp time, const PriceSize bid, const PriceSize ask, uint16_t seq) { if (!has_depth()) { auto oldtime = hdata().time(); diff --git a/src/lib/md/MDProducerHandle.h b/src/lib/md/MDProducerHandle.h index 5f5e5f52b..2e21384da 100644 --- a/src/lib/md/MDProducerHandle.h +++ b/src/lib/md/MDProducerHandle.h @@ -23,7 +23,7 @@ public: MDProducerHandle& operator=(MDProducerHandle const& other); const FixedSecurity& security() const { return cdata().security; } - FixedSecurity* mutable_security() { return key() == invalid_security_key ? &cdata().security : nullptr; } + FixedSecurity* mootable_security() { return key() == invalid_security_key ? &cdata().security : nullptr; } bool has_depth() const { return cheader().has_depth; } void set_key(security_key); @@ -32,7 +32,6 @@ public: bts_assert_d(key == cdata().security.key); return key; } - void set_legs(const std::vector& legs); void update_bbo(TimeStamp time, const PriceSize bid_price, const PriceSize ask_price, uint16_t seq=0); void update_direct(TimeStamp time, const std::array& bids, const std::array& asks, uint32_t seq=0) { diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index d98fc948e..9302fa69d 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -30,38 +30,52 @@ struct SecurityIDEntry { }; struct SecurityIDRecorderState { - SecurityIDRecorderState() : + SecurityIDRecorderState(SecurityIDRecorderParams& p) : work(), pending_handle(), + thread_id(), db(nullptr, sqlite3_close), timer(), - id((size_t)this) + lookup_key(p.lookup_key), + unresolved_legs() {} + tbb::concurrent_queue> work; tbb::concurrent_queue pending_handle; + std::thread::id thread_id; sqlite3_ptr db; unique_ptr timer; - size_t id; + + KeyLookupOption lookup_key; + std::map> unresolved_legs; + + // Oh no, I got methods in my state void run(utils::Dispatcher& dispatcher) { timer = std::make_unique( - dispatcher, 500, + dispatcher, 200, [=]() { do_work(); }); timer->start(); } void do_work() { + thread_id = std::this_thread::get_id(); + work_spreads(); work_handles(); work_db(); } + void work_spreads(); void work_handles(); void work_db(); + + void check_access() { bts_assert(thread_id == std::this_thread::get_id()); } void assign_key(const api::Security& security, const SecurityIDRecorder::Callback& key_assigned); + void add_unresolved_spread(MDProducerHandle& handle); security_key search(const std::string& descriptor); security_key insert(const std::string& descriptor, SecurityType); }; SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params): - _self(new SecurityIDRecorderState()) + _self(new SecurityIDRecorderState(params)) { if (params.db) { _self->db = std::move(params.db); @@ -99,6 +113,15 @@ void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { } void SecurityIDRecorder::assign_key(const api::Security& security, const SecurityIDRecorder::Callback& callback) { + if (security.Contract_case() == Security::ContractCase::kSpread) { + for(const auto& leg : security.spread().legs()) { + if (leg.security().id() == invalid_security_key) { + LOG(md,Error, "Security {} has legs which will never be resolved.", security.has_glbx() ? security.glbx().security_desc() : security.symbol()); + // No point in trying to add this to the queue + return; + } + } + } _self->assign_key(security, callback); } @@ -115,6 +138,43 @@ void SecurityIDRecorder::do_work() { _self->do_work(); } +void SecurityIDRecorderState::add_unresolved_spread(MDProducerHandle& handle) { + bts_assert_d(lookup_key); + FixedSpreadSecurityXL* pspread = static_cast(handle.mootable_security()); + if (pspread != nullptr) { + for (size_t x=0; x < pspread->leg_count(); ++x) { + auto& leg = pspread->legs[x]; + bts_assert_d(leg.exchange_key.value() != 0); + if (leg.key == invalid_security_key) { + // Because leg is a reference to shared memory we know it will not move + for (const auto* existing : unresolved_legs[leg.exchange_key]) { + if (existing == &leg) goto next; + } + unresolved_legs[leg.exchange_key].push_back(&leg); + } + next:; + } + } +} + +void SecurityIDRecorderState::work_spreads() { + check_access(); + if (!lookup_key) return; + KeyLookup& lookup = *lookup_key; + int count = 0; + for (auto iter = unresolved_legs.begin(); count < 50 && iter != unresolved_legs.end(); iter++) { + const auto& pair = *iter; + ++count; + SecurityKey key = lookup(pair.first); + if (key.value() != invalid_security_key) { + for (FixedSpreadLeg* leg : pair.second) { + leg->key = key.value(); + } + unresolved_legs.erase(iter); + } + } +} + bool has_all_leg_keys(const MDProducerHandle& handle) { if (handle.security().type == SecurityType::SECURITYTYPE_SPREAD) { const FixedSpreadSecurityXL* pspread = static_cast(&handle.security()); @@ -124,11 +184,12 @@ bool has_all_leg_keys(const MDProducerHandle& handle) { } } } - // SyntheticFixedSecurity is guarunteed to have all of its legs + // SyntheticFixedSecurity is guaranteed to have all of its legs return true; } void SecurityIDRecorderState::work_handles() { + check_access(); Security security; MDProducerHandle handle; vector retry; @@ -137,6 +198,8 @@ void SecurityIDRecorderState::work_handles() { security.Clear(); // if a spread does not have all of its legs we need to wait if (!has_all_leg_keys(handle)) { + // add + add_unresolved_spread(handle); retry.push_back(handle); } else { @@ -154,6 +217,7 @@ void SecurityIDRecorderState::work_handles() { } void SecurityIDRecorderState::work_db() { + check_access(); // Drain the work queue // Query the database for each security. If it has an ID set it // If it does not have an ID add it to a list diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index 659f5a74a..19895d84c 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -8,6 +8,7 @@ #include #include #include +#include typedef struct sqlite3 sqlite3; @@ -19,12 +20,17 @@ namespace md { * Provides a callback when ids are safely persisted. */ +typedef const std::function KeyLookup; +typedef std::optional KeyLookupOption; struct SecurityIDRecorderParams { - SecurityIDRecorderParams() : dispatcher(), db(sqlite::null_db()) {} - SecurityIDRecorderParams(utils::Dispatcher& d): dispatcher(&d), db(sqlite::null_db()) {} - SecurityIDRecorderParams(sqlite3_ptr& db): dispatcher(), db(std::move(db)) {} + SecurityIDRecorderParams() : dispatcher(), db(sqlite::null_db()), lookup_key() {} + SecurityIDRecorderParams(utils::Dispatcher& d): dispatcher(&d), db(sqlite::null_db()), lookup_key() {} + SecurityIDRecorderParams(utils::Dispatcher& d, KeyLookup& get_key): dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {} + SecurityIDRecorderParams(sqlite3_ptr& db): dispatcher(), db(std::move(db)), lookup_key() {} + SecurityIDRecorderParams(sqlite3_ptr& db, KeyLookup& get_key): dispatcher(), db(std::move(db)), lookup_key(get_key) {} utils::Dispatcher* dispatcher; sqlite3_ptr db; + KeyLookupOption lookup_key; }; struct SecurityIDRecorderState; diff --git a/src/lib/md/SecurityIndex.h b/src/lib/md/SecurityIndex.h index 6ea7eea4b..1caea89c2 100644 --- a/src/lib/md/SecurityIndex.h +++ b/src/lib/md/SecurityIndex.h @@ -4,7 +4,6 @@ #include "lib/logging/logging.h" #include "MDProducerHandle.h" #include "common.h" -#include "SpreadResolver.h" #include #include #include @@ -26,8 +25,7 @@ class SimpleSecurityIndex { public: SimpleSecurityIndex(): - _map(), - _spread_resolver() + _map() {} ~SimpleSecurityIndex() {} @@ -79,35 +77,11 @@ public: return add_security(key, std::move(data)); } - // - // Spread resolve logic - - void add_spread_legs(ExchangeKey key, const std::vector& legs) { - _spread_resolver.add(key, legs); - } - - // Resolves spreads and returns the number spreads resolved - size_t resolve_spreads() { - return _spread_resolver.resolve( - [&](ExchangeKey key) { return get_key(key); }, - [&](ExchangeKey& key, std::vector& legs) { on_spread_resolved(key, legs); }); - } - -protected: - void on_spread_resolved(ExchangeKey& key, std::vector& legs) { - auto* entry = get_security(key); - bts_assert_d(entry); - if (entry) { - entry->handle.set_legs(legs); - } - } - private: BTS_NO_COPY(SimpleSecurityIndex); typedef std::unordered_map> MapType; MapType _map; mutable tbb::queuing_rw_mutex _mutex; - SpreadResolver _spread_resolver; }; @@ -120,8 +94,7 @@ class ConcurrentSecurityIndex { public: ConcurrentSecurityIndex() : - _map(), - _spread_resolver() + _map() {} ~ConcurrentSecurityIndex() {} @@ -171,33 +144,10 @@ public: return add_security(key, std::move(data)); } - // - // Spread resolve logic - - void add_spread_legs(ExchangeKey key, const std::vector& legs) { - _spread_resolver.add(key, legs); - } - - // Resolves spreads and returns the number spreads resolved - size_t resolve_spreads() { - return _spread_resolver.resolve( - [&](ExchangeKey key) { return get_key(key); }, - [&](ExchangeKey& key, std::vector& legs) { on_spread_resolved(key, legs); }); - } - -protected: - void on_spread_resolved(ExchangeKey& key, std::vector& legs) { - auto* entry = get_security(key); - bts_assert_d(entry); - if (entry) { - entry->handle.set_legs(legs); - } - } private: BTS_NO_COPY(ConcurrentSecurityIndex); typedef tbb::concurrent_hash_map MapType; MapType _map; - SpreadResolver _spread_resolver; }; }}//namespace diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index a0405bdfe..55b020b89 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -250,13 +250,17 @@ size_t num_legs(const Security& sec) { } return 0; } + uint64_t exchange_id(const Security& security) { switch(security.ExchangeData_case()) { case Security::ExchangeDataCase::kCboe: return security.cboe().exchange_id(); case Security::ExchangeDataCase::kGlbx: return security.glbx().exchange_id(); case Security::ExchangeDataCase::kCfe: return security.cfe().exchange_id(); case Security::ExchangeDataCase::kIce: bts_assert(false); - case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: bts_assert(false); + // Debatable whether this should be an error or not. + // If is possible that we might create a spread with no exchange in the future + // and if we did this would be a runtime error. + case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: return 0; } return -1; } @@ -319,6 +323,7 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit for (int x = 0; x < src.spread().legs_size(); ++x) { const auto& src_leg = src.spread().legs(x); spread.legs[x].key = set_security_key ? src_leg.security().id() : invalid_security_key; + spread.legs[x].exchange_key = ExchangeKey(exchange_id(src_leg.security())); spread.legs[x].leg_side = src_leg.side(); spread.legs[x].leg_quantity = src_leg.quantity(); spread.legs[x].leg_price = src_leg.price_multiplier(); diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index 80d44b287..0ad9e4ffa 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -6,6 +6,7 @@ #include "lib/utils/bts_array_size.h" #include "lib/utils/bts_assert.h" #include "lib/utils/bts_no_copy.h" +#include "lib/utils/ValueWrapper.h" #include #include @@ -13,6 +14,9 @@ namespace bts { namespace md { +using SecurityKey = utils::ValueWrapper; +using ExchangeKey = utils::ValueWrapper; + GCC_DIAG_OFF("-Weffc++") // No Virtual Destructors by design #define STRING_PROPERTY(NAME) const char* NAME() const { return _##NAME; } \ @@ -95,6 +99,7 @@ struct BTS_EXPORT FixedSpreadLeg { FixedSpreadLeg(); security_key key; + ExchangeKey exchange_key; Side leg_side; double leg_quantity; double leg_price; diff --git a/src/lib/md/SpreadResolver.h b/src/lib/md/SpreadResolver.h deleted file mode 100644 index 8885bb189..000000000 --- a/src/lib/md/SpreadResolver.h +++ /dev/null @@ -1,79 +0,0 @@ -#pragma once - -#include "lib/utils/bts_no_copy.h" -#include -#include - -namespace bts { -namespace md { - - -/* - * SpreadResolved accepts exchange spread definitions and resolves the legs - */ - -template -class SpreadResolver { - struct ResolverEntry { - ResolverEntry(ExchangeKey key) : exchange_key(key), legs() {} - ResolverEntry() : exchange_key(), legs() {} - ExchangeKey exchange_key; - std::vector legs; - }; -public: - SpreadResolver() : _queue() {} - template - void add(ExchangeKey key, LegContainer legs) { - ResolverEntry entry(key); - entry.legs.insert(end(entry.legs), begin(legs), end(legs)); - _queue.push(entry); - } - - // callback should accept const ExchangeKey& key, vector& legs) - // When the callback is called the legs will be completely resolved - template - size_t resolve(MapKey keymap, CallBack on_resolved) { - // Drain the queue and call keymap to resolve each leg - size_t result = 0; - std::vector unresolved; - std::vector legs; - while(!_queue.empty()) { - legs.clear(); - ResolverEntry entry; - if(_queue.try_pop(entry)) { - // Attempt to look up each spread leg - for(ExchangeKey key : entry.legs) { - security_key result = keymap(key); - if (result == invalid_security_key) { - unresolved.push_back(entry); - break; - } - else { - legs.push_back(result); - } - } - if (legs.size() == entry.legs.size()) { - ++result; - on_resolved(entry.exchange_key, legs); - } - } - } - - // If any legs remain unresolved push the entry back in the queue for later - for(auto& entry : unresolved) { - _queue.push(entry); - } - - return result; - } - - size_t count() const { return _queue.unsafe_size(); } -private: - BTS_NO_COPY(SpreadResolver); - tbb::concurrent_queue _queue; -}; - - - -}}//namespace - diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index ba81b8413..9da770f46 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -231,9 +231,15 @@ public: // Add spread to SecurityIDRecorder and work it auto db = sqlite::open_db(":memory:"); REQUIRE(db); - SecurityIDRecorder idsvc{{db}}; + + SecurityIDRecorder idsvc { {db, [&](ExchangeKey k) -> SecurityKey { + for (const auto fut : { f1handle->producer(), f2handle->producer() }) { + if (k.value() == fut.security().numeric_id) return SecurityKey(fut.key()); + } + return SecurityKey(invalid_security_key); + } } }; + auto workwork = [&]() { idsvc.do_work(); idsvc.do_work(); }; - // auto workwork = [&]() { for(int i=5;--i;) { idsvc.do_work(); } }; idsvc.ensure_key(s1handle->producer()); workwork(); @@ -580,50 +586,6 @@ public: REQUIRE(bbo_count > 0); } - void test_spread_resolver() { - SpreadResolver resolver; - vector legs; - legs.push_back("1"); - legs.push_back("2"); - legs.push_back("3"); - legs.push_back("4"); - resolver.add("1234", legs); - legs.clear(); - legs.push_back("4"); - legs.push_back("5"); - resolver.add("45", legs); - auto key_map = [](string& key) -> security_key { - if (key == "1") return 1; - if (key == "2") return 2; - if (key == "3") return 3; - if (key == "4") return 4; - if (key == "5") return invalid_security_key; - REQUIRE(false); - return 0; - }; - auto key_map2 = [](string& key) -> security_key { - if (key == "4") return 4; - if (key == "5") return 5; - REQUIRE(false); - return 0; - }; - - // 2 unresolved spreads - REQUIRE(resolver.count() == 2); - resolver.resolve(key_map, [&](const string& key, const vector& legs) { - REQUIRE(key == "1234"); - REQUIRE(legs.size() == 4); - }); - // 1 spread remains - REQUIRE(resolver.count() == 1); - resolver.resolve(key_map2, [&](const string& key, const vector& legs) { - REQUIRE(key == "45"); - REQUIRE(legs.size() == 2); - }); - REQUIRE(resolver.count() == 0); - } - - void test_no_double_security_callback() { MDProducer producer(TEST_PLATFORM); vector producers = create_test_securities(producer); @@ -1092,9 +1054,6 @@ TEST_CASE("CO2Tests","[md]") { SECTION("test_consumer_defer_handle2") { co2_tests->test_consumer_defer_handle2(); } - SECTION("test_spread_resolver") { - co2_tests->test_spread_resolver(); - } SECTION("test_no_double_security_callback") { co2_tests->test_no_double_security_callback(); } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index 1a54c9bd0..c29a1d01c 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -312,8 +312,6 @@ void MDP3Listener::onSpread(onixs::Handler&, const onixs::Security&, const Instr FixedSpreadSecurityL fsec; set_security_common(sec, fsec); - std::vector leg_exchange_ids; - leg_exchange_ids.reserve(sec.legs().size()); onixs::Decimal min_tick = tick_rule(sec); @@ -341,10 +339,10 @@ void MDP3Listener::onSpread(onixs::Handler&, const onixs::Security&, const Instr } fsec.legs[x].leg_quantity = leg.legRatioQty(); fsec.legs[x].leg_side = leg.legSide() == onixs::LegSide::BuySide ? api::SIDE_BUY : api::SIDE_SELL; - leg_exchange_ids.push_back(leg.legSecurityId()); + fsec.legs[x].exchange_key = ExchangeKey(leg.legSecurityId()); + fsec.legs[x].key = invalid_security_key; } add_security(fsec); - _index.add_spread_legs(fsec.numeric_id, leg_exchange_ids); } template diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 679f92f3c..425a3e8d3 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -26,7 +26,9 @@ namespace bts { namespace md { MDP3Main::MDP3Main() : - _producer(PLATFORM_MDP3ONIXS, { utils::Dispatcher::current() }), + _index(), + _recorder ({ utils::Dispatcher::current(), [=](ExchangeKey k) { return SecurityKey(_index.get_key(k.value())); } }), + _producer(PLATFORM_MDP3ONIXS, _recorder ), // _settlement_recorder(PLATFORM_MDP3ONIXS), _engines(), _handlers(), @@ -152,10 +154,6 @@ void MDP3Main::run(std::vector& symbols) { // Timers auto& disp = utils::Dispatcher::current(); - utils::DispatcherTimer spread_resolve_timer(disp, 1000, [&]() { - _index.resolve_spreads(); - }); - spread_resolve_timer.start(); // utils::DispatcherTimer settlement_timer(disp, 100, [&]() { // _settlement_recorder.do_work(); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index a14825fdc..4eb1fbf62 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -38,6 +38,7 @@ private: BTS_NO_COPY(MDP3Main); CMESecurityIndex _index; + SecurityIDRecorder _recorder; MDProducer _producer; // co2::SettlementRecorder _settlement_recorder; std::map> _engines; From 38668f8de14666acdbb4a715aff44570c764fc78 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 16 Nov 2018 13:09:05 -0500 Subject: [PATCH 4/6] Code review feedback --- src/lib/md/MDProducerHandle.h | 2 +- src/lib/md/SecurityIDRecorder.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/md/MDProducerHandle.h b/src/lib/md/MDProducerHandle.h index a846d6fdc..da5afcf2d 100644 --- a/src/lib/md/MDProducerHandle.h +++ b/src/lib/md/MDProducerHandle.h @@ -23,7 +23,7 @@ public: MDProducerHandle& operator=(MDProducerHandle const& other); const FixedSecurity& security() const { return cdata().security; } - FixedSecurity* mootable_security() { return key() == invalid_security_key ? &cdata().security : nullptr; } + FixedSecurity* mutable_security() { return key() == invalid_security_key ? &cdata().security : nullptr; } bool has_depth() const { return cheader().has_depth; } void set_key(security_key); diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 9302fa69d..bd1b82466 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -140,7 +140,7 @@ void SecurityIDRecorder::do_work() { void SecurityIDRecorderState::add_unresolved_spread(MDProducerHandle& handle) { bts_assert_d(lookup_key); - FixedSpreadSecurityXL* pspread = static_cast(handle.mootable_security()); + FixedSpreadSecurityXL* pspread = static_cast(handle.mutable_security()); if (pspread != nullptr) { for (size_t x=0; x < pspread->leg_count(); ++x) { auto& leg = pspread->legs[x]; From 2551a0f11e8a475ab864c9a6b17b66c202546a84 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 20 Nov 2018 16:19:11 -0500 Subject: [PATCH 5/6] Code review feedback --- src/lib/md/SecurityIDRecorder.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index bd1b82466..f8cc32613 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -162,7 +162,7 @@ void SecurityIDRecorderState::work_spreads() { if (!lookup_key) return; KeyLookup& lookup = *lookup_key; int count = 0; - for (auto iter = unresolved_legs.begin(); count < 50 && iter != unresolved_legs.end(); iter++) { + for (auto iter = unresolved_legs.begin(); count < 500 && iter != unresolved_legs.end();) { const auto& pair = *iter; ++count; SecurityKey key = lookup(pair.first); @@ -170,7 +170,10 @@ void SecurityIDRecorderState::work_spreads() { for (FixedSpreadLeg* leg : pair.second) { leg->key = key.value(); } - unresolved_legs.erase(iter); + iter = unresolved_legs.erase(iter); + } + else { + ++iter; } } } @@ -252,7 +255,7 @@ void SecurityIDRecorderState::work_db() { new_descriptors.push_back(descriptor); ++count; // Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat - if (count > 500) { + if (count > 200) { break; } } From a3b91a436309a259ef5c17ba0dd382a62885e04f Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 20 Nov 2018 16:55:55 -0500 Subject: [PATCH 6/6] Should have included exchange key in Shmdump --- src/lib/md/Diag/ShmDumper.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/md/Diag/ShmDumper.cpp b/src/lib/md/Diag/ShmDumper.cpp index f41da63aa..66eccfe24 100644 --- a/src/lib/md/Diag/ShmDumper.cpp +++ b/src/lib/md/Diag/ShmDumper.cpp @@ -116,6 +116,7 @@ string dump_security(const FixedSecurity& security) { auto& leg = psec->legs[x]; if (x > 0) res << ","; res << "{ 'key': " << leg.key + << ", 'exkey': " << leg.exchange_key << ", 'quantity': " << leg.leg_quantity << ", 'leg_price': " << leg.leg_price << ", 'delta': " << leg.leg_delta