From fcc29ec2d1e39ea8ad408e2da4df99a5957781de Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 5 Jan 2018 12:35:17 -0500 Subject: [PATCH] Load security db in mdp3 startup --- src/lib/md/SecurityDB.cpp | 35 ++++++++++++++----- src/lib/md/SecurityDB.h | 10 ++++-- src/lib/md/Shm/FixedSecurity.cpp | 6 ++++ src/lib/md/Shm/FixedSecurity.h | 1 + .../md/platforms/mdp3_onixs/MDP3Listener.h | 4 ++- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 11 +++++- 6 files changed, 53 insertions(+), 14 deletions(-) diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index 8a659d931..a416132da 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -44,16 +44,16 @@ sqlite3_ptr open_db() { sqlite3_exec_nofail(db, "PRAGMA syncronous=0"); sqlite3_exec_nofail(db, - "CREATE TABLE IF NOT EXIST securities (key INTEGER NOT NULL, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL"); + "CREATE TABLE IF NOT EXIST securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL"); return dbptr; } -std::unique_ptr load_security_db( +size_t db_load_securities( const std::set& symbols, const std::function& callback) { - auto security_db = std::make_unique(); + size_t count = 0; auto dbptr = open_db(); // Load all existing securities and call the callback @@ -76,16 +76,32 @@ std::unique_ptr load_security_db( auto len = sqlite3_column_bytes(select, 4); if (security.ParseFromArray(data, len)) { bts_assert(security_key == security.id()); - security_db->known_keys.insert(SecurityKey(security.id())); + count++; callback(security); } } result = sqlite3_step(select); } - return security_db; + return count; } +// std::unique_ptr fetch_known_securities() { +// auto security_db = std::make_unique(); +// auto dbptr = open_db(); +// auto db = dbptr.get(); +// auto select_stmt = prepare_query(db, "select key from securities where expiration > ?"); +// auto select = select_stmt.get(); +// sqlite3_bind_int(select, 1, datetime::today().to_short()); +// auto result = sqlite3_step(select); +// while (result == SQLITE_ROW) { +// uint32_t security_key = sqlite3_column_int(select, 0); +// security_db->known_keys.insert(SecurityKey(security_key)); +// result = sqlite3_step(select); +// } +// return security_db; +// } + bool store_if_new(SecurityDB& db, const api::Security& security) { auto key = SecurityKey(security.id()); if (db.known_keys.find(key) == db.known_keys.end()) { @@ -112,10 +128,11 @@ bool store_if_new(SecurityDB& db, const api::Security& security) { namespace bts { namespace md { struct SecurityDB {}; -std::unique_ptr load_security_db( - const std::set&, - const std::function&) { - return std::make_unique(); + +size_t db_load_securities( + const std::set& symbols, + const std::function& callback) { + return 0; } bool store_if_new(SecurityDB&, const api::Security&) { return false; diff --git a/src/lib/md/SecurityDB.h b/src/lib/md/SecurityDB.h index 19099bfd5..fc35ba90a 100644 --- a/src/lib/md/SecurityDB.h +++ b/src/lib/md/SecurityDB.h @@ -8,7 +8,6 @@ #include #include -#include namespace bts { namespace api {class Security;} @@ -16,11 +15,16 @@ namespace md { struct SecurityDB; -std::unique_ptr load_security_db( +// Loads securities matching the given filter for producers to pre-populate security lists +// note: possible to use this to optimize initial memory layout? sort by expiration? +size_t db_load_securities( const std::set& symbols, const std::function& callback); -bool store_if_new(SecurityDB& db, const api::Security& security); +// Fetch an index of security keys +std::unique_ptr fetch_known_securities(); + +bool store_if_new(SecurityDB* db, const api::Security& security); datetime::Date expiration(const api::Security& security); diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index 037f09ef1..b37b282a6 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -218,6 +218,12 @@ SecurityType securitytype_of_contractcase(Security::ContractCase cc) { SecurityType securitytype_of(const Security& s) { return securitytype_of_contractcase(s.Contract_case()); } +size_t num_legs(const Security& sec) { + if (sec.Contract_case() == api::Security::ContractCase::kSpread) { + return sec.spread().legs_size(); + } + return 0; +} void convert_security(const Security& src, FixedSecurity& dest, bool set_security_key) { dest.key = set_security_key ? src.id() : invalid_security_key; diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index be8fb7cc2..9eafbcbd1 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -138,6 +138,7 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit SecurityType securitytype_of_contractcase(Security::ContractCase cc); SecurityType securitytype_of(const Security&); +size_t num_legs(const Security&); // Stack allocates a security of the given type template diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index 221a89b74..ab8bf7c5d 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -91,8 +91,10 @@ public: void add_filter_symbol(const std::string& symbol); void heartbeat(); -private: + // Add a new security to the index void add_security(const FixedSecurity& security); + +private: void dequeue_securities(); enum SecurityFilterAction { ADD, SKIP, DELETE }; template diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index f3548c8a7..53e102c1f 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -3,6 +3,7 @@ #include "MDP3Utils.h" #include "lib/config/config.h" #include "lib/md/coconut2.h" +#include "lib/md/SecurityDB.h" #include "lib/os/os.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/proto_helper.h" @@ -16,7 +17,7 @@ using namespace std; using namespace bts::co2; -// using namespace onixs; +using std::size_t; namespace bts { namespace md { @@ -322,6 +323,14 @@ unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, con listener->add_filter_symbol(symbol.symbol()); } + // Load the list of known securities and add them to the index for fast startup + db_load_securities(channel.symbols, [&](const api::Security& security) { + with_security(securitytype_of(security), num_legs(security), [&](FixedSecurity& fs) { + convert_security(security, fs, true); + listener->add_security(fs); + }); + }); + handler.registerListener(*listener); handler.registerMarketDataListener(*listener); handler.registerSecurityListener(*listener);