From fcc29ec2d1e39ea8ad408e2da4df99a5957781de Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 5 Jan 2018 12:35:17 -0500 Subject: [PATCH 1/7] Load security db in mdp3 startup --- src/lib/md/SecurityDB.cpp | 35 ++++++++++++++----- src/lib/md/SecurityDB.h | 10 ++++-- src/lib/md/Shm/FixedSecurity.cpp | 6 ++++ src/lib/md/Shm/FixedSecurity.h | 1 + .../md/platforms/mdp3_onixs/MDP3Listener.h | 4 ++- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 11 +++++- 6 files changed, 53 insertions(+), 14 deletions(-) diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index 8a659d931..a416132da 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -44,16 +44,16 @@ sqlite3_ptr open_db() { sqlite3_exec_nofail(db, "PRAGMA syncronous=0"); sqlite3_exec_nofail(db, - "CREATE TABLE IF NOT EXIST securities (key INTEGER NOT NULL, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL"); + "CREATE TABLE IF NOT EXIST securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL"); return dbptr; } -std::unique_ptr load_security_db( +size_t db_load_securities( const std::set& symbols, const std::function& callback) { - auto security_db = std::make_unique(); + size_t count = 0; auto dbptr = open_db(); // Load all existing securities and call the callback @@ -76,16 +76,32 @@ std::unique_ptr load_security_db( auto len = sqlite3_column_bytes(select, 4); if (security.ParseFromArray(data, len)) { bts_assert(security_key == security.id()); - security_db->known_keys.insert(SecurityKey(security.id())); + count++; callback(security); } } result = sqlite3_step(select); } - return security_db; + return count; } +// std::unique_ptr fetch_known_securities() { +// auto security_db = std::make_unique(); +// auto dbptr = open_db(); +// auto db = dbptr.get(); +// auto select_stmt = prepare_query(db, "select key from securities where expiration > ?"); +// auto select = select_stmt.get(); +// sqlite3_bind_int(select, 1, datetime::today().to_short()); +// auto result = sqlite3_step(select); +// while (result == SQLITE_ROW) { +// uint32_t security_key = sqlite3_column_int(select, 0); +// security_db->known_keys.insert(SecurityKey(security_key)); +// result = sqlite3_step(select); +// } +// return security_db; +// } + bool store_if_new(SecurityDB& db, const api::Security& security) { auto key = SecurityKey(security.id()); if (db.known_keys.find(key) == db.known_keys.end()) { @@ -112,10 +128,11 @@ bool store_if_new(SecurityDB& db, const api::Security& security) { namespace bts { namespace md { struct SecurityDB {}; -std::unique_ptr load_security_db( - const std::set&, - const std::function&) { - return std::make_unique(); + +size_t db_load_securities( + const std::set& symbols, + const std::function& callback) { + return 0; } bool store_if_new(SecurityDB&, const api::Security&) { return false; diff --git a/src/lib/md/SecurityDB.h b/src/lib/md/SecurityDB.h index 19099bfd5..fc35ba90a 100644 --- a/src/lib/md/SecurityDB.h +++ b/src/lib/md/SecurityDB.h @@ -8,7 +8,6 @@ #include #include -#include namespace bts { namespace api {class Security;} @@ -16,11 +15,16 @@ namespace md { struct SecurityDB; -std::unique_ptr load_security_db( +// Loads securities matching the given filter for producers to pre-populate security lists +// note: possible to use this to optimize initial memory layout? sort by expiration? +size_t db_load_securities( const std::set& symbols, const std::function& callback); -bool store_if_new(SecurityDB& db, const api::Security& security); +// Fetch an index of security keys +std::unique_ptr fetch_known_securities(); + +bool store_if_new(SecurityDB* db, const api::Security& security); datetime::Date expiration(const api::Security& security); diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index 037f09ef1..b37b282a6 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -218,6 +218,12 @@ SecurityType securitytype_of_contractcase(Security::ContractCase cc) { SecurityType securitytype_of(const Security& s) { return securitytype_of_contractcase(s.Contract_case()); } +size_t num_legs(const Security& sec) { + if (sec.Contract_case() == api::Security::ContractCase::kSpread) { + return sec.spread().legs_size(); + } + return 0; +} void convert_security(const Security& src, FixedSecurity& dest, bool set_security_key) { dest.key = set_security_key ? src.id() : invalid_security_key; diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index be8fb7cc2..9eafbcbd1 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -138,6 +138,7 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit SecurityType securitytype_of_contractcase(Security::ContractCase cc); SecurityType securitytype_of(const Security&); +size_t num_legs(const Security&); // Stack allocates a security of the given type template diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index 221a89b74..ab8bf7c5d 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -91,8 +91,10 @@ public: void add_filter_symbol(const std::string& symbol); void heartbeat(); -private: + // Add a new security to the index void add_security(const FixedSecurity& security); + +private: void dequeue_securities(); enum SecurityFilterAction { ADD, SKIP, DELETE }; template diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index f3548c8a7..53e102c1f 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -3,6 +3,7 @@ #include "MDP3Utils.h" #include "lib/config/config.h" #include "lib/md/coconut2.h" +#include "lib/md/SecurityDB.h" #include "lib/os/os.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/proto_helper.h" @@ -16,7 +17,7 @@ using namespace std; using namespace bts::co2; -// using namespace onixs; +using std::size_t; namespace bts { namespace md { @@ -322,6 +323,14 @@ unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, con listener->add_filter_symbol(symbol.symbol()); } + // Load the list of known securities and add them to the index for fast startup + db_load_securities(channel.symbols, [&](const api::Security& security) { + with_security(securitytype_of(security), num_legs(security), [&](FixedSecurity& fs) { + convert_security(security, fs, true); + listener->add_security(fs); + }); + }); + handler.registerListener(*listener); handler.registerMarketDataListener(*listener); handler.registerSecurityListener(*listener); From 4a84764dde69485c9a9def6fb259193bcc2bbaae Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 5 Jan 2018 21:36:13 -0500 Subject: [PATCH 2/7] Simplify SecurityIDRecorder - have it record securities to DB when key is assigned. --- src/lib/md/SecurityDB.cpp | 56 +++------ src/lib/md/SecurityDB.h | 13 +- src/lib/md/SecurityIDRecorder.cpp | 112 ++++++------------ src/lib/md/SecurityIDRecorder.h | 11 +- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 39 +----- .../md/platforms/mdp3_onixs/MDP3Listener.h | 5 +- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 4 +- 7 files changed, 64 insertions(+), 176 deletions(-) diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index a416132da..41a405c77 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -4,6 +4,7 @@ #include "lib/utils/Sqlite_utils.h" #include "lib/config/config.h" #include "lib/config/protos/system.pb.h" +#include "lib/md/Shm/FixedSecurity.h" #include #include @@ -14,14 +15,7 @@ using namespace bts::utils; namespace bts { namespace md { - -// schema: -// id, security (binary), expiration time - -struct SecurityDB { - // Store the symbols for debugging purposes - boost::container::flat_set known_keys; -}; +namespace db { /* * Schema: @@ -49,7 +43,7 @@ sqlite3_ptr open_db() { return dbptr; } -size_t db_load_securities( +size_t load_securities( const std::set& symbols, const std::function& callback) { @@ -86,28 +80,15 @@ size_t db_load_securities( return count; } -// std::unique_ptr fetch_known_securities() { -// auto security_db = std::make_unique(); -// auto dbptr = open_db(); -// auto db = dbptr.get(); -// auto select_stmt = prepare_query(db, "select key from securities where expiration > ?"); -// auto select = select_stmt.get(); -// sqlite3_bind_int(select, 1, datetime::today().to_short()); -// auto result = sqlite3_step(select); -// while (result == SQLITE_ROW) { -// uint32_t security_key = sqlite3_column_int(select, 0); -// security_db->known_keys.insert(SecurityKey(security_key)); -// result = sqlite3_step(select); -// } -// return security_db; -// } - -bool store_if_new(SecurityDB& db, const api::Security& security) { - auto key = SecurityKey(security.id()); - if (db.known_keys.find(key) == db.known_keys.end()) { - db.known_keys.insert(key); - auto dbptr = open_db(); - auto insert = utils::prepare_query(dbptr, "insert into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?)"); +void store_if_new(const std::vector& handles) { + auto dbptr = open_db(); + auto insert = utils::prepare_query(dbptr, "insert into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?)"); + api::Security security; + sqlite3_exec_nofail(dbptr, "BEGIN TRANSACTION"); + for (const auto& handle : handles) { + bts_assert(handle.key() != invalid_security_key); + security.Clear(); + convert_security(handle.security(), &security); auto buffer = security.SerializeAsString(); SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, security.id()), dbptr); SQLITE_CHK(sqlite3_bind_text(insert.get(), 2, security.symbol().c_str(), security.symbol().size(), SQLITE_STATIC), dbptr); @@ -119,24 +100,21 @@ bool store_if_new(SecurityDB& db, const api::Security& security) { if (result != SQLITE_DONE) { LOG(md, Warn, "Failed to insert security into security db. key={}", security.id()); } - return true; } - return false; + sqlite3_exec_nofail(dbptr, "COMMIT TRANSACTION"); } #else namespace bts { namespace md { -struct SecurityDB {}; +namespace db { -size_t db_load_securities( +size_t load_securities( const std::set& symbols, const std::function& callback) { return 0; } -bool store_if_new(SecurityDB&, const api::Security&) { - return false; -} +void store_if_new(const std::vector& handles) {} #endif @@ -169,4 +147,4 @@ datetime::Date expiration(const api::Security& security) { } } -}}//namespace \ No newline at end of file +}}}//namespace \ No newline at end of file diff --git a/src/lib/md/SecurityDB.h b/src/lib/md/SecurityDB.h index fc35ba90a..e1aab5603 100644 --- a/src/lib/md/SecurityDB.h +++ b/src/lib/md/SecurityDB.h @@ -3,30 +3,29 @@ #include "lib/domain_model/common.h" #include "lib/domain_model/Security.pb.h" #include "lib/utils/datetime.h" +#include "lib/md/MDProducerHandle.h" #include "common.h" #include "UserSymbol.h" #include +#include #include namespace bts { namespace api {class Security;} namespace md { +namespace db { struct SecurityDB; // Loads securities matching the given filter for producers to pre-populate security lists // note: possible to use this to optimize initial memory layout? sort by expiration? -size_t db_load_securities( +size_t load_securities( const std::set& symbols, const std::function& callback); -// Fetch an index of security keys -std::unique_ptr fetch_known_securities(); - -bool store_if_new(SecurityDB* db, const api::Security& security); +void store_if_new(const std::vector& handles); datetime::Date expiration(const api::Security& security); - -}}//namespace \ No newline at end of file +}}}//namespace \ No newline at end of file diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 69567de2c..dc83813c2 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -6,6 +6,7 @@ #include "lib/config/config.h" #include "lib/utils/Sqlite_utils.h" #include "lib/logging/logging.h" +#include "lib/md/SecurityDB.h" #include #include // for reservation @@ -17,27 +18,18 @@ using namespace std; using namespace bts::utils; +using std::size_t; namespace bts { namespace md { -//GCC_DIAG_OFF("-Weffc++") // Plain Struct -struct SecurityIDTask { - std::string descriptor; - SecurityType type; - SecurityIDRecorder::callback cb; - void* arg; -}; -//GCC_DIAG_RESTORE() struct SecurityIDRecorderState { SecurityIDRecorderState() : - work(), pending_handle(), db(nullptr, sqlite3_close), select_stmt(nullptr, sqlite3_finalize), insert_stmt(nullptr, sqlite3_finalize) {} - tbb::concurrent_queue work; tbb::concurrent_queue pending_handle; sqlite3_ptr db; sqlite3_stmt_ptr select_stmt; @@ -75,100 +67,62 @@ void SecurityIDRecorder::init() { _self->select_stmt = prepare_query(_self->db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); } -void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType type, callback cb, void* arg) { - SecurityIDTask task; - task.descriptor = descriptor; - task.type = type; - task.cb = cb; - task.arg = arg; - _self->work.push(task); -} - -void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType type, Callback cb) { - Callback* pcb = new Callback(cb); - get_key(descriptor, type, [](void* data, security_key key) { - Callback* pcb = static_cast(data); - DEFER(if (pcb) delete pcb); // exception safe - (*pcb)(key); - }, pcb); -} - void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { _self->pending_handle.push(handle); } bool SecurityIDRecorder::do_work() { - // Check if any handles need work first - work_pending_handles(); - // Drain the work queue - // Query the database for each security. If it has an ID return it + // Query the database for each security. If it has an ID set it // If it does not have an ID add it to a list // Then persist that list - SecurityIDTask temp; - vector new_securities; + // Once a security has a key it is public so record it in the security database + MDProducerHandle temp; + vector new_handles; + vector new_descriptors; size_t count = 0; bool incomplete = false; - while (_self->work.try_pop(temp)) { - security_key key = search(temp.descriptor); - if (key != invalid_security_key) { - temp.cb(temp.arg, key); - } - else { - new_securities.push_back(temp); - ++count; - // Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat - if (count > 500) { - incomplete = true; - break; + string descriptor; + while (_self->pending_handle.try_pop(temp)) { + if (temp.key() == invalid_security_key) { + descriptor = security_descriptor(temp.security()); + security_key key = search(descriptor); + if (key != invalid_security_key) { + temp.set_key(key); + } + else { + // always push in parallel + new_handles.push_back(temp); + new_descriptors.push_back(descriptor); + ++count; + // Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat + if (count > 500) { + incomplete = true; + break; + } } } } - if (new_securities.size() > 0) { + if (new_handles.size() > 0) { sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); - for (SecurityIDTask& security : new_securities) { - security_key key = insert(security.descriptor, security.type); + for (size_t x = 0; x < new_handles.size(); ++x) { + security_key key = insert(new_descriptors[x], new_handles[x].security().type); bts_assert_d(key); - security.cb(security.arg, key); + new_handles[x].set_key(key); } sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + + // record the securities + db::store_if_new(new_handles); } + return incomplete; } -void SecurityIDRecorder::work_pending_handles() { - MDProducerHandle handle; - vector defer; - while(_self->pending_handle.try_pop(handle)) { - // A spread will not have a descriptor until all of its leg keys are available - string descriptor = security_descriptor(handle.security()); - if (descriptor.empty()) { - defer.push_back(handle); - continue; - } - - MDProducerHandle* phandle = new MDProducerHandle(handle); - DEFER(if (phandle) delete phandle); - get_key(descriptor, handle.security().type, [](void* arg, security_key key) { - auto* phandle = static_cast(arg); - if (phandle) { - phandle->set_key(key); - delete phandle; - } - }, phandle); - phandle = nullptr; - } - - for (auto& handle : defer) { - _self->pending_handle.push(handle); - } -} - - security_key SecurityIDRecorder::search(const std::string& descriptor) { sqlite3_bind_text(_self->select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); DEFER(sqlite3_reset(_self->select_stmt.get())); diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index 649770fee..207a93c38 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -2,7 +2,6 @@ #ifndef WIN32 #include "common.h" #include "MDProducerHandle.h" -#include "lib/utils/bts_export.h" #include #include @@ -19,7 +18,7 @@ namespace md { */ struct SecurityIDRecorderState; -class BTS_EXPORT SecurityIDRecorder +class SecurityIDRecorder { public: SecurityIDRecorder(); @@ -27,12 +26,6 @@ public: typedef void(*callback)(void*, security_key); typedef std::function Callback; - // Asyncronously retrieve the security_key for the given descriptor - void get_key(const std::string& descriptor, SecurityType, callback cb, void* arg); - - // A less efficient but higher level get_key which performs heap allocation in the call path - void get_key(const std::string& descriptor, SecurityType, Callback cb); - // A higher level interface which allows you to just pass in a handle and have it generate a descriptor and set the key void ensure_key(MDProducerHandle handle); @@ -47,8 +40,6 @@ private: security_key search(const std::string& descriptor); security_key insert(const std::string& descriptor, SecurityType); - void work_pending_handles(); - BTS_NO_COPY(SecurityIDRecorder); }; diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index 7e2814191..c38942a52 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -82,8 +82,7 @@ MDP3Listener::MDP3Listener( MDProducer& producer, SecurityIDRecorder& recorder, co2::SettlementRecorder& settles, - std::vector channels, - utils::Dispatcher& disp) : + std::vector channels) : _index(index), _producer(producer), _packet_arrived(), @@ -93,17 +92,13 @@ MDP3Listener::MDP3Listener( _symbol_filter_storage(), _symbol_filter(), _channels(channels), - _security_recorder_input_timer(disp, 900, [&]() { dequeue_securities(); }), - _security_recorder_input(), _concurrency_monitor(0), _book_updates() { _book_updates.reserve(32); - _security_recorder_input_timer.start(); } MDP3Listener::~MDP3Listener() { - _security_recorder_input_timer.stop(); } // @@ -568,35 +563,9 @@ void MDP3Listener::add_security(const FixedSecurity& security) { // Add it to the index so it can receive messages SecurityData* data = _index.add_security(security.numeric_id, handle); - // Computing the descriptor may be expensive so defer that processing to the background worker thread (see dequeue_securities) - _security_recorder_input.push(data); -} - -void MDP3Listener::dequeue_securities() { - // already running on '_background' thread - vector defer; - - SecurityData* data = nullptr; - while (_security_recorder_input.try_pop(data)) { - bts_assert(data->handle.security().key == invalid_security_key); - - string descriptor = security_descriptor(data->handle.security()); - if (descriptor.empty()) { - defer.push_back(data); - } - else { - _security_recorder.get_key(descriptor, data->handle.security().type, [](void* pv_handle, security_key key) { - SecurityData* pdata = (SecurityData*)pv_handle; - pdata->handle.set_key(key); - }, data); - // Restore saved settlements - _settlement_recorder.restore_settlement(data->handle); - } - } - - // try again later (when spread legs have resolved) - for (auto* retry : defer) { - _security_recorder_input.push(retry); + // Assign a security key + if (data->handle.key() == invalid_security_key) { + _security_recorder.ensure_key(data->handle); } } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index ab8bf7c5d..979091a6b 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -18,7 +18,7 @@ class MDP3Listener : public onixs::SecurityListener { public: - MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector, utils::Dispatcher&); + MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector); virtual ~MDP3Listener(); // @@ -95,7 +95,6 @@ public: void add_security(const FixedSecurity& security); private: - void dequeue_securities(); enum SecurityFilterAction { ADD, SKIP, DELETE }; template SecurityFilterAction filter_security(int exchange_id, const Sec& sec) { @@ -124,8 +123,6 @@ private: std::vector _symbol_filter_storage; std::set _symbol_filter; std::vector _channels; - utils::DispatcherTimer _security_recorder_input_timer; - tbb::concurrent_queue _security_recorder_input; std::atomic _concurrency_monitor; struct BookData { diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 53e102c1f..1ac976386 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -318,13 +318,13 @@ unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, con producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } - unique_ptr listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels, utils::Dispatcher::current())); + unique_ptr listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels)); for (auto& symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } // Load the list of known securities and add them to the index for fast startup - db_load_securities(channel.symbols, [&](const api::Security& security) { + db::load_securities(channel.symbols, [&](const api::Security& security) { with_security(securitytype_of(security), num_legs(security), [&](FixedSecurity& fs) { convert_security(security, fs, true); listener->add_security(fs); From dba3c541beeff859dc7b05cc8433e4127fc357a0 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 9 Jan 2018 18:17:47 -0500 Subject: [PATCH 3/7] SyntheticSecurityService spark-322 --- src/lib/CMakeDeps.cmake | 2 +- .../templates/cpp/server.pb.h | 118 +++++++++--------- .../services/SyntheticSecurityService.proto | 49 ++++++++ src/lib/domain_model/SecurityKey.h | 5 + src/lib/domain_model/common.h | 31 +++-- src/lib/md/SecurityDB.cpp | 39 +++--- src/lib/md/SecurityDB.h | 2 +- src/lib/md/SecurityIDRecorder.cpp | 74 +++++++---- src/lib/md/SecurityIDRecorder.h | 12 +- src/lib/md/Shm/FixedSecurity.cpp | 10 +- src/lib/md/Shm/FixedSecurity.h | 3 +- src/lib/md/SyntheticSecurityService.cpp | 102 +++++++++++++++ src/lib/md/SyntheticSecurityService.h | 46 +++++++ src/lib/md/common.cpp | 1 + src/lib/md/common.h | 1 + src/lib/sparkproto/SparkProtos.proto | 1 + src/lib/utils/LateJoinEvent.h | 58 +++++++++ 17 files changed, 429 insertions(+), 125 deletions(-) create mode 100644 src/lib/api/services/SyntheticSecurityService.proto create mode 100644 src/lib/domain_model/SecurityKey.h create mode 100644 src/lib/md/SyntheticSecurityService.cpp create mode 100644 src/lib/md/SyntheticSecurityService.h create mode 100644 src/lib/utils/LateJoinEvent.h diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index 07c3e2ca6..033ec0ed5 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/services/AccountService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OrderService2.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/windows_test_driver.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/services/AccountService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OrderService2.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/windows_test_driver.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp) \ No newline at end of file diff --git a/src/lib/api/service_compiler/templates/cpp/server.pb.h b/src/lib/api/service_compiler/templates/cpp/server.pb.h index e582225e0..7fb2c6e67 100644 --- a/src/lib/api/service_compiler/templates/cpp/server.pb.h +++ b/src/lib/api/service_compiler/templates/cpp/server.pb.h @@ -1,60 +1,58 @@ -#pragma once -#include "{{filename name}}.pb.h" -#include "lib/api/public/ApiServer.h" -#include "lib/utils/bts_export.h" -#include -#include -#include -#include -#include - -// #include -// #include -namespace bts { namespace api { - class ServiceContext; -}} - -{{ns-open package}} - -// Forward declaractions -{{#each message_type}} -class {{name}};{{/each}} - -{{#each service}} -// {{comment}} -class {{name}}Base : public bts::api::ApiServer { -protected: - {{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier=""); - - const char* name() const override; - - {{#each method}} - {{#if server_streaming}} - typedef std::function*, const std::vector<{{output_type_cpp}}>*, std::set<{{key_type.cpptype}}>*)> {{cap name}}Publisher; - {{cap name}}Publisher make_{{name}}_publisher() const; - {{cap name}}Publisher make_{{name}}_publisher(const std::string &) const; - {{cap name}}Publisher make_{{name}}_publisher(const {{input_type}}& qualifier) const; - {{else}} - // {{comment}} - virtual void {{name}}( - const {{input_type}}& request, - const api::Response<{{output_type_cpp}}>& reply) = 0; - {{/if}} - {{/each}} - - enum ServiceMethods { - SYS_PING = -1, - {{#each method}} - {{upper name}} = {{@index}}, - {{/each}} - }; - -private: - void dispatch(const bts::api::RequestHeader&, google::protobuf::io::ZeroCopyInputStream&, OnResponse) override; - {{#each method}}{{#if server_streaming}} - {{cap name}}Publisher make_{{name}}_publisher(TableHandle) const; - {{/if}}{{/each}} -}; - -{{/each}} -{{ns-close package}} +#pragma once +#include "{{filename name}}.pb.h" +#include "lib/api/public/ApiServer.h" +#include "lib/utils/bts_export.h" +#include +#include +#include +#include +#include + +namespace bts { namespace api { + class ServiceContext; +}} + +{{ns-open package}} + +// Forward declaractions +{{#each message_type}} +class {{name}};{{/each}} + +{{#each service}} +// {{comment}} +class {{name}}Base : public bts::api::ApiServer { +protected: + {{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier=""); + + const char* name() const override; + + {{#each method}} + {{#if server_streaming}} + typedef std::function*, const std::vector<{{output_type_cpp}}>*, std::set<{{key_type.cpptype}}>*)> {{cap name}}Publisher; + {{cap name}}Publisher make_{{name}}_publisher() const; + {{cap name}}Publisher make_{{name}}_publisher(const std::string &) const; + {{cap name}}Publisher make_{{name}}_publisher(const {{input_type}}& qualifier) const; + {{else}} + // {{comment}} + virtual void {{name}}( + const {{input_type}}& request, + const api::Response<{{output_type_cpp}}>& reply) = 0; + {{/if}} + {{/each}} + + enum ServiceMethods { + SYS_PING = -1, + {{#each method}} + {{upper name}} = {{@index}}, + {{/each}} + }; + +private: + void dispatch(const bts::api::RequestHeader&, google::protobuf::io::ZeroCopyInputStream&, OnResponse) override; + {{#each method}}{{#if server_streaming}} + {{cap name}}Publisher make_{{name}}_publisher(TableHandle) const; + {{/if}}{{/each}} +}; + +{{/each}} +{{ns-close package}} diff --git a/src/lib/api/services/SyntheticSecurityService.proto b/src/lib/api/services/SyntheticSecurityService.proto new file mode 100644 index 000000000..00d4a14f3 --- /dev/null +++ b/src/lib/api/services/SyntheticSecurityService.proto @@ -0,0 +1,49 @@ +syntax="proto3"; +package bts.api.synthetic; +option csharp_namespace="bts.api.synthetic"; + +import "lib/api/apioptions.proto"; +import "lib/domain_model/Security.proto"; +//import "google/protobuf/timestamp.proto"; +import "lib/sparkproto/SparkProtos.proto"; + +enum SSError { + SSERROR_OK = 0; + SSERROR_UNKNOWN_SECURITY = 1; + SSERROR_INCOMPLETE_DATA = 2; + SSERROR_NOT_SYNTHETIC = 3; + SSERROR_ID_ASSIGNED = 4; +} + +message UpdateResult { + SSError error = 1; +} +message CreateResult { + uint32 key = 1; + SSError error = 2; +} + +message Metadata { + uint32 security_key = 1; + spark.ExecutionStrategy strategy = 2; +} +message MetadataQuery { + repeated uint32 security_key = 1; +} +message MetadataCollection { + repeated Metadata strategy = 1; +} + +service SyntheticSecurityService { + option(bts.api.service_description)="Create or query synthetic securities."; + + // Create synthetic + rpc create_synthetic(api.Security) returns (CreateResult); + // Subscribe synthetics -- code only + + // set/update metadata // spark.ExecutionStrategy + rpc update_security_metadata(MetadataCollection) returns (UpdateResult); + // query metadata + rpc query_security_metadata(MetadataQuery) returns (MetadataCollection); + // subscribe metadata -- code only +} diff --git a/src/lib/domain_model/SecurityKey.h b/src/lib/domain_model/SecurityKey.h new file mode 100644 index 000000000..d9e3021b2 --- /dev/null +++ b/src/lib/domain_model/SecurityKey.h @@ -0,0 +1,5 @@ +#pragma once +#include "lib/utils/ValueWrapper.h" +namespace bts { +using SecurityKey = utils::ValueWrapper; +} \ No newline at end of file diff --git a/src/lib/domain_model/common.h b/src/lib/domain_model/common.h index 2f474c0c1..af6f2a590 100644 --- a/src/lib/domain_model/common.h +++ b/src/lib/domain_model/common.h @@ -1,17 +1,14 @@ -#pragma once - -#include -#include - -#include "lib/utils/ValueWrapper.h" - -namespace bts { -using SecurityKey = utils::ValueWrapper; - - - -using Price = utils::ValueWrapper; -using Quantity = utils::ValueWrapper; -using UserLogin = utils::ValueWrapper; -} - +#pragma once + +#include +#include + +#include "lib/utils/ValueWrapper.h" +#include "lib/domain_model/SecurityKey.h" + +namespace bts { +using Price = utils::ValueWrapper; +using Quantity = utils::ValueWrapper; +using UserLogin = utils::ValueWrapper; +} + diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index 41a405c77..52402fc2d 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -58,7 +58,7 @@ size_t load_securities( auto result = sqlite3_step(select); Security security; while (result == SQLITE_ROW) { - uint32_t security_key = sqlite3_column_int(select, 0); + security_key security_key = sqlite3_column_int(select, 0); auto symbol = std::string( (char*)sqlite3_column_text(select, 1), sqlite3_column_bytes(select, 1)); @@ -73,6 +73,9 @@ size_t load_securities( count++; callback(security); } + else { + LOG(md, Warn, "Failed to parse security from database. id = {}", security_key); + } } result = sqlite3_step(select); } @@ -80,25 +83,23 @@ size_t load_securities( return count; } -void store_if_new(const std::vector& handles) { +void store_if_new(const std::vector& securities) { auto dbptr = open_db(); - auto insert = utils::prepare_query(dbptr, "insert into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?)"); - api::Security security; + auto insert = utils::prepare_query(dbptr, "insert into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); sqlite3_exec_nofail(dbptr, "BEGIN TRANSACTION"); - for (const auto& handle : handles) { - bts_assert(handle.key() != invalid_security_key); - security.Clear(); - convert_security(handle.security(), &security); - auto buffer = security.SerializeAsString(); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, security.id()), dbptr); - SQLITE_CHK(sqlite3_bind_text(insert.get(), 2, security.symbol().c_str(), security.symbol().size(), SQLITE_STATIC), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 3, security.exchange()), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 4, securitytype_of(security)), dbptr); - SQLITE_CHK(sqlite3_bind_blob(insert.get(), 5, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 6, datetime::today().to_short()), dbptr); - auto result = sqlite3_step(insert.get()); - if (result != SQLITE_DONE) { - LOG(md, Warn, "Failed to insert security into security db. key={}", security.id()); + std::string buffer; + for (const api::Security* psecurity : securities) { + if (psecurity && psecurity->SerializeToString(&buffer)) { + SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, psecurity->id()), dbptr); + SQLITE_CHK(sqlite3_bind_text(insert.get(), 2, psecurity->symbol().c_str(), psecurity->symbol().size(), SQLITE_STATIC), dbptr); + SQLITE_CHK(sqlite3_bind_int(insert.get(), 3, psecurity->exchange()), dbptr); + SQLITE_CHK(sqlite3_bind_int(insert.get(), 4, securitytype_of(*psecurity)), dbptr); + SQLITE_CHK(sqlite3_bind_blob(insert.get(), 5, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr); + SQLITE_CHK(sqlite3_bind_int(insert.get(), 6, datetime::today().to_short()), dbptr); + auto result = sqlite3_step(insert.get()); + if (result != SQLITE_DONE) { + LOG(md, Warn, "Failed to insert security into security db. key={}", psecurity->id()); + } } } sqlite3_exec_nofail(dbptr, "COMMIT TRANSACTION"); @@ -114,7 +115,7 @@ size_t load_securities( const std::function& callback) { return 0; } -void store_if_new(const std::vector& handles) {} +void store_if_new(const std::vector& handles) {} #endif diff --git a/src/lib/md/SecurityDB.h b/src/lib/md/SecurityDB.h index e1aab5603..ed678ba38 100644 --- a/src/lib/md/SecurityDB.h +++ b/src/lib/md/SecurityDB.h @@ -24,7 +24,7 @@ size_t load_securities( const std::set& symbols, const std::function& callback); -void store_if_new(const std::vector& handles); +void store_if_new(const std::vector& handles); datetime::Date expiration(const api::Security& security); diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index dc83813c2..b67da9774 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -23,13 +23,20 @@ using std::size_t; namespace bts { namespace md { +struct SecurityIDEntry { + api::Security security; + SecurityIDRecorder::Callback callback; +}; + struct SecurityIDRecorderState { SecurityIDRecorderState() : + work(), pending_handle(), db(nullptr, sqlite3_close), select_stmt(nullptr, sqlite3_finalize), insert_stmt(nullptr, sqlite3_finalize) {} + tbb::concurrent_queue> work; tbb::concurrent_queue pending_handle; sqlite3_ptr db; sqlite3_stmt_ptr select_stmt; @@ -42,9 +49,7 @@ SecurityIDRecorder::SecurityIDRecorder(): init(); } -SecurityIDRecorder::~SecurityIDRecorder() -{ -} +SecurityIDRecorder::~SecurityIDRecorder() { } void SecurityIDRecorder::init() { string db_dir = config::get().data_path() + "/md/"; @@ -70,57 +75,84 @@ void SecurityIDRecorder::init() { void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { _self->pending_handle.push(handle); } +void SecurityIDRecorder::assign_key(const api::Security& security, const SecurityIDRecorder::Callback& callback) { + if (security.id() != invalid_security_key) { + auto work = std::make_shared(); + work->security = security; + work->callback = callback; + _self->work.push(work); + } +} -bool SecurityIDRecorder::do_work() { +void SecurityIDRecorder::do_work() { + work_handles(); + work_db(); +} + +void SecurityIDRecorder::work_handles() { + Security security; + MDProducerHandle handle; + while (_self->pending_handle.try_pop(handle)) { + security.Clear(); + convert_security(handle.security(), &security); + assign_key(security, [handle](const api::Security& security) mutable { + handle.set_key(security.id()); + }); + } +} + +void SecurityIDRecorder::work_db() { // Drain the work queue // Query the database for each security. If it has an ID set it // If it does not have an ID add it to a list // Then persist that list // Once a security has a key it is public so record it in the security database - MDProducerHandle temp; - vector new_handles; + std::shared_ptr temp; + vector> new_entries; vector new_descriptors; size_t count = 0; - bool incomplete = false; string descriptor; - while (_self->pending_handle.try_pop(temp)) { - if (temp.key() == invalid_security_key) { - descriptor = security_descriptor(temp.security()); + while (_self->work.try_pop(temp)) { + descriptor = security_descriptor(temp->security); security_key key = search(descriptor); if (key != invalid_security_key) { - temp.set_key(key); + temp->security.set_id(key); + temp->callback(temp->security); } else { // always push in parallel - new_handles.push_back(temp); + new_entries.push_back(move(temp)); new_descriptors.push_back(descriptor); ++count; // Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat if (count > 500) { - incomplete = true; break; } } - } } - if (new_handles.size() > 0) { + if (new_entries.size() > 0) { sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); - for (size_t x = 0; x < new_handles.size(); ++x) { - security_key key = insert(new_descriptors[x], new_handles[x].security().type); + for (size_t x = 0; x < new_entries.size(); ++x) { + Security& security = new_entries[x]->security; + security_key key = insert(new_descriptors[x], securitytype_of(security)); bts_assert_d(key); - new_handles[x].set_key(key); + security.set_id(key); + new_entries[x]->callback(security); } sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); // record the securities - db::store_if_new(new_handles); + std::vector save_these; + save_these.reserve(new_entries.size()); + for (const auto& e : new_entries) { + save_these.push_back(&e->security); + } + db::store_if_new(save_these); } - - return incomplete; } security_key SecurityIDRecorder::search(const std::string& descriptor) { diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index 207a93c38..67b7b5c29 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -23,15 +23,16 @@ class SecurityIDRecorder public: SecurityIDRecorder(); ~SecurityIDRecorder(); - typedef void(*callback)(void*, security_key); - typedef std::function Callback; + typedef std::function Callback; - // A higher level interface which allows you to just pass in a handle and have it generate a descriptor and set the key + // Fast - for use by market data producers void ensure_key(MDProducerHandle handle); + // Slow - for use by services + void assign_key(const api::Security& security, const Callback& key_assigned); // Someone must call work periodically to do lookups and flush the reserved ids to disk. // returne true if there is more work to be done - bool do_work(); + void do_work(); private: std::unique_ptr _self; @@ -40,6 +41,9 @@ private: security_key search(const std::string& descriptor); security_key insert(const std::string& descriptor, SecurityType); + void work_handles(); + void work_db(); + BTS_NO_COPY(SecurityIDRecorder); }; diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index b37b282a6..d609a5efc 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -76,7 +76,7 @@ void write_date_string(uint32_t year, uint32_t month, uint32_t day, std::ostring << year - 2000; } -const std::string security_descriptor(const FixedSecurity& security) { +std::string security_descriptor(const FixedSecurity& security) { const char separator = ' '; std::ostringstream result; // If the same symbol is listed at 2 different exchanges it should have a different ID @@ -118,6 +118,14 @@ const std::string security_descriptor(const FixedSecurity& security) { return result.str(); } +std::string security_descriptor(const api::Security& security) { + std::string result; + with_security(security.Contract_case(), num_legs(security), [&](FixedSecurity fs) { + convert_security(security, fs, true); + result = security_descriptor(fs); + }); + return result; +} void get_contract_spec(const FixedSecurity& source, api::ContractSpecification& spec) { spec.set_cash_per_point(source.cme_unit_of_measure_qty); diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index 9eafbcbd1..35d9588c1 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -128,7 +128,8 @@ typedef FixedSpreadSecurity<16> FixedSpreadSecurityS; typedef FixedSpreadSecurity<128> FixedSpreadSecurityL; typedef FixedSpreadSecurity<512> FixedSpreadSecurityXL; -const std::string security_descriptor(const FixedSecurity&); +std::string security_descriptor(const api::Security&); +// std::string security_descriptor(const FixedSecurity&); // Convert a fixedsecurity to a normal Security - caveat: spreads will be normalized Security convert_security(const FixedSecurity&); diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp new file mode 100644 index 000000000..5741ea693 --- /dev/null +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -0,0 +1,102 @@ +#ifndef WIN32 +#include "SyntheticSecurityService.h" + +#include "lib/config/config.h" +#include "lib/api/public/ServiceContext.h" +#include "lib/md/common.h" +#include "lib/md/SecurityIDRecorder.h" +#include "lib/md/SecurityDB.h" +#include "lib/md/get_symbols.h" +#include "lib/utils/Dispatcher.h" + +#include +#include + +namespace bts { +namespace api { +namespace synthetic { + +struct SecurityLess { + bool operator()(const api::Security& left, const api::Security& right) { + return left.id() < right.id(); + } +}; + +struct SyntheticSecurityService::Pimpl { + md::SecurityIDRecorder recorder; + utils::LateJoinEvent on_security; + std::unordered_map meta; +}; + +SyntheticSecurityService::SyntheticSecurityService(api::ServiceContext& context) + : SyntheticSecurityServiceBase(context), + _me(new Pimpl()) { + + // load securities + std::set symbols; + md::get_system_md_symbols(symbols); + md::db::load_securities(symbols, [=](const api::Security& security) { + if (security.Contract_case() == api::Security::kSynthetic) { + _me->on_security.publish(security); + } + }); + + // Load metadata + +} +SyntheticSecurityService::~SyntheticSecurityService() { } + +utils::LateJoinEvent& + SyntheticSecurityService::on_security() { + return _me->on_security; +} + +void send_reply(const api::Response& fn, SSError err, uint32_t key=md::invalid_security_key) { + CreateResult r; + r.set_key(key); + r.set_error(err); + fn(r); +} + + +void SyntheticSecurityService::create_synthetic( + const Security& security, + const api::Response& reply) { + + // Validate security is synthetic + if (security.Contract_case() != Security::kSynthetic) { + send_reply(reply, SSERROR_NOT_SYNTHETIC); + return; + } + if (security.id() != md::invalid_security_key) { + send_reply(reply, SSERROR_ID_ASSIGNED); + return; + } + + // Assign security key + _me->recorder.assign_key(security, [=](const api::Security& security) { + // Add to list of known securities + _me->on_security.publish(security); + // signal success + send_reply(reply, SSERROR_OK, security.id()); + }); +} + +void SyntheticSecurityService::update_security_metadata( + const MetadataCollection& request, + const api::Response& reply) { + (void)request; + (void)reply; +} + +void SyntheticSecurityService::query_security_metadata( + const MetadataQuery& request, + const api::Response& reply) { + (void)request; + (void)reply; +} + + +}}}//namespace + +#endif \ No newline at end of file diff --git a/src/lib/md/SyntheticSecurityService.h b/src/lib/md/SyntheticSecurityService.h new file mode 100644 index 000000000..ebcf478b0 --- /dev/null +++ b/src/lib/md/SyntheticSecurityService.h @@ -0,0 +1,46 @@ +#pragma once +#include "lib/md/common.h" +#include "lib/api/services/SyntheticSecurityService.pb.h" +#include "lib/api/services/SyntheticSecurityService.server.pb.h" +#include "lib/utils/LateJoinEvent.h" +#include +#include + + +namespace bts { +namespace api { +namespace synthetic { + + +class SyntheticSecurityService : private api::synthetic::SyntheticSecurityServiceBase +{ + struct Pimpl; + std::unique_ptr _me; + + public: + SyntheticSecurityService(api::ServiceContext& context); + ~SyntheticSecurityService(); + + // on_security + utils::LateJoinEvent& on_security(); + + // on_execution_strategy + + private: + void create_synthetic( + const Security& request, + const api::Response& reply) override; + + void update_security_metadata( + const MetadataCollection& request, + const api::Response& reply) override; + + void query_security_metadata( + const MetadataQuery& request, + const api::Response& reply) override; + +}; + +}}}//namespace + + diff --git a/src/lib/md/common.cpp b/src/lib/md/common.cpp index 465eef209..4071f33ec 100644 --- a/src/lib/md/common.cpp +++ b/src/lib/md/common.cpp @@ -19,6 +19,7 @@ namespace bts { os << sym.symbol() << "-" << api::SecurityType_Name(sym.type()); return os; } + } namespace co2 { diff --git a/src/lib/md/common.h b/src/lib/md/common.h index 45ff264c3..cb3ebb646 100644 --- a/src/lib/md/common.h +++ b/src/lib/md/common.h @@ -3,6 +3,7 @@ #include "lib/utils/bts_export.h" #include "lib/utils/datetime.h" +#include "lib/domain_model/SecurityKey.h" #include "lib/md/protos/co2.pb.h" #include "lib/domain_model/Security.pb.h" diff --git a/src/lib/sparkproto/SparkProtos.proto b/src/lib/sparkproto/SparkProtos.proto index 80d19554e..ad36d121d 100644 --- a/src/lib/sparkproto/SparkProtos.proto +++ b/src/lib/sparkproto/SparkProtos.proto @@ -791,6 +791,7 @@ message CrossingHedgeOrder message ExecutionStrategy { + bool deleted = 1; map leg_params_map = 20; } diff --git a/src/lib/utils/LateJoinEvent.h b/src/lib/utils/LateJoinEvent.h new file mode 100644 index 000000000..85d7f4faf --- /dev/null +++ b/src/lib/utils/LateJoinEvent.h @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include +#include + +namespace bts { +namespace utils { + +template +class LateJoinEvent { + typedef std::function Callback; + typedef int Handle; + Handle _id; + std::vector _items; + std::list> _subscribers; + + void broadcast(const T& item) { + for(const auto& t : _subscribers) { + std::get(t)(item); + } + } + + public: + LateJoinEvent() : + _id(0), + _items(), + _subscribers() {} + ~LateJoinEvent() {} + + void publish(T&& item) { + _items.push_back(move(item)); + broadcast(_items.back()); + } + void publish(const T& item) { + _items.push_back(item); + broadcast(item); + } + + Handle subscribe(Callback& cb) { + auto id = ++_id; + _subscribers.push_back(std::make_tuple(id, cb)); + for (const auto& item : _items) { + cb(item); + } + return id; + } + void unsubscribe(Handle h) { + _subscribers.erase( + std::remove_if(_subscribers.begin(), + _subscribers.end(), + [h](std::tuple& t) { return std::get(t) == h; })); + } +}; + + +}}//namespace \ No newline at end of file From 9d62b4592e6c452b5a8ccb61bae531af79efaf93 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Jan 2018 11:19:39 -0500 Subject: [PATCH 4/7] spark 334 Load/save security metadata --- .../services/SyntheticSecurityService.proto | 34 ++++--- src/lib/md/SecurityDB.cpp | 48 ++++++++++ src/lib/md/SecurityDB.h | 5 + src/lib/md/SyntheticSecurityService.cpp | 95 +++++++++++++++---- src/lib/md/SyntheticSecurityService.h | 11 ++- 5 files changed, 154 insertions(+), 39 deletions(-) diff --git a/src/lib/api/services/SyntheticSecurityService.proto b/src/lib/api/services/SyntheticSecurityService.proto index 00d4a14f3..89d096bc2 100644 --- a/src/lib/api/services/SyntheticSecurityService.proto +++ b/src/lib/api/services/SyntheticSecurityService.proto @@ -7,20 +7,23 @@ import "lib/domain_model/Security.proto"; //import "google/protobuf/timestamp.proto"; import "lib/sparkproto/SparkProtos.proto"; -enum SSError { - SSERROR_OK = 0; - SSERROR_UNKNOWN_SECURITY = 1; - SSERROR_INCOMPLETE_DATA = 2; - SSERROR_NOT_SYNTHETIC = 3; - SSERROR_ID_ASSIGNED = 4; -} - -message UpdateResult { - SSError error = 1; -} message CreateResult { + enum Error { + ERROR_OK = 0; + ERROR_NOT_SYNTHETIC = 1; + ERROR_ID_ALREADY_ASSIGNED = 2; + } uint32 key = 1; - SSError error = 2; + Error error = 2; +} +message UpdateResult { + enum Error { + ERROR_OK = 0; + ERROR_MISSING_KEY = 1; + ERROR_UNKNOWN_SECURITY = 2; + ERROR_DATABASE = 3; + } + Error error = 1; } message Metadata { @@ -31,7 +34,7 @@ message MetadataQuery { repeated uint32 security_key = 1; } message MetadataCollection { - repeated Metadata strategy = 1; + repeated Metadata data = 1; } service SyntheticSecurityService { @@ -42,8 +45,9 @@ service SyntheticSecurityService { // Subscribe synthetics -- code only // set/update metadata // spark.ExecutionStrategy - rpc update_security_metadata(MetadataCollection) returns (UpdateResult); + rpc update_security_metadata(Metadata) returns (UpdateResult); // query metadata - rpc query_security_metadata(MetadataQuery) returns (MetadataCollection); + rpc query_security_metadata(MetadataQuery) returns (MetadataCollection) + { option (bts.api.method_description)="Returns the metadata for the given set of securities if it exists. If no data for a particular key exists it will not be present in the output collection."; } // subscribe metadata -- code only } diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index 52402fc2d..ff377fe5d 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -40,6 +40,9 @@ sqlite3_ptr open_db() { sqlite3_exec_nofail(db, "CREATE TABLE IF NOT EXIST securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL"); + sqlite3_exec_nofail(db, + "CREATE TABLE IF NOT EXIST metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); + return dbptr; } @@ -105,6 +108,45 @@ void store_if_new(const std::vector& securities) { sqlite3_exec_nofail(dbptr, "COMMIT TRANSACTION"); } + +std::vector load_metadata() { + std::vector values; + auto dbptr = open_db(); + auto db = dbptr.get(); + auto select_stmt = prepare_query(db, "select key,data from metadata"); + auto select = select_stmt.get(); + auto result = sqlite3_step(select); + while(result == SQLITE_ROW) { + uint32_t key = sqlite3_column_int(select, 0); + auto blob = sqlite3_column_blob(select, 1); + auto len = sqlite3_column_bytes(select, 1); + api::synthetic::Metadata data; + if (data.ParseFromArray(blob, len)) { + bts_assert(key == data.security_key()); + values.push_back(data); + } + else { + LOG(md, Warn, "Failed to parse security metadata for security_key={}", key); + } + result = sqlite3_step(select); + } + return values; +} +bool store_metadata(const api::synthetic::Metadata& data) { + auto dbptr = open_db(); + auto insert = utils::prepare_query(dbptr, "insert into metadata (key, data) values (?,?)"); + + SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, data.security_key()), dbptr); + std::string buffer = data.SerializeAsString(); + SQLITE_CHK(sqlite3_bind_blob(insert.get(), 2, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr); + auto result = sqlite3_step(insert.get()); + if (result != SQLITE_DONE) { + LOG(md, Warn, "Failed to insert metadata. sqlite result {}", result); + return false; + } + return true; +} + #else namespace bts { namespace md { @@ -116,6 +158,12 @@ size_t load_securities( return 0; } void store_if_new(const std::vector& handles) {} +// std::vector load_metadata() { +// std::vector result; +// return result; +// } +// void store_metadata(const api::synthetic::Metadata&) { +// } #endif diff --git a/src/lib/md/SecurityDB.h b/src/lib/md/SecurityDB.h index ed678ba38..3e7b9e2ec 100644 --- a/src/lib/md/SecurityDB.h +++ b/src/lib/md/SecurityDB.h @@ -4,6 +4,7 @@ #include "lib/domain_model/Security.pb.h" #include "lib/utils/datetime.h" #include "lib/md/MDProducerHandle.h" +#include "lib/api/services/SyntheticSecurityService.pb.h" #include "common.h" #include "UserSymbol.h" @@ -26,6 +27,10 @@ size_t load_securities( void store_if_new(const std::vector& handles); +// metadata +std::vector load_metadata(); +bool store_metadata(const api::synthetic::Metadata&); + datetime::Date expiration(const api::Security& security); }}}//namespace \ No newline at end of file diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp index 5741ea693..f0964d6ee 100644 --- a/src/lib/md/SyntheticSecurityService.cpp +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -3,14 +3,13 @@ #include "lib/config/config.h" #include "lib/api/public/ServiceContext.h" +#include "lib/api/services/PersistenceService.pb.h" #include "lib/md/common.h" #include "lib/md/SecurityIDRecorder.h" #include "lib/md/SecurityDB.h" #include "lib/md/get_symbols.h" #include "lib/utils/Dispatcher.h" - #include -#include namespace bts { namespace api { @@ -23,9 +22,20 @@ struct SecurityLess { }; struct SyntheticSecurityService::Pimpl { + Pimpl() : + recorder(), + timer(utils::Dispatcher::background(), 200, [&]() { recorder.do_work(); }), + known_synthetics(), + on_security(rxcpp::identity_current_thread(), rxcpp::composite_subscription()), + meta(), + on_metadata() + {} md::SecurityIDRecorder recorder; - utils::LateJoinEvent on_security; - std::unordered_map meta; + utils::DispatcherTimer timer; + std::set known_synthetics; + rxcpp::subjects::replay on_security; + std::unordered_map meta; + rxcpp::subjects::subject on_metadata; }; SyntheticSecurityService::SyntheticSecurityService(api::ServiceContext& context) @@ -35,65 +45,110 @@ SyntheticSecurityService::SyntheticSecurityService(api::ServiceContext& context) // load securities std::set symbols; md::get_system_md_symbols(symbols); + auto o = _me->on_security.get_subscriber(); md::db::load_securities(symbols, [=](const api::Security& security) { if (security.Contract_case() == api::Security::kSynthetic) { - _me->on_security.publish(security); + o.on_next(security); + _me->known_synthetics.insert(SecurityKey(security.id())); } }); // Load metadata + auto metadata = md::db::load_metadata(); + for (const auto& entry : metadata) { + _me->meta.insert(std::make_pair(entry.security_key(), entry)); + } } SyntheticSecurityService::~SyntheticSecurityService() { } -utils::LateJoinEvent& - SyntheticSecurityService::on_security() { - return _me->on_security; +rxcpp::observable SyntheticSecurityService::on_security() { + return _me->on_security.get_observable(); +} +const Metadata* SyntheticSecurityService::get_metadata(SecurityKey key) { + auto iter = _me->meta.find(key); + if (iter != _me->meta.end()) { + return &iter->second; + } + return nullptr; +} +rxcpp::observable SyntheticSecurityService::on_metadata() { + return _me->on_metadata.get_observable(); } -void send_reply(const api::Response& fn, SSError err, uint32_t key=md::invalid_security_key) { +void send_reply(const api::Response& fn, CreateResult::Error err, uint32_t key=md::invalid_security_key) { CreateResult r; r.set_key(key); r.set_error(err); fn(r); } - void SyntheticSecurityService::create_synthetic( const Security& security, const api::Response& reply) { + CreateResult result; // Validate security is synthetic if (security.Contract_case() != Security::kSynthetic) { - send_reply(reply, SSERROR_NOT_SYNTHETIC); + send_reply(reply, CreateResult::ERROR_NOT_SYNTHETIC); return; } if (security.id() != md::invalid_security_key) { - send_reply(reply, SSERROR_ID_ASSIGNED); + send_reply(reply, CreateResult::ERROR_ID_ALREADY_ASSIGNED); return; } // Assign security key _me->recorder.assign_key(security, [=](const api::Security& security) { - // Add to list of known securities - _me->on_security.publish(security); + // Notify listeners that a new security is added + _me->known_synthetics.insert(SecurityKey(security.id())); + _me->on_security.get_subscriber().on_next(security); // signal success - send_reply(reply, SSERROR_OK, security.id()); + send_reply(reply, CreateResult::ERROR_OK, security.id()); }); } void SyntheticSecurityService::update_security_metadata( - const MetadataCollection& request, + const Metadata& request, const api::Response& reply) { - (void)request; - (void)reply; + UpdateResult result; + if (request.security_key() == md::invalid_security_key) { + result.set_error(UpdateResult::ERROR_MISSING_KEY); + reply(result); + return; + } + auto key = SecurityKey(request.security_key()); + if (_me->known_synthetics.find(key) == _me->known_synthetics.end()) { + result.set_error(UpdateResult::ERROR_UNKNOWN_SECURITY); + reply(result); + return; + } + // update the on disk database + if (!md::db::store_metadata(request)) { + result.set_error(UpdateResult::ERROR_DATABASE); + reply(result); + return; + } + // update the local database + _me->meta[key] = request; + // notify subscribers + + reply(result); } void SyntheticSecurityService::query_security_metadata( const MetadataQuery& request, const api::Response& reply) { - (void)request; - (void)reply; + + MetadataCollection result; + for (const auto k : request.security_key()) { + SecurityKey key(k); + auto iter = _me->meta.find(key); + if (iter != _me->meta.end()) { + result.mutable_data()->Add()->CopyFrom(iter->second); + } + } + reply(result); } diff --git a/src/lib/md/SyntheticSecurityService.h b/src/lib/md/SyntheticSecurityService.h index ebcf478b0..2186b3ea0 100644 --- a/src/lib/md/SyntheticSecurityService.h +++ b/src/lib/md/SyntheticSecurityService.h @@ -2,8 +2,9 @@ #include "lib/md/common.h" #include "lib/api/services/SyntheticSecurityService.pb.h" #include "lib/api/services/SyntheticSecurityService.server.pb.h" -#include "lib/utils/LateJoinEvent.h" +#include "lib/domain_model/SecurityKey.h" #include +#include #include @@ -22,9 +23,11 @@ class SyntheticSecurityService : private api::synthetic::SyntheticSecurityServic ~SyntheticSecurityService(); // on_security - utils::LateJoinEvent& on_security(); + rxcpp::observable on_security(); - // on_execution_strategy + // query/subscribe metadata + const Metadata* get_metadata(SecurityKey); + rxcpp::observable on_metadata(); private: void create_synthetic( @@ -32,7 +35,7 @@ class SyntheticSecurityService : private api::synthetic::SyntheticSecurityServic const api::Response& reply) override; void update_security_metadata( - const MetadataCollection& request, + const Metadata& request, const api::Response& reply) override; void query_security_metadata( From 6ec7443677b006489b18706aeb0483dbf291155e Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Jan 2018 16:57:44 -0500 Subject: [PATCH 5/7] spark-335 Hide details of syntheticsecurityservice in preparation for using in mdconsumer. --- src/lib/md/ISyntheticSecurityService.h | 19 ++++ src/lib/md/SyntheticSecurityService.cpp | 37 ++++---- src/lib/md/SyntheticSecurityService.h | 16 ++-- src/lib/utils/Event.h | 112 ++++++++++++++++++++++++ src/lib/utils/LateJoinEvent.h | 58 ------------ 5 files changed, 159 insertions(+), 83 deletions(-) create mode 100644 src/lib/md/ISyntheticSecurityService.h create mode 100644 src/lib/utils/Event.h delete mode 100644 src/lib/utils/LateJoinEvent.h diff --git a/src/lib/md/ISyntheticSecurityService.h b/src/lib/md/ISyntheticSecurityService.h new file mode 100644 index 000000000..af6e63a53 --- /dev/null +++ b/src/lib/md/ISyntheticSecurityService.h @@ -0,0 +1,19 @@ +#pragma once + +#include "lib/api/services/SyntheticSecurityService.pb.h" +#include "lib/domain_model/SecurityKey.h" +#include "lib/utils/Event.h" +#include + +namespace bts { +namespace api { +namespace synthetic { + +struct ISyntheticSecurityService { + virtual const std::unordered_map securities() = 0; + virtual utils::EventSubscriber& on_securities() = 0; + virtual const Metadata* get_metadata(SecurityKey) = 0; + virtual utils::EventSubscriber& on_metadata() = 0; +}; + +}}} \ No newline at end of file diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp index f0964d6ee..98e553fee 100644 --- a/src/lib/md/SyntheticSecurityService.cpp +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -9,6 +9,7 @@ #include "lib/md/SecurityDB.h" #include "lib/md/get_symbols.h" #include "lib/utils/Dispatcher.h" +#include "lib/utils/Event.h" #include namespace bts { @@ -25,17 +26,15 @@ struct SyntheticSecurityService::Pimpl { Pimpl() : recorder(), timer(utils::Dispatcher::background(), 200, [&]() { recorder.do_work(); }), - known_synthetics(), - on_security(rxcpp::identity_current_thread(), rxcpp::composite_subscription()), - meta(), - on_metadata() + synthetics(), on_security(), + meta(), on_metadata() {} md::SecurityIDRecorder recorder; utils::DispatcherTimer timer; - std::set known_synthetics; - rxcpp::subjects::replay on_security; + std::unordered_map synthetics; + utils::Event on_security; std::unordered_map meta; - rxcpp::subjects::subject on_metadata; + utils::Event on_metadata; }; SyntheticSecurityService::SyntheticSecurityService(api::ServiceContext& context) @@ -45,11 +44,10 @@ SyntheticSecurityService::SyntheticSecurityService(api::ServiceContext& context) // load securities std::set symbols; md::get_system_md_symbols(symbols); - auto o = _me->on_security.get_subscriber(); md::db::load_securities(symbols, [=](const api::Security& security) { if (security.Contract_case() == api::Security::kSynthetic) { - o.on_next(security); - _me->known_synthetics.insert(SecurityKey(security.id())); + auto key = SecurityKey(security.id()); + _me->synthetics.insert(std::make_pair(key, security)); } }); @@ -62,8 +60,12 @@ SyntheticSecurityService::SyntheticSecurityService(api::ServiceContext& context) } SyntheticSecurityService::~SyntheticSecurityService() { } -rxcpp::observable SyntheticSecurityService::on_security() { - return _me->on_security.get_observable(); + +const std::unordered_map SyntheticSecurityService::securities() { + return _me->synthetics; +} +utils::EventSubscriber& SyntheticSecurityService::on_securities() { + return _me->on_security; } const Metadata* SyntheticSecurityService::get_metadata(SecurityKey key) { auto iter = _me->meta.find(key); @@ -72,8 +74,8 @@ const Metadata* SyntheticSecurityService::get_metadata(SecurityKey key) { } return nullptr; } -rxcpp::observable SyntheticSecurityService::on_metadata() { - return _me->on_metadata.get_observable(); +utils::EventSubscriber& SyntheticSecurityService::on_metadata() { + return _me->on_metadata; } void send_reply(const api::Response& fn, CreateResult::Error err, uint32_t key=md::invalid_security_key) { @@ -101,8 +103,8 @@ void SyntheticSecurityService::create_synthetic( // Assign security key _me->recorder.assign_key(security, [=](const api::Security& security) { // Notify listeners that a new security is added - _me->known_synthetics.insert(SecurityKey(security.id())); - _me->on_security.get_subscriber().on_next(security); + _me->synthetics.insert(std::make_pair(security.id(), security)); + _me->on_security.publish(security); // signal success send_reply(reply, CreateResult::ERROR_OK, security.id()); }); @@ -118,7 +120,7 @@ void SyntheticSecurityService::update_security_metadata( return; } auto key = SecurityKey(request.security_key()); - if (_me->known_synthetics.find(key) == _me->known_synthetics.end()) { + if (_me->synthetics.find(key) == _me->synthetics.end()) { result.set_error(UpdateResult::ERROR_UNKNOWN_SECURITY); reply(result); return; @@ -132,6 +134,7 @@ void SyntheticSecurityService::update_security_metadata( // update the local database _me->meta[key] = request; // notify subscribers + _me->on_metadata.publish(request); reply(result); } diff --git a/src/lib/md/SyntheticSecurityService.h b/src/lib/md/SyntheticSecurityService.h index 2186b3ea0..0f07c0080 100644 --- a/src/lib/md/SyntheticSecurityService.h +++ b/src/lib/md/SyntheticSecurityService.h @@ -3,9 +3,9 @@ #include "lib/api/services/SyntheticSecurityService.pb.h" #include "lib/api/services/SyntheticSecurityService.server.pb.h" #include "lib/domain_model/SecurityKey.h" +#include "ISyntheticSecurityService.h" #include #include -#include namespace bts { @@ -13,7 +13,9 @@ namespace api { namespace synthetic { -class SyntheticSecurityService : private api::synthetic::SyntheticSecurityServiceBase +class SyntheticSecurityService : + private api::synthetic::SyntheticSecurityServiceBase, + public ISyntheticSecurityService { struct Pimpl; std::unique_ptr _me; @@ -22,12 +24,10 @@ class SyntheticSecurityService : private api::synthetic::SyntheticSecurityServic SyntheticSecurityService(api::ServiceContext& context); ~SyntheticSecurityService(); - // on_security - rxcpp::observable on_security(); - - // query/subscribe metadata - const Metadata* get_metadata(SecurityKey); - rxcpp::observable on_metadata(); + const std::unordered_map securities() override; + utils::EventSubscriber& on_securities() override; + const Metadata* get_metadata(SecurityKey) override; + utils::EventSubscriber& on_metadata() override; private: void create_synthetic( diff --git a/src/lib/utils/Event.h b/src/lib/utils/Event.h new file mode 100644 index 000000000..03d2fe9c0 --- /dev/null +++ b/src/lib/utils/Event.h @@ -0,0 +1,112 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace bts { +namespace utils { + +template +struct EventSubscriber { + typedef int Handle; + typedef std::function Callback; + + EventSubscriber() : _id(0), _subscribers(), _lock() {} + + Handle subscribe(const Callback& cb) { + std::lock_guard l(_lock); + auto id = ++_id; + _subscribers.push_back(std::make_tuple(id, cb)); + return id; + } + void unsubscribe(Handle h) { + std::lock_guard l(_lock); + _subscribers.erase( + std::remove_if(_subscribers.begin(), + _subscribers.end(), + [h](std::tuple& t) { return std::get(t) == h; })); + } +protected: + void broadcast(const T& item) { + std::lock_guard l(_lock); + for (const auto& t : _subscribers) { + std::get(t)(item); + } + } +private: + Handle _id; + std::list> _subscribers; + std::mutex _lock; + +}; + +template +struct EventPublisher { + typedef int Handle; + virtual void publish(const T&) = 0; +}; + + +template +class Event : + public EventSubscriber, + public EventPublisher { + +public: + void publish(const T& item) { + this->broadcast(item); + } +}; + + +// template +// class ReplayEvent { +// typedef std::function Callback; +// typedef int Handle; +// Handle _id; +// std::vector _items; +// std::list> _subscribers; + +// void broadcast(const T& item) { +// for(const auto& t : _subscribers) { +// std::get(t)(item); +// } +// } + +// public: +// ReplayEvent() : +// _id(0), +// _items(), +// _subscribers() {} +// ~ReplayEvent() {} + +// void publish(T&& item) { +// _items.push_back(move(item)); +// broadcast(_items.back()); +// } +// void publish(const T& item) { +// _items.push_back(item); +// broadcast(item); +// } + +// Handle subscribe(Callback& cb) { +// auto id = ++_id; +// _subscribers.push_back(std::make_tuple(id, cb)); +// for (const auto& item : _items) { +// cb(item); +// } +// return id; +// } +// void unsubscribe(Handle h) { +// _subscribers.erase( +// std::remove_if(_subscribers.begin(), +// _subscribers.end(), +// [h](std::tuple& t) { return std::get(t) == h; })); +// } +// }; + + +}}//namespace \ No newline at end of file diff --git a/src/lib/utils/LateJoinEvent.h b/src/lib/utils/LateJoinEvent.h deleted file mode 100644 index 85d7f4faf..000000000 --- a/src/lib/utils/LateJoinEvent.h +++ /dev/null @@ -1,58 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace bts { -namespace utils { - -template -class LateJoinEvent { - typedef std::function Callback; - typedef int Handle; - Handle _id; - std::vector _items; - std::list> _subscribers; - - void broadcast(const T& item) { - for(const auto& t : _subscribers) { - std::get(t)(item); - } - } - - public: - LateJoinEvent() : - _id(0), - _items(), - _subscribers() {} - ~LateJoinEvent() {} - - void publish(T&& item) { - _items.push_back(move(item)); - broadcast(_items.back()); - } - void publish(const T& item) { - _items.push_back(item); - broadcast(item); - } - - Handle subscribe(Callback& cb) { - auto id = ++_id; - _subscribers.push_back(std::make_tuple(id, cb)); - for (const auto& item : _items) { - cb(item); - } - return id; - } - void unsubscribe(Handle h) { - _subscribers.erase( - std::remove_if(_subscribers.begin(), - _subscribers.end(), - [h](std::tuple& t) { return std::get(t) == h; })); - } -}; - - -}}//namespace \ No newline at end of file From 013f8ed1843c9dc8ca02682658f9327480f793f8 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Jan 2018 16:58:11 -0500 Subject: [PATCH 6/7] run SyntheticSecurityService in main --- src/server/sparkd/cmds/Run.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 89884d146..79040893d 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -15,6 +15,7 @@ #include "server/sparkd/md/run_md.h" #include "server/sparkd/exchange_sim/ExchangeSim.h" #include "server/sparkd/orderman2/server/OrderServiceServer2.h" +#include "lib/md/SyntheticSecurityService.h" #include "lib/md/relay/QueryServer.h" @@ -37,6 +38,7 @@ int Run::execute(std::vector const &) user::UserServiceServer login_server(sc); account::AccountServiceServer account_server(sc); md::run_market_datapublishers(); + api::synthetic::SyntheticSecurityService synthetic_service(sc); md::QueryServerPtr md_query_server(md::run_query_server(sc)); orderman2::OrderServiceServer2 order_service_server(sc); From 7cfc556ea2bed1fa6fa98d8a6d97965ca11d429f Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 10 Jan 2018 17:34:52 -0500 Subject: [PATCH 7/7] Fix typo in SQL --- src/lib/md/SecurityDB.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index ff377fe5d..11742b376 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -38,10 +38,10 @@ sqlite3_ptr open_db() { sqlite3_exec_nofail(db, "PRAGMA syncronous=0"); sqlite3_exec_nofail(db, - "CREATE TABLE IF NOT EXIST securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL"); + "CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);"); sqlite3_exec_nofail(db, - "CREATE TABLE IF NOT EXIST metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); + "CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); return dbptr; }