From 4b3cdf9ca4b640e07fed52d9b4f37182dec18e01 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 14 Nov 2018 19:58:31 -0500 Subject: [PATCH] Add tests for SecurityIDRecorder += SpreadResolver --- src/lib/md/SecurityIDRecorder.cpp | 44 ++++++----- src/lib/md/SecurityIDRecorder.h | 18 +++-- src/lib/md/Shm/FixedSecurity.cpp | 14 +++- src/lib/md/Shm/FixedSecurity.h | 2 + src/lib/md/SyntheticSecurityService.cpp | 2 +- src/lib/md/relay/RelayRecorder.cpp | 2 +- src/lib/md/test/md2_tests.cpp | 78 +++++++++++++++++-- .../sparkd/md/platforms/celoxica/CXMain.cpp | 2 +- 8 files changed, 127 insertions(+), 35 deletions(-) diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 192ca68e2..d98fc948e 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -7,7 +7,7 @@ #include "lib/logging/logging.h" #include "lib/md/SecurityDB.h" #include -#include // for reservation +#include #include "lib/config/protos/MarketData.pb.h" #include "lib/config/protos/system.pb.h" @@ -22,6 +22,8 @@ using std::size_t; namespace bts { namespace md { +uint64_t exchange_id(const api::Security& security); + struct SecurityIDEntry { api::Security security; SecurityIDRecorder::Callback callback; @@ -58,17 +60,16 @@ struct SecurityIDRecorderState { security_key insert(const std::string& descriptor, SecurityType); }; -SecurityIDRecorder::SecurityIDRecorder(): +SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params): _self(new SecurityIDRecorderState()) { + if (params.db) { + _self->db = std::move(params.db); + } init(); -} - -SecurityIDRecorder::SecurityIDRecorder(utils::Dispatcher& dispatcher): - _self(new SecurityIDRecorderState()) -{ - init(); - _self->run(dispatcher); + if (params.dispatcher) { + _self->run(*params.dispatcher); + } } SecurityIDRecorder::~SecurityIDRecorder() { } @@ -81,13 +82,16 @@ void SecurityIDRecorder::init() { boost::filesystem::create_directories(db_dir); } - _self->db = open_db(db_path.c_str()); + if (!_self->db) { + _self->db = sqlite::open_db(db_path.c_str()); + bts_assert(_self->db); + } // Set a long busy timeout sqlite3_busy_timeout(_self->db.get(), 9999); - sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); - sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0"); - sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);"); + sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); + sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0"); + sqlite::exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);"); } void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { @@ -196,7 +200,7 @@ void SecurityIDRecorderState::work_db() { if (!new_entries.empty()) { - sqlite3_exec_nofail(db.get(), "BEGIN TRANSACTION"); + sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION"); for (size_t x = 0; x < new_entries.size(); ++x) { Security& security = new_entries[x]->security; @@ -205,7 +209,7 @@ void SecurityIDRecorderState::work_db() { security.set_id(key); } - sqlite3_exec_nofail(db.get(), "COMMIT TRANSACTION"); + sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } // record the securities @@ -231,10 +235,10 @@ void SecurityIDRecorderState::work_db() { void SecurityIDRecorder::bulk_insert(const std::vector& data, const char* prefix) { if (data.size() == 0) return; - sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); + sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); std::string descriptor; { - auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)"); + auto insert_stmt = sqlite::prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)"); for (const api::Security* sec : data) { descriptor = prefix + security_descriptor(*sec); auto type = securitytype_of(*sec); @@ -246,11 +250,11 @@ void SecurityIDRecorder::bulk_insert(const std::vector& da sqlite3_reset(insert_stmt.get()); } } - sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); } security_key SecurityIDRecorderState::search(const std::string& descriptor) { - auto select_stmt = prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); + auto select_stmt = sqlite::prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); sqlite3_bind_text(select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); int result = sqlite3_step(select_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { @@ -260,7 +264,7 @@ security_key SecurityIDRecorderState::search(const std::string& descriptor) { } security_key SecurityIDRecorderState::insert(const std::string& descriptor, SecurityType type) { - auto insert_stmt = prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); + auto insert_stmt = sqlite::prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); sqlite3_bind_text(insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); sqlite3_bind_int(insert_stmt.get(), 2, (int)type); int result = sqlite3_step(insert_stmt.get()); diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index f63731ff1..659f5a74a 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -3,11 +3,13 @@ #include "common.h" #include "MDProducerHandle.h" +#include "lib/utils/Sqlite_utils.h" #include #include #include #include -#include + +typedef struct sqlite3 sqlite3; namespace bts { namespace md { @@ -17,13 +19,19 @@ namespace md { * Provides a callback when ids are safely persisted. */ +struct SecurityIDRecorderParams { + SecurityIDRecorderParams() : dispatcher(), db(sqlite::null_db()) {} + SecurityIDRecorderParams(utils::Dispatcher& d): dispatcher(&d), db(sqlite::null_db()) {} + SecurityIDRecorderParams(sqlite3_ptr& db): dispatcher(), db(std::move(db)) {} + utils::Dispatcher* dispatcher; + sqlite3_ptr db; +}; struct SecurityIDRecorderState; + class SecurityIDRecorder { public: - SecurityIDRecorder(); - // Automatically call do_work on this dispatcher - SecurityIDRecorder(utils::Dispatcher&); + SecurityIDRecorder(SecurityIDRecorderParams); ~SecurityIDRecorder(); typedef std::function Callback; @@ -32,7 +40,7 @@ public: // Slow (copies security) - for use by services void assign_key(const api::Security& security, const Callback& key_assigned); - // Someone must call work periodically to do lookups and flush the reserved ids to disk. + // If a dispatcher is not provided you must call work periodically to do lookups and flush the reserved ids to disk. void do_work(); void bulk_insert(const std::vector&, const char* prefix); diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index cc8d304ea..a0405bdfe 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -250,6 +250,16 @@ size_t num_legs(const Security& sec) { } return 0; } +uint64_t exchange_id(const Security& security) { + switch(security.ExchangeData_case()) { + case Security::ExchangeDataCase::kCboe: return security.cboe().exchange_id(); + case Security::ExchangeDataCase::kGlbx: return security.glbx().exchange_id(); + case Security::ExchangeDataCase::kCfe: return security.cfe().exchange_id(); + case Security::ExchangeDataCase::kIce: bts_assert(false); + case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: bts_assert(false); + } + return -1; +} void convert_security(const Security& src, FixedSecurity& dest, bool set_security_key) { dest.key = set_security_key ? src.id() : invalid_security_key; @@ -272,7 +282,9 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit case Security::ExchangeDataCase::kCfe: dest.cboe_class_key = src.cfe().class_key(); break; - default: + case Security::ExchangeDataCase::kIce: + bts_assert_d(false); + case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: break; } dest.price_break = src.contract_spec().price_break() / src.contract_spec().price_multiplier(); diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index b8fd82d55..80d44b287 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -180,6 +180,8 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit SecurityType securitytype_of_contractcase(Security::ContractCase cc); SecurityType securitytype_of(const Security&); size_t num_legs(const Security&); +uint64_t exchange_id(const Security& security); + // Stack allocates a security of the given type template diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp index 8d6ca60aa..eeea9168b 100644 --- a/src/lib/md/SyntheticSecurityService.cpp +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -25,7 +25,7 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher MetadataPublishe struct SyntheticSecurityService::Pimpl { Pimpl(const MetadataStatefulPublisher& fn) : - recorder(utils::Dispatcher::background()), + recorder({utils::Dispatcher::background()}), synthetics(), on_security(), meta(), on_metadata(), metadata_publisher(fn) { diff --git a/src/lib/md/relay/RelayRecorder.cpp b/src/lib/md/relay/RelayRecorder.cpp index 55ca64edb..a6f453e73 100644 --- a/src/lib/md/relay/RelayRecorder.cpp +++ b/src/lib/md/relay/RelayRecorder.cpp @@ -130,7 +130,7 @@ void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) { CoconutReplayHeader header; size_t count = 0; #ifndef _WIN32 - md::SecurityIDRecorder idrec; + md::SecurityIDRecorder idrec {{}}; #endif if (readbuf(in, &header) && header.magic() == magic && header.version() == 3) { co2::MDReply message; diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 9a3301311..ba81b8413 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -13,6 +13,7 @@ #include "lib/metrics/metrics.h" #include "lib/config/protos/MarketData.pb.h" +#include "lib/utils/Sqlite_utils.h" #include #include @@ -27,7 +28,6 @@ #include #include - using namespace std; namespace bts { @@ -151,6 +151,7 @@ void assign_ids(vector::iterator begin, vector(); + auto hot_header = std::make_shared(); + + // Create a spread with 2 legs + auto f1 = mkfuture("FO1", invalid_security_key, 2018, 1, 17); + f1.mutable_glbx()->set_exchange_id(100); + REQUIRE(f1.id() == invalid_security_key); + auto f1handle = std::make_unique (f1, cold_header, hot_header); + + auto f2 = mkfuture("FO2", invalid_security_key, 2018, 1, 20); + f2.mutable_glbx()->set_exchange_id(200); + auto f2handle = std::make_unique (f2, cold_header, hot_header); + + auto s1 = mkspread("FOO", invalid_security_key, { f1, f2 }); + auto s1handle = std::make_unique (s1, cold_header, hot_header); + + // Add spread to SecurityIDRecorder and work it + auto db = sqlite::open_db(":memory:"); + REQUIRE(db); + SecurityIDRecorder idsvc{{db}}; + auto workwork = [&]() { idsvc.do_work(); idsvc.do_work(); }; + // auto workwork = [&]() { for(int i=5;--i;) { idsvc.do_work(); } }; + idsvc.ensure_key(s1handle->producer()); + workwork(); + + // Expect no key + REQUIRE(s1handle->consumer().key() == invalid_security_key); + + // Add leg spreads + idsvc.ensure_key(f1handle->producer()); + idsvc.ensure_key(f2handle->producer()); + workwork(); + + // Expect legs to have keys + REQUIRE(f1handle->consumer().key() != invalid_security_key); + REQUIRE(f2handle->consumer().key() != invalid_security_key); + + // Expect spread to have keys + REQUIRE(s1handle->consumer().key() != invalid_security_key); } #endif @@ -1002,7 +1064,11 @@ TEST_CASE("CO2Tests","[md]") { #ifndef WIN32 SECTION("test_security_id_recorder") { - co2_tests->test_security_id_recorder(); + co2_tests->test_security_id_recorder(":memory:"); + co2_tests->test_security_id_recorder("file::memory:?cache=shared"); + } + SECTION("test_security_id_recorder_spreads") { + co2_tests->test_security_id_recorder_spreads(); } #endif SECTION("test_price_size") { diff --git a/src/server/sparkd/md/platforms/celoxica/CXMain.cpp b/src/server/sparkd/md/platforms/celoxica/CXMain.cpp index bfb96e727..1c39dcd72 100644 --- a/src/server/sparkd/md/platforms/celoxica/CXMain.cpp +++ b/src/server/sparkd/md/platforms/celoxica/CXMain.cpp @@ -43,7 +43,7 @@ void run_cx_server(UserSymbol* psyms, size_t symcount) CXMain::CXMain() : _producer(co2::PLATFORM_CELOXICA), - _security_recorder(), + _security_recorder({}), _settlement_recorder(co2::PLATFORM_CELOXICA), _listeners() {