From bd8c97244a23ae0714ca5d3220820f00a94a50be Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 10 Oct 2024 15:55:32 +0000 Subject: [PATCH] refactor: Return row id in IStore::retrieve_one This is a prerequisite to synchronizing security ids stored via IDGenerator (the "new" way to store ids, added when we implemented Deribit), and SecurityDB (the original way from spark days). The problem was that these 2 implementations were using separate databases. We need to unify the interfaces between these storage services to avoid breaking existing code. --- src/lib/core/TimeSpan.h | 10 + src/lib/md/Diag/ShmDumper.cpp | 77 +--- src/lib/md/SecurityDB.cpp | 22 -- src/lib/md/SecurityIDRecorder.cpp | 189 ++++------ src/lib/md/SecurityIDRecorder.h | 23 +- src/lib/md/Shm/FixedSecurity.cpp | 24 +- src/lib/md/Shm/FixedSecurity.h | 7 +- src/lib/md/SyntheticSecurityService.cpp | 5 +- src/lib/md/cme/mdp3_data_directory.h | 32 ++ src/lib/md/test/md2_tests.cpp | 29 +- src/lib/os/spark_data_directory.cpp | 45 +++ src/lib/os/spark_data_directory.h | 28 ++ src/lib/persistence/IDGenerator.cpp | 20 +- src/lib/persistence/IStore.h | 39 +- src/lib/persistence/SqliteStore.cpp | 342 ++++++++++++------ src/lib/persistence/SqliteStore.h | 13 +- src/lib/persistence/store_row.h | 20 + src/lib/utils/checked_arithmetic.h | 37 ++ .../analytics/AnalyticsServiceServer.cpp | 8 +- .../exchange_sim/ExchangeSimApplication.cpp | 5 +- .../sparkd/md/emumba/EmumbaDeribitData.h | 9 +- .../sparkd/md/platforms/emumba/EmumbaMain.cpp | 9 +- .../sparkd/md/platforms/ice_onixs/ICEMain.cpp | 135 ++++--- .../sparkd/md/platforms/ice_onixs/ICEMain.h | 36 +- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 78 ++-- .../sparkd/md/platforms/mdp3_onixs/MDP3Main.h | 20 +- .../sparkd/position/PositionServiceServer.cpp | 17 +- src/server/sparkd/user/AdminServiceDB.cpp | 6 +- 28 files changed, 719 insertions(+), 566 deletions(-) create mode 100644 src/lib/md/cme/mdp3_data_directory.h create mode 100644 src/lib/os/spark_data_directory.cpp create mode 100644 src/lib/os/spark_data_directory.h create mode 100644 src/lib/persistence/store_row.h create mode 100644 src/lib/utils/checked_arithmetic.h diff --git a/src/lib/core/TimeSpan.h b/src/lib/core/TimeSpan.h index 23970e616..5e0d14458 100644 --- a/src/lib/core/TimeSpan.h +++ b/src/lib/core/TimeSpan.h @@ -50,7 +50,9 @@ class TimeSpan { constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept; +constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept; +constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept; std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept; @@ -116,10 +118,18 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept { return lhs.nanos() < rhs.nanos(); } +constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept { + return lhs.nanos() <= rhs.nanos(); +} + constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept { return lhs.nanos() > rhs.nanos(); } +constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept { + return lhs.nanos() >= rhs.nanos(); +} + } // namespace bts::core FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string()); diff --git a/src/lib/md/Diag/ShmDumper.cpp b/src/lib/md/Diag/ShmDumper.cpp index 4dbb10599..dc04f67f7 100644 --- a/src/lib/md/Diag/ShmDumper.cpp +++ b/src/lib/md/Diag/ShmDumper.cpp @@ -53,82 +53,7 @@ string dump_header(const ColdHeader &header) { } string ShmDumper::dump_security(const FixedSecurity &security) { - ostringstream res; - res << "{"; - res << "'key': " << security.key << ", "; - res << "'symbol': '" << security.symbol() << "', "; - res << "'type': '" << api::SecurityType_Name(security.type) << "', "; - res << "'exchange': '" << api::Venue_Name(security.exchange) << "', "; - res << "'alt_symbol': '" << security.alt_symbol() << "', "; - res << "'cfi_code': '" << security.cfi_code() << "', "; - res << "'security_desc': '" << security.security_desc() << "', "; - res << "'platform_id': '" << security.platform_id() << "', "; - res << "'display_factor': " << (int)security.display_factor << ", "; - res << "'numeric_id': " << security.numeric_id << ", "; - res << "'cboe_class_key': " << security.cboe_class_key << ", "; - res << "'main_frac': " << (int)security.cme_main_fraction << ", "; - res << "'sub_fract': " << (int)security.cme_sub_fraction << ", "; - res << "'price_format': " << (int)security.cme_price_display_format << ", "; - res << "'market_segment': " << (int)security.cme_market_segment_id << ", "; - - res << "'price_break': " << security.price_break << ", "; - res << "'low_tick': " << security.low_tick << ", "; - res << "'high_tick': " << security.high_tick << ", "; - res << "'cpp': " << security.cash_per_point << ", "; - - res << "'market_description ': " << security.ice_market_description() - << ", "; - res << "'cleared_alias ': " << security.ice_cleared_alias() << ", "; - res << "'product_description ': " << security.ice_product_name() << ", "; - res << "'hub_alias ': " << security.ice_hub_alias() << ", "; - res << "'strip_name': " << security.ice_strip_name() << ", "; - res << "'mic_code': " << security.mic_code() << ", "; - - switch (security.type) { - case SECURITYTYPE_SPOT: - break; - case SECURITYTYPE_FUTURE: { - auto *psec = static_cast(&security); - res << "'expiration': '"; - securities::write_date_string( - psec->expiration_year, psec->expiration_month, psec->expiration_day, - res); - res << "' "; - break; - } - case SECURITYTYPE_OPTION: { - auto *psec = static_cast(&security); - res << "'expiration': '"; - securities::write_date_string( - psec->expiration_year, psec->expiration_month, psec->expiration_day, - res); - res << "', "; - res << "'strike': " << psec->strike << ", "; - res << "'callput': '" << api::PutOrCall_Name(psec->call_put) << "' "; - break; - } - case SECURITYTYPE_SPREAD: { - auto *psec = static_cast(&security); - res << "'legs': ["; - for (size_t x = 0; x < psec->leg_count(); ++x) { - auto &leg = psec->legs[x]; - if (x > 0) - res << ","; - res << "{ 'key': " << leg.key << ", 'exkey': " << leg.exchange_key - << ", 'quantity': " << leg.leg_quantity - << ", 'leg_price': " << leg.leg_price - << ", 'delta': " << leg.leg_delta << ", 'side': '" - << api::Side_Name(leg.leg_side) << "' }"; - } - res << "]"; - break; - } - default: - break; - } - - res << "}"; - return res.str(); + return JsonDumper::dump(security); } string dump(datetime::Date date) { diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index 27b84e073..c33e5814d 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -61,26 +61,6 @@ datetime::Date sec_expiration_cut_off() { return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day()); } -void apply_compatibilit_mutations(Security &security) { - // Migrate from old tick stuff - if (security.has_glbx()) { - if (security.has_glbx()) { - if (security.contract_spec().main_fraction() == 0) { - security.mutable_contract_spec()->set_main_fraction( - security.glbx().do_not_use_main_fraction()); - } - if (security.contract_spec().sub_fraction() == 0) { - security.mutable_contract_spec()->set_sub_fraction( - security.glbx().do_not_use_sub_fraction()); - } - if (security.contract_spec().price_display_format() == 0) { - security.mutable_contract_spec()->set_price_display_format( - security.glbx().do_not_use_price_display_format()); - } - } - } -} - size_t load_securities( std::function const &sym_filter, const std::function &callback) { @@ -104,7 +84,6 @@ size_t load_securities( if (sym_filter(us)) { auto blob = select_stmt.getColumn("security_blob"); if (security.ParseFromArray(blob, blob.getBytes())) { - apply_compatibilit_mutations(security); bts_assert(security_key == security.id()); count++; callback(security); @@ -146,7 +125,6 @@ SecurityPtr find_security(SecurityKey key) { auto result = std::make_shared(); auto blob = select.getColumn(0); if (result->ParseFromArray(blob, blob.getBytes())) { - apply_compatibilit_mutations(*result); return result; } } diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 3313a037f..f0abccdf8 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -1,19 +1,25 @@ #ifndef _WIN32 #include "SecurityIDRecorder.h" #include "lib/domain_model/Security.h" +#include "lib/persistence/IStore.h" +#include "lib/persistence/SqliteStore.h" #include #include "lib/config/config.h" #include "lib/logging/logging.h" #include "lib/utils/Sqlite_utils.h" +#include "lib/utils/checked_arithmetic.h" #include #include #include "lib/config/protos/market_data.pb.h" #include "lib/config/protos/system_config.pb.h" +#include #include +#include #include +#include using namespace std; using namespace bts::utils; @@ -26,9 +32,32 @@ using bts::api::Security; using bts::api::SecurityType; using bts::api::Venue; using bts::config::SystemConfig; +using bts::persistence::IStore; +using bts::persistence::SqliteStore; uint64_t exchange_id(const api::Security &security); +static std::unique_ptr get_default_store() { + return std::make_unique("key_ids"); +} + +static persistence::StoreRow get_row( + Venue venue, + SecurityType type, + const std::string &name, + const std::string &prefix = "") { + return persistence::StoreRow{ + fmt::format( + "{}{}.{}", std::string{prefix}, Venue_Name(venue).substr(6), + SecurityType_Name(type).substr(13)), + name, ""}; +} + +static persistence::StoreRow get_row( + const api::Security &s, const std::string &prefix = "") { + return get_row(s.exchange(), bts::securities::security_type(s), prefix); +} + struct SecurityIdRequest { Venue venue; SecurityType type; @@ -38,14 +67,16 @@ struct SecurityIdRequest { }; struct SecurityIDRecorderState { - SecurityIDRecorderState(SecurityIDRecorderParams &p) - : pending_handle(), thread_id(), db(nullptr, sqlite3_close), timer(), - lookup_key(p.lookup_key), unresolved_legs() {} + SecurityIDRecorderState( + std::unique_ptr store, KeyLookupOption lookup_key) + : store(store == nullptr ? get_default_store() : std::move(store)), + pending_handle(), thread_id(), timer(), lookup_key(lookup_key), + unresolved_legs() {} + std::unique_ptr store{nullptr}; tbb::concurrent_queue> id_request_queue; tbb::concurrent_queue pending_handle; std::thread::id thread_id; - sqlite3_ptr db; unique_ptr timer; KeyLookupOption lookup_key; @@ -81,46 +112,28 @@ struct SecurityIDRecorderState { void check_access() { bts_assert(thread_id == std::this_thread::get_id()); } void add_unresolved_spread(MDProducerHandle &handle); - security_key search(const std::string &descriptor); - security_key insert(const std::string &descriptor, SecurityType); + security_key search(const persistence::StoreRow &row); + security_key insert(const persistence::StoreRow &row); }; -SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params) - : _self(new SecurityIDRecorderState(params)) { - if (params.db) { - _self->db = std::move(params.db); - } +SecurityIDRecorder::SecurityIDRecorder( + std::unique_ptr store, + KeyLookupOption lookup_key, + utils::Dispatcher *dispatcher) + : _self(std::make_unique( + store == nullptr + ? std::make_unique("key_ids") + : std::move(store), + lookup_key)) { init(); - if (params.dispatcher) { - _self->run(*params.dispatcher); + if (dispatcher != nullptr) { + _self->run(*dispatcher); } } SecurityIDRecorder::~SecurityIDRecorder() {} -void SecurityIDRecorder::init() { - LOG(md, Info, "SecurityIDRecorder::init"); - string db_dir = - config::get().data_path() + "databases/md/"; - string db_path = db_dir + "/security_keys.sqlite"; - - if (!boost::filesystem::exists(db_dir)) { - boost::filesystem::create_directories(db_dir); - } - - if (!_self->db) { - _self->db = sqlite::open_db(db_path.c_str()); - bts_assert(_self->db); - } - - // Set a long busy timeout - sqlite3_busy_timeout(_self->db.get(), 9999); - sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); - sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0"); - sqlite::exec_nofail( - _self->db, "CREATE TABLE IF NOT EXISTS store (descriptor STRING " - "NOT NULL PRIMARY KEY, type int);"); -} +void SecurityIDRecorder::init() {} void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}", @@ -134,7 +147,7 @@ void SecurityIDRecorder::assign_key( assign_key( security.exchange(), bts::securities::security_type(security), bts::securities::descriptor(security), - [&callback, msec = security]( + [callback, msec = security]( api::Venue, api::SecurityType, const std::string &, uint32_t id) mutable { msec.set_id(id); @@ -269,27 +282,25 @@ void SecurityIDRecorderState::work_db() { // SPARK-1446 Need to try to add existing securities to the db in case we // deleted them by hand vector> existing_securities; - vector> defer; - vector new_descriptors; + vector new_rows; size_t count = 0; - string descriptor; while (id_request_queue.try_pop(temp)) { - descriptor = temp->name; - - // If we're unable to create a descriptor try again later - if (descriptor == "") { - defer.push_back(std::move(temp)); - continue; + security_key key = invalid_security_key; + auto row = get_row(temp->venue, temp->type, temp->name); + try { + key = search(row); + } catch (const std::overflow_error &ex) { + LOG(persist, Error, + "SecurityIDRecorderState::work_db. Row search threw ex: {}", + std::string{ex.what()}); } - - security_key key = search(descriptor); if (key != invalid_security_key) { temp->callback(temp->venue, temp->type, temp->name, key); existing_securities.push_back(std::move(temp)); } else { // always push in parallel new_entries.push_back(std::make_pair(key, std::move(temp))); - new_descriptors.push_back(descriptor); + new_rows.push_back(std::move(row)); ++count; // Limit the amount of time we spend writing to sqlite during startup // to avoid blocking the heartbeat @@ -299,22 +310,14 @@ void SecurityIDRecorderState::work_db() { } } - for (auto &d : defer) { - LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}", - d->name); - id_request_queue.push(std::move(d)); - } - if (!new_entries.empty()) { - sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION"); - + store->insert_many_resolve_ids(new_rows); for (size_t x = 0; x < new_entries.size(); ++x) { auto &entry = new_entries[x]; - entry.first = insert(new_descriptors[x], entry.second->type); + auto &row = new_rows[x]; + entry.first = row.id; bts_assert_d(entry.first != invalid_security_key); } - - sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } if (!new_entries.empty()) { @@ -327,65 +330,27 @@ void SecurityIDRecorderState::work_db() { void SecurityIDRecorder::bulk_insert( const std::vector &data, const char *prefix) { - LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size()); - if (data.size() == 0) - return; - sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); - std::string descriptor; - { - auto insert_stmt = sqlite::prepare_query( - _self->db, "INSERT INTO security_keys (rowid, descriptor, type) " - "values (?, ?, ?)"); - for (const api::Security *sec : data) { - descriptor = prefix + bts::securities::unique_descriptor(*sec); - auto type = securitytype_of(*sec); - - sqlite3_bind_int(insert_stmt.get(), 1, sec->id()); - sqlite3_bind_text( - insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - sqlite3_bind_int(insert_stmt.get(), 3, (int)type); - sqlite3_step(insert_stmt.get()); - sqlite3_reset(insert_stmt.get()); - } + std::vector rows; + rows.reserve(data.size()); + for (const auto *s : data) { + rows.push_back(get_row(*s, prefix)); } - sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + _self->store->insert_many_resolve_ids(rows); } -security_key SecurityIDRecorderState::search(const std::string &descriptor) { - LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor); - auto select_stmt = sqlite::prepare_query( - db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); - sqlite3_bind_text( - select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - int result = sqlite3_step(select_stmt.get()); - if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_column_int(select_stmt.get(), 0); +security_key SecurityIDRecorderState::search(const persistence::StoreRow &row) { + auto [exists, rowid, val] = store->retrieve_one(row.ns, row.key); + if (exists) { + if (!val.empty()) { + bts_assert(stol(val) == rowid); + } + return system::math::checked_ltou(rowid); } return invalid_security_key; } -security_key SecurityIDRecorderState::insert( - const std::string &descriptor, SecurityType type) { - LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor); - auto insert_stmt = sqlite::prepare_query( - db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); - sqlite3_bind_text( - insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - sqlite3_bind_int(insert_stmt.get(), 2, (int)type); - int result = sqlite3_step(insert_stmt.get()); - if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_last_insert_rowid(db.get()); - } - // Just in case our insert failed due to duplicate row - auto key = search(descriptor); - if (key == invalid_security_key) { - LOG(md, Error, "Failed to assign security key. SQLite return code: {}", - result); - } - return key; +security_key SecurityIDRecorderState::insert(const persistence::StoreRow &row) { + return store->insert(row.ns, row.key, row.value); } } // namespace md diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index 862ba79b8..2925e0c20 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -2,7 +2,7 @@ #ifndef _WIN32 #include "MDProducerHandle.h" #include "common.h" - +#include "lib/persistence/IStore.h" #include "lib/utils/Sqlite_utils.h" #include #include @@ -22,26 +22,15 @@ namespace md { typedef const std::function KeyLookup; typedef std::optional KeyLookupOption; -struct SecurityIDRecorderParams { - SecurityIDRecorderParams() - : dispatcher(), db(sqlite::null_db()), lookup_key() {} - SecurityIDRecorderParams(utils::Dispatcher &d) - : dispatcher(&d), db(sqlite::null_db()), lookup_key() {} - SecurityIDRecorderParams(utils::Dispatcher &d, KeyLookup &get_key) - : dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {} - SecurityIDRecorderParams(sqlite3_ptr &db) - : dispatcher(), db(std::move(db)), lookup_key() {} - SecurityIDRecorderParams(sqlite3_ptr &db, KeyLookup &get_key) - : dispatcher(), db(std::move(db)), lookup_key(get_key) {} - utils::Dispatcher *dispatcher; - sqlite3_ptr db; - KeyLookupOption lookup_key; -}; + struct SecurityIDRecorderState; class SecurityIDRecorder { public: - SecurityIDRecorder(SecurityIDRecorderParams); + SecurityIDRecorder( + std::unique_ptr store = nullptr, + KeyLookupOption lookup_key = std::nullopt, + utils::Dispatcher *dispatcher = nullptr); ~SecurityIDRecorder(); typedef std::function Callback; diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index 252edf917..cd2ebc9ee 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -217,26 +217,12 @@ void convert_security(const FixedSecurity &source, Security *presult) { get_contract_spec(source, *dest.mutable_contract_spec()); } -SecurityType securitytype_of_contractcase(Security::ContractCase cc) { - switch (cc) { - case Security::ContractCase::kFuture: - return SECURITYTYPE_FUTURE; - case Security::ContractCase::kOption: - return SECURITYTYPE_OPTION; - case Security::ContractCase::kSpot: - return api::SECURITYTYPE_SPOT; - case Security::ContractCase::kSpread: - return SECURITYTYPE_SPREAD; - case Security::ContractCase::kSynthetic: - return SECURITYTYPE_SYNTHETIC; - case Security::ContractCase::CONTRACT_NOT_SET: - bts_assert(cc != Security::ContractCase::CONTRACT_NOT_SET); - } - bts_assert(false); -} SecurityType securitytype_of(const Security &s) { - return securitytype_of_contractcase(s.Contract_case()); + auto type = bts::securities::security_type(s.Contract_case()); + bts_assert(type != api::SecurityType::SECURITYTYPE_UNSET); + return type; } + size_t num_legs(const Security &sec) { if (sec.Contract_case() == api::Security::ContractCase::kSpread) { return sec.spread().legs_size(); @@ -284,7 +270,7 @@ uint64_t exchange_id(const Security &security) { void convert_security( const Security &src, FixedSecurity &dest, bool set_security_key) { dest.key = set_security_key ? src.id() : invalid_security_key; - dest.type = securitytype_of_contractcase(src.Contract_case()); + dest.type = bts::securities::security_type(src.Contract_case()); dest.symbol(src.symbol()); dest.currency(src.currency()); dest.exchange = src.exchange(); diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index 3afb6c79e..4b39e6c51 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -215,10 +215,9 @@ void convert_security(const FixedSecurity &, Security *); void convert_security( const Security &src, FixedSecurity &dest, bool set_security_key = false); -SecurityType securitytype_of_contractcase(Security::ContractCase cc); -SecurityType securitytype_of(const Security &); -size_t num_legs(const Security &); -uint64_t exchange_id(const Security &security); +SecurityType securitytype_of(const api::Security &); +size_t num_legs(const api::Security &); +uint64_t exchange_id(const api::Security &security); void to_json(nlohmann::json &j, const FixedSpreadLeg &f); diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp index c0301e8f4..6daca8b4c 100644 --- a/src/lib/md/SyntheticSecurityService.cpp +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -35,8 +35,9 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher struct SyntheticSecurityService::Pimpl { Pimpl(const MetadataStatefulPublisher &fn) - : recorder(utils::Dispatcher::background()), synthetics(), on_security(), - meta(), on_metadata(), metadata_publisher(fn) {} + : recorder(nullptr, std::nullopt, &utils::Dispatcher::background()), + synthetics(), on_security(), meta(), on_metadata(), + metadata_publisher(fn) {} SecurityIDRecorder recorder; std::unordered_map synthetics; utils::Event on_security; diff --git a/src/lib/md/cme/mdp3_data_directory.h b/src/lib/md/cme/mdp3_data_directory.h new file mode 100644 index 000000000..740d46dbb --- /dev/null +++ b/src/lib/md/cme/mdp3_data_directory.h @@ -0,0 +1,32 @@ +#pragma once + +#include "lib/os/spark_data_directory.h" + +#include +#include + +namespace bts::md { + +struct Mdp3DataDirectory { + boost::filesystem::path log_dir() const { return root.log_dir() / "onixs"; } + + boost::filesystem::path data_dir() const { return root.root() / "cme_mdp3"; } + + boost::filesystem::path config_file() const { + return data_dir() / "config.xml"; + } + + boost::filesystem::path templates_file() const { + return data_dir() / "templates.xml"; + } + + void create_dirs() { + boost::filesystem::create_directories(data_dir()); + boost::filesystem::create_directories(log_dir()); + } + + private: + spark::SparkDataDirectory root{}; +}; + +} // namespace bts::md diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 13cdbb323..f4cff0acc 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -9,6 +9,7 @@ #include "lib/config/config.h" #include "lib/metrics/metrics.h" #include "lib/os/os.h" +#include "lib/persistence/SqliteStore.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/Shuffle.h" #include "lib/utils/bts_chrono.h" @@ -160,12 +161,10 @@ class Co2Tests { #ifndef _WIN32 void test_security_id_recorder(const std::string &db_name) { // Verify callbacks happen after work complete - auto db1 = sqlite::open_db(db_name.c_str()); - REQUIRE(db1); - auto db2 = sqlite::open_db(db_name.c_str()); - REQUIRE(db2); - - SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}}; + SecurityIDRecorder idsvc{ + std::make_unique(db_name)}; + SecurityIDRecorder idsvc2{ + std::make_unique(db_name)}; security_key result = invalid_security_key; Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850); Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20); @@ -215,18 +214,18 @@ class Co2Tests { std::make_unique(s1, cold_header, hot_header); // Add spread to SecurityIDRecorder and work it - auto db = sqlite::open_db(":memory:"); + auto db = std::make_unique(":memory:"); REQUIRE(db); SecurityIDRecorder idsvc{ - {db, [&](ExchangeKey k) -> SecurityKey { - for (const auto &fut : - {f1handle->producer(), f2handle->producer()}) { - if (k.value() == fut.security().numeric_id) - return SecurityKey(fut.key()); - } - return SecurityKey(invalid_security_key); - }}}; + std::move(db), [&](ExchangeKey k) -> SecurityKey { + for (const auto &fut : + {f1handle->producer(), f2handle->producer()}) { + if (k.value() == fut.security().numeric_id) + return SecurityKey(fut.key()); + } + return SecurityKey(invalid_security_key); + }}; auto workwork = [&]() { idsvc.do_work(); diff --git a/src/lib/os/spark_data_directory.cpp b/src/lib/os/spark_data_directory.cpp new file mode 100644 index 000000000..fea551a9e --- /dev/null +++ b/src/lib/os/spark_data_directory.cpp @@ -0,0 +1,45 @@ +#include "spark_data_directory.h" +#include "lib/config/config.h" +#include "lib/config/protos/system_config.pb.h" + +#include +#include +#include + +namespace bts::spark { + +SparkDataDirectory::SparkDataDirectory() + : SparkDataDirectory(config::get().data_path()) {} + +SparkDataDirectory::SparkDataDirectory(const std::string &root) : root_(root) {} + +boost::filesystem::path SparkDataDirectory::root() const { return root_; } + +boost::filesystem::path SparkDataDirectory::log_dir() const { + return root_ / "logging"; +} + +boost::filesystem::path SparkDataDirectory::log_file() const { + return log_dir() / "log"; +} + +boost::filesystem::path SparkDataDirectory::log_cache_file() const { + return log_dir() / "log_cache"; +} + +boost::filesystem::path SparkDataDirectory::database_dir( + const std::string &name) const { + return root_ / "databases" / (name + "/"); +} + +boost::filesystem::path SparkDataDirectory::database_file( + const std::string &name) const { + return root_ / "databases" / name / (name + ".sqlite"); +} + +boost::filesystem::path SparkDataDirectory::database_file( + const std::string &dir_name, const std::string &db_name) const { + return root_ / "databases" / dir_name / (db_name + ".sqlite"); +} + +} // namespace bts::spark diff --git a/src/lib/os/spark_data_directory.h b/src/lib/os/spark_data_directory.h new file mode 100644 index 000000000..e33248f85 --- /dev/null +++ b/src/lib/os/spark_data_directory.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +namespace bts::spark { + +class SparkDataDirectory { + public: + SparkDataDirectory(); + + explicit SparkDataDirectory(const std::string &root); + + boost::filesystem::path root() const; + boost::filesystem::path log_dir() const; + boost::filesystem::path log_file() const; + boost::filesystem::path log_cache_file() const; + + boost::filesystem::path database_dir(const std::string &name) const; + boost::filesystem::path database_file(const std::string &name) const; + boost::filesystem::path database_file( + const std::string &dir_name, const std::string &db_name) const; + + private: + boost::filesystem::path root_; +}; + +} // namespace bts::spark diff --git a/src/lib/persistence/IDGenerator.cpp b/src/lib/persistence/IDGenerator.cpp index 2ee855258..7cf38724e 100644 --- a/src/lib/persistence/IDGenerator.cpp +++ b/src/lib/persistence/IDGenerator.cpp @@ -60,24 +60,21 @@ std::optional IDGenerator::get_id( bool add_if_new) { std::string ns = fmt::format( "{}.{}", Venue_Name(venue).substr(6), - SecurityType_Name(type).substr(12)); + SecurityType_Name(type).substr(14)); return get_id(ns, name, add_if_new); } std::optional IDGenerator::get_id( const std::string &ns, const std::string &key, bool add_if_new) { std::lock_guard myLock(_instance_mx); - auto [exists, val] = me->kv_store->retrieve_one(ns, key); + auto [exists, rowid, val] = me->kv_store->retrieve_one(ns, key); if (exists) { - uint64_t rv = str_to_id(val); - LOG(Gen, Diag, "IDGenerator::get_id ns: {} key:{} id: {}", ns, key, rv); - return std::optional(rv); - + LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rowid); + return std::optional(rowid); } else { if (add_if_new) { auto all = me->kv_store->retrieve_all(); - uint64_t new_id = get_next_id(all); - me->kv_store->insert(ns, key, std::to_string(new_id)); + auto new_id = me->kv_store->insert(ns, key, ""); LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, new_id); return std::optional(new_id); @@ -88,13 +85,6 @@ std::optional IDGenerator::get_id( } } -std::uint64_t IDGenerator::get_or_add( - const std::string &ns, const std::string &key) { - auto id = get_id(ns, key, true); - bts_assert(id.has_value()); - return *id; -} - std::uint64_t IDGenerator::get_or_add( api::domain::Venue venue, api::domain::SecurityType type, diff --git a/src/lib/persistence/IStore.h b/src/lib/persistence/IStore.h index 1f33ddf28..29e38eb7a 100644 --- a/src/lib/persistence/IStore.h +++ b/src/lib/persistence/IStore.h @@ -1,8 +1,13 @@ #pragma once +#include "lib/persistence/store_row.h" + +#include #include #include #include +#include +#include namespace bts { namespace persistence { @@ -11,17 +16,45 @@ class IStore { public: virtual std::map> retrieve_all() = 0; - virtual std::pair - retrieve_one(std::string const &ns, std::string const &k) = 0; + + /** + * Finds the row with the specified namespace and key. + * + * @return std::tuple The first item indicates + * whether or not a row was found. For successful queries, the second and third + * items are the rowid and value, respectively. + */ + virtual std::tuple retrieve_one( + std::string const &ns, std::string const &k) = 0; virtual void remove(std::string const &ns, std::set const &keys) = 0; virtual void remove_all(std::string const &ns) = 0; - virtual void insert( + virtual std::int64_t insert( std::string const &ns, std::string const &k, std::string const &v) = 0; + + void insert_many(const std::vector &rows); + void insert_many_resolve_ids(std::vector &rows); + virtual void update( std::string const &ns, std::string const &k, std::string const &v) = 0; virtual ~IStore(); }; +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline void IStore::insert_many(const std::vector &rows) { + for (const auto &row : rows) { + insert(row.ns, row.key, row.value); + } +} + +inline void IStore::insert_many_resolve_ids(std::vector &rows) { + for (auto &row : rows) { + row.id = insert(row.ns, row.key, row.value); + } +} + } // namespace persistence } // namespace bts diff --git a/src/lib/persistence/SqliteStore.cpp b/src/lib/persistence/SqliteStore.cpp index 84e741ea3..50bf7cbf6 100644 --- a/src/lib/persistence/SqliteStore.cpp +++ b/src/lib/persistence/SqliteStore.cpp @@ -35,26 +35,28 @@ static boost::filesystem::path get_db_path(const std::string &str) { } struct SqliteStore::PImpl { + std::string const db; + boost::filesystem::path db_path; + sqlite3_ptr conn{nullptr, sqlite3_close}; + sqlite3_stmt_ptr insert{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr insert_many{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr insert_with_id{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr remove{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr remove_all{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr update{nullptr, sqlite3_finalize}; + PImpl(const boost::filesystem::path &db_path) - : db(db_path.stem().string()), db_path(db_path), - conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize), - remove(nullptr, sqlite3_finalize), - remove_all(nullptr, sqlite3_finalize), - update(nullptr, sqlite3_finalize) { + : db(db_path.stem().string()), db_path(db_path) { bts_assert(db.size()); } PImpl(std::string const &store_root, std::string const &db_name) : PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {} - std::string const db; - boost::filesystem::path db_path; - sqlite3_ptr conn; - sqlite3_stmt_ptr insert; - sqlite3_stmt_ptr remove; - sqlite3_stmt_ptr remove_all; - sqlite3_stmt_ptr update; - BTS_NO_COPY(PImpl); + explicit PImpl(sqlite3_ptr conn) : db(), db_path(), conn(std::move(conn)) {} + + PImpl(const PImpl &) = delete; + PImpl &operator=(const PImpl &) = delete; }; SqliteStore::SqliteStore(std::string const &db) @@ -64,54 +66,64 @@ SqliteStore::SqliteStore(std::string const &db) init_db(); } +SqliteStore::SqliteStore(sqlite3_ptr db) + : me(std::make_unique(std::move(db))) { + LOG(persist, Info, "SqliteStore ctr. db: "); + init_db(); +} + void SqliteStore::init_db() { - auto path = me->db_path; - auto parent = path.parent_path(); + const auto path = me->db_path; - if (!boost::filesystem::is_directory(parent)) - bts_assert(boost::filesystem::create_directories(parent)); - - if (!boost::filesystem::exists(path)) { - LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", - me->db, path.string()); - - me->conn = sqlite::open_db(path.c_str()); - bts_assert_msg(me->conn, path.c_str()); - - const char *create_schema_sql = - "create table store ( ns text, key text, value text)"; - sqlite::exec_nofail(me->conn, create_schema_sql); - - const char *key_index_sql = - "create unique index key_index on store(ns,key)"; - sqlite::exec_nofail(me->conn, key_index_sql); - - const char *ns_index_sql = "create index ns_index on store(ns)"; - sqlite::exec_nofail(me->conn, ns_index_sql); - } else { - me->conn = sqlite::open_db(path.c_str()); + if (!path.empty()) { + if (!boost::filesystem::is_directory(path.parent_path())) { + bts_assert(boost::filesystem::create_directories(path.parent_path())); + } + if (!boost::filesystem::exists(path)) { + LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", + me->db, path.string()); + } } + auto &conn = me->conn; + if (conn == nullptr) { + conn = sqlite::open_db(path.c_str()); + bts_assert(conn != nullptr); + } + sqlite::exec_nofail( + conn, + "CREATE TABLE IF NOT EXISTS store (ns TEXT, key TEXT, value TEXT)"); + sqlite::exec_nofail( + conn, "CREATE UNIQUE INDEX IF NOT EXISTS key_index on store(ns, key)"); + sqlite::exec_nofail( + conn, "CREATE INDEX IF NOT EXISTS ns_index on store(ns)"); + conn = sqlite::open_db(path.c_str()); + // Asynchronous for speed - sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;"); - sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL"); + sqlite::exec_nofail(conn, "PRAGMA synchronous = 0;"); + sqlite::exec_nofail(conn, "PRAGMA journal_mode=WAL"); // prepare insert statement const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)"; - me->insert = sqlite::prepare_query(me->conn, insert_sql); + me->insert = sqlite::prepare_query(conn, insert_sql); + + // prepare insert_with_id statement + const char *insert_with_id_sql = + "insert into store (rowid, ns, key, value) values (?,?,?,?)"; + me->insert_with_id = sqlite::prepare_query(conn, insert_with_id_sql); // prepare update statement const char *update_sql = "update store set value = ? where ns = ? and key = ?"; - me->update = sqlite::prepare_query(me->conn, update_sql); + me->update = sqlite::prepare_query(conn, update_sql); // prepare remove statement const char *remove_all_sql = "delete from store where ns = ?"; - me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql); + me->remove_all = sqlite::prepare_query(conn, remove_all_sql); // prepare remove statement const char *remove_sql = "delete from store where ns = ? and key = ?"; - me->remove = sqlite::prepare_query(me->conn, remove_sql); + me->remove = sqlite::prepare_query(conn, remove_sql); } SqliteStore::~SqliteStore() {} @@ -140,25 +152,27 @@ SqliteStore::Data SqliteStore::retrieve_all() { return result; } -std::pair SqliteStore::retrieve_one( +std::tuple SqliteStore::retrieve_one( std::string const &ns, std::string const &k) { auto query_str = fmt::format( - "select ns,key,value from store where ns='{}' and key='{}'", ns, k); + "select rowid,ns,key,value from store where ns='{}' and key='{}'", ns, + k); sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str()); - std::pair result{false, ""}; - + auto result = std::make_tuple(false, 0, ""); auto pq = query.get(); auto sqlite_result = sqlite3_step(pq); if (sqlite_result == SQLITE_ROW) { + std::int64_t rowid = sqlite3_column_int64(pq, 0); std::string ns( - (char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0)); - std::string key( (char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1)); + std::string key( + (char *)sqlite3_column_text(pq, 2), sqlite3_column_bytes(pq, 2)); std::string val( - (char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2)); - result.first = true; - result.second = std::move(val); + (char *)sqlite3_column_blob(pq, 3), sqlite3_column_bytes(pq, 3)); + std::get<0>(result) = true; + std::get<1>(result) = rowid; + std::get<2>(result) = std::move(val); } return result; } @@ -167,91 +181,185 @@ void SqliteStore::remove( std::string const &ns, std::set const &keys) { bts_assert(ns.size()); - for (auto &k : keys) { - SQLITE_CHK( - sqlite3_bind_text( - me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->remove.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not delete row in store. Will retry once. error_code: {} " - "ns:{}, k: {}", - r, ns, k); - sleep_for(2s); - r = sqlite3_step(me->remove.get()); + try { + for (auto &k : keys) { + SQLITE_CHK( + sqlite3_bind_text( + me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->remove.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to delete row. Bailing out. error_code: " - "{}", - r); - bts_assert(r == SQLITE_DONE); + "Could not delete row in store. Will retry once. error_code: " + "{} " + "ns:{}, k: {}", + r, ns, k); + sleep_for(2s); + r = sqlite3_step(me->remove.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to delete row. Bailing out. " + "error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); } - SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::remove threw. what: {}", + std::string{ex.what()}); } } void SqliteStore::remove_all(std::string const &ns) { bts_assert(ns.size()); - SQLITE_CHK( - sqlite3_bind_text( - me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->remove_all.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not delete all rows in store. Will retry once. error_code: {} " - "ns:{}", - r, ns); - sleep_for(2s); - r = sqlite3_step(me->remove_all.get()); + try { + SQLITE_CHK( + sqlite3_bind_text( + me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->remove_all.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to delete all rows. Bailing out. " - "error_code: " - "{}", - r); - bts_assert(r == SQLITE_DONE); + "Could not delete all rows in store. Will retry once. error_code: " + "{} " + "ns:{}", + r, ns); + sleep_for(2s); + r = sqlite3_step(me->remove_all.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to delete all rows. Bailing out. " + "error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::remove_all threw. what: {}", + std::string{ex.what()}); } - SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); } -void SqliteStore::insert( +std::int64_t SqliteStore::insert( std::string const &ns, std::string const &k, std::string const &v) { - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->insert.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not insert row in store. Will retry once. error_code: {} ns: " - "{}, k: {}, v: {}", - r, ns, k, v); - sleep_for(2s); - r = sqlite3_step(me->insert.get()); + try { + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->insert.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to insert row. Bailing out. error_code: {}", - r); - bts_assert(r == SQLITE_DONE); + "Could not insert row in store. Will retry once. error_code: {} " + "ns: " + "{}, k: {}, v: {}", + r, ns, k, v); + sleep_for(2s); + r = sqlite3_step(me->insert.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to insert row. Bailing out. error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + const auto rowid = sqlite3_last_insert_rowid(me->conn.get()); + SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); + return rowid; + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::insert threw. what: {}", + std::string{ex.what()}); } - SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); + return 0; +} + +void SqliteStore::insert_many(const std::vector &rows) { + LOG(md, Info, "SqliteStore::insert_many size: {}", rows.size()); + if (rows.empty()) { + return; + } + try { + sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION"); + sqlite3 *conn_rawp = me->conn.get(); + sqlite3_stmt *insert_rawp = me->insert.get(); + sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get(); + for (const auto &row : rows) { + int col_idx = 1; + sqlite3_stmt *stmt; + if (row.id > 0) { + stmt = insert_with_id_rawp; + sqlite3_bind_int(stmt, col_idx++, row.id); + } else { + stmt = insert_rawp; + } + sqlite3_bind_text( + stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC); + sqlite3_bind_text( + stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC); + int result = sqlite3_step(stmt); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + sqlite3_last_insert_rowid(conn_rawp); + } + sqlite3_reset(stmt); + } + sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION"); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::insert_many threw. what: {}", + std::string{ex.what()}); + } +} + +void SqliteStore::insert_many_resolve_ids(std::vector &rows) { + LOG(md, Info, "SqliteStore::insert_many_resolve_ids size: {}", rows.size()); + if (rows.empty()) { + return; + } + sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION"); + sqlite3 *conn_rawp = me->conn.get(); + sqlite3_stmt *insert_rawp = me->insert.get(); + sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get(); + for (auto &row : rows) { + int col_idx = 1; + sqlite3_stmt *stmt; + if (row.id > 0) { + stmt = insert_with_id_rawp; + sqlite3_bind_int(stmt, col_idx++, row.id); + } else { + stmt = insert_rawp; + } + sqlite3_bind_text( + stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC); + sqlite3_bind_text( + stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC); + int result = sqlite3_step(stmt); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + if (row.id > 0) { + bts_assert(sqlite3_last_insert_rowid(conn_rawp) == row.id); + } else { + row.id = sqlite3_last_insert_rowid(conn_rawp); + } + } + sqlite3_reset(stmt); + } + sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION"); } void SqliteStore::update( diff --git a/src/lib/persistence/SqliteStore.h b/src/lib/persistence/SqliteStore.h index 4f0caf725..6a833df80 100644 --- a/src/lib/persistence/SqliteStore.h +++ b/src/lib/persistence/SqliteStore.h @@ -1,7 +1,12 @@ #pragma once #include "IStore.h" +#include "lib/persistence/store_row.h" +#include "lib/utils/Sqlite_utils.h" + +#include #include +#include namespace bts { namespace persistence { @@ -9,22 +14,26 @@ namespace persistence { class SqliteStore : public IStore { public: SqliteStore(std::string const &db); + explicit SqliteStore(sqlite3_ptr db); ~SqliteStore(); static SqliteStore from_path(const std::string &full_path); typedef std::map> Data; Data retrieve_all() override; - std::pair retrieve_one( + + std::tuple retrieve_one( std::string const &ns, std::string const &k) override; void remove( std::string const &ns, std::set const &keys) override; void remove_all(std::string const &ns) override; - void insert( + std::int64_t insert( std::string const &ns, std::string const &k, std::string const &v) override; + void insert_many(const std::vector &rows); + void insert_many_resolve_ids(std::vector &rows); void update( std::string const &ns, std::string const &k, diff --git a/src/lib/persistence/store_row.h b/src/lib/persistence/store_row.h new file mode 100644 index 000000000..1c7b7f079 --- /dev/null +++ b/src/lib/persistence/store_row.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace bts::persistence { + +struct StoreRow { + std::string ns; + std::string key; + std::string value; + /** + * Represents the unique 'rowid' in the sqlite db. A value of 0 indicates that + * the rowid isn't known, and should either be generated by the db (for inserts), + * or resolved by finding the row with matching ns, key, value (for retrieves). + */ + std::int64_t id{0}; +}; + +} // namespace bts::persistence diff --git a/src/lib/utils/checked_arithmetic.h b/src/lib/utils/checked_arithmetic.h new file mode 100644 index 000000000..91e60a6b9 --- /dev/null +++ b/src/lib/utils/checked_arithmetic.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +namespace bts::system::math { + +static std::uint32_t checked_ltou(std::int64_t l); +static std::uint64_t checked_ltoul(std::int64_t l); + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::uint32_t checked_ltou(std::int64_t l) { + if (l < 0) { + throw std::overflow_error{ + "Failed to convert int64 to uint32 (value < 0)"}; + } + if (l > + static_cast(std::numeric_limits::max())) { + throw std::overflow_error{ + "Failed to convert int64 to uint32. value > max)"}; + } + return static_cast(l); +} + +inline std::uint64_t checked_ltoul(std::int64_t l) { + if (l < 0) { + throw std::overflow_error{ + "Failed to convert int64 to uint64 (value < 0)"}; + } + return static_cast(l); +} + +} // namespace bts::system::math diff --git a/src/server/sparkd/analytics/AnalyticsServiceServer.cpp b/src/server/sparkd/analytics/AnalyticsServiceServer.cpp index 7cd0cc84a..83ff900fd 100644 --- a/src/server/sparkd/analytics/AnalyticsServiceServer.cpp +++ b/src/server/sparkd/analytics/AnalyticsServiceServer.cpp @@ -52,7 +52,7 @@ void add_or_update_to_table( const std::string &ns, const std::string &key, const std::string &val) { - auto [exists, existing_val] = store->retrieve_one(ns, key); + auto [exists, rowid, existing_val] = store->retrieve_one(ns, key); LOG(Gen, Info, "add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns, key, val, exists, existing_val); @@ -386,7 +386,8 @@ void AnalyticsServiceServer::save_report( const api::Response<::bts::api::ResultMessage> &resp) { Report report = report_arg; std::string key = std::to_string(report.id()); - auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key); + auto [present, rowid, val] = + me->reports_store->retrieve_one(me->reports_ns, key); if (!present) { int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1; report.set_id(new_id); @@ -408,7 +409,8 @@ void AnalyticsServiceServer::delete_report( const bts::api::analytics::IntWrapper &id, const api::Response<::bts::api::ResultMessage> &resp) { std::string key = std::to_string(id.data()); - auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key); + auto [present, rowid, val] = + me->reports_store->retrieve_one(me->reports_ns, key); api::ResultMessage res; res.set_success(false); if (!present) { diff --git a/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp b/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp index 055e718b4..5acc34f92 100644 --- a/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp +++ b/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp @@ -581,10 +581,11 @@ void negate_id(api::Security *sec) { } void ExchangeSimApplication::persist_book(OrderBook const &b) { - auto existing = me->store->retrieve_one("books", b.sec_desc()); + auto [exists, rowid, oldval] = + me->store->retrieve_one("books", b.sec_desc()); auto val = utils::pb::to_string(b); - if (existing.first) + if (exists) me->store->update("books", b.sec_desc(), val); else me->store->insert("books", b.sec_desc(), val); diff --git a/src/server/sparkd/md/emumba/EmumbaDeribitData.h b/src/server/sparkd/md/emumba/EmumbaDeribitData.h index 81295af80..c35d44075 100644 --- a/src/server/sparkd/md/emumba/EmumbaDeribitData.h +++ b/src/server/sparkd/md/emumba/EmumbaDeribitData.h @@ -18,10 +18,11 @@ struct EmumbaDeribitData { EmumbaDeribitData() : sec_index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(sec_index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(sec_index.get_key(k.value())); + }, + &utils::Dispatcher::current()), md_producer(recorder), listener(sec_index, md_producer) {} }; } // namespace bts::md diff --git a/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp b/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp index 748a89251..70e8e0eee 100644 --- a/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp +++ b/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp @@ -43,10 +43,11 @@ struct EmumbaMain::PImpl { PImpl() : sec_index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(sec_index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(sec_index.get_key(k.value())); + }, + &utils::Dispatcher::current()), producer(recorder), listener(sec_index, producer) { const auto config_path = Path{config::get().config_path()} / diff --git a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp index 4bcd0c9d4..0dc491f5d 100644 --- a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp +++ b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp @@ -1,70 +1,66 @@ #include "ICEMain.h" #include "lib/config/config.h" -#include "lib/md/SecurityDB.h" -#include "lib/md/coconut2.h" -#include "lib/md/ice/ICEUtils.h" -#include "lib/md/protos/co2.pb.h" -#include "lib/md/vendor/OnixsUtils.h" -#include "lib/os/os.h" -#include "lib/utils/Dispatcher.h" -#include "lib/utils/bts_chrono.h" -#include "lib/utils/helper_macros.h" -#include "lib/utils/proto_helper.h" - +#include "lib/config/protos/deployment_type.pb.h" #include "lib/config/protos/market_data.pb.h" #include "lib/config/protos/system_config.pb.h" -#include "lib/utils/thread_manager.h" +#include "lib/md/ice/ICEUtils.h" +#include "lib/md/vendor/OnixsUtils.h" +#include #include #include -#include - -void recursive_copy( - const boost::filesystem::path &src, const boost::filesystem::path &dst) { - if (boost::filesystem::exists(dst)) - throw std::runtime_error(dst.generic_string() + " exists"); - if (boost::filesystem::is_directory(src)) { - boost::filesystem::create_directories(dst); - for (boost::filesystem::directory_entry &item : - boost::filesystem::directory_iterator(src)) - recursive_copy(item.path(), dst / item.path().filename()); - } else if (boost::filesystem::is_regular_file(src)) - boost::filesystem::copy(src, dst); - else - throw std::runtime_error(dst.generic_string() + " not dir or file"); -} using namespace std; using namespace bts::co2; using std::size_t; -namespace bts { -namespace md { -namespace ice { +namespace bts::md::ice { using bts::config::SystemConfig; using Path = boost::filesystem::path; -ICEMain::ICEMain() - : _index(), _recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(_index.get_key(k.value())); - }}), - _producer(_recorder), _handlers(), _listeners() {} +struct ICEMain::PImpl { + ICEOptions options; + std::string config_file; + ICESecurityIndex index; + SecurityIDRecorder recorder; + MDProducer producer; + std::vector>> + handlers; + std::vector> listeners; -ICEMain::~ICEMain() {} + PImpl(ICEOptions options) + : options(options), + config_file((Path{config::get().data_path()} / + "onixs_ice" / this->options.ice_config_file()) + .string()), + index(), recorder( + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(index.get_key(k.value())); + }, + &utils::Dispatcher::current()), + producer(recorder), handlers(), listeners() {} -const IceChannelConfig channel_settings_for(const ICEChannel &channel) { - const auto &config = config::get(); - for (const auto &ch : config.channel()) { - if (ch.name() == channel.name) - return ch; + IceChannelConfig channel_settings_for(const ICEChannel &channel) { + for (const auto &ch : options.channel()) { + if (ch.name() == channel.name) + return ch; + } + return {}; } - return {}; +}; + +ICEMain::ICEMain() : ICEMain(config::get()) {} + +ICEMain::ICEMain(ICEOptions options) + : me_(std::make_unique(std::move(options))) { + me_->options = std::move(options); } +ICEMain::~ICEMain() = default; + onixs_ice::MarketSubscriptions market_subcription_for_channel( const ICEChannel &chan) { onixs_ice::MarketSubscriptions subs; @@ -90,23 +86,30 @@ onixs_ice::MarketSubscriptions market_subcription_for_channel( return subs; } -void ICEMain::start(const std::vector &symbols) { - auto ice_conf_location = - Path{config::get().data_path()} / "onixs_ice/"; - if (!boost::filesystem::is_directory(ice_conf_location)) { - auto conf_path = Path{os::sparkd_wd()} / "data_files" / "onixs_ice/"; - recursive_copy(Path{conf_path}, ice_conf_location); +static void validate_config_file(const std::string &config_file) { + if (!boost::filesystem::exists(config_file)) { + const std::string config_url = + config::get().type() == "dev" + ? "https://community.theice.com/docs/DOC-19989" + : "https://drive.google.com/" + "open?id=13_eyF2pKtn3F7chgsa_75ftUdf7emMbp"; + bts_assert_msgf( + false, "Ice config file not found '{}'. Please download from {}", + config_file, config_url); } +} +void ICEMain::start(const std::vector &symbols) { await_ice_config(); + validate_config_file(me_->config_file); auto license = onixs_create_temporary_license(); const auto &channels = parse_ice_channels(symbols); for (const auto &channel : channels) { unique_ptr phandler = prepare_handler(channel, license.path.string()); - _listeners.push_back(prepare_listener(*phandler, channel)); - _handlers.push_back(std::make_pair(channel, std::move(phandler))); + me_->listeners.push_back(prepare_listener(*phandler, channel)); + me_->handlers.push_back(std::make_pair(channel, std::move(phandler))); std::stringstream symstr; for (const auto &sym : channel.symbols) @@ -115,7 +118,7 @@ void ICEMain::start(const std::vector &symbols) { symstr.str()); } - for (auto &pair : _handlers) { + for (auto &pair : me_->handlers) { LOG(md, Info, "ICE: Init Handler {}", pair.first.to_string()); sleep_for(16s); onixs_ice::MarketSubscriptions subs = @@ -132,7 +135,7 @@ void ICEMain::start(const std::vector &symbols) { } void ICEMain::send_heartbeats() { - for (const auto &listener : _listeners) + for (const auto &listener : me_->listeners) listener->heartbeat(); } @@ -140,20 +143,18 @@ void ICEMain::send_heartbeats() { // Initialize HandlerSettings and allocate the channel Handler unique_ptr ICEMain::prepare_handler( const ICEChannel &channel, const std::string &license) { - const auto &options = channel_settings_for(channel); + const auto &options = me_->channel_settings_for(channel); LOG(md, Info, "Loaded channel options for channel {}:\n{}", channel.to_string(), options.DebugString()); onixs_ice::HandlerSettings settings; settings.licenseDirectory = license; - // - // Logging settings.logLevel = (onixs_ice::LogLevels::Enum)options.onixs_log_mode(); - settings.logDirectory = ice_log_path(); - - // - // Config file - settings.connectivityConfiguration = ice_config_path(); + settings.logDirectory = + (boost::filesystem::path{config::get().data_path()} / + "logging" / "onixs_ice/") + .string(); + settings.connectivityConfiguration = me_->config_file; // // Network settings @@ -231,12 +232,12 @@ unique_ptr ICEMain::prepare_listener( // heartbeats vector producer_channels; for (auto &symbol : channel.symbols) { - producer_channels.push_back(_producer.get_channel( + producer_channels.push_back(me_->producer.get_channel( symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } unique_ptr listener(new ICEListener( - _index, _producer, /*_settlement_recorder,*/ producer_channels)); + me_->index, me_->producer, /*_settlement_recorder,*/ producer_channels)); for (auto &symbol : channel.symbols) listener->add_filter_symbol(symbol.symbol()); @@ -254,6 +255,4 @@ unique_ptr ICEMain::prepare_listener( return listener; } -} // namespace ice -} // namespace md -} // namespace bts +} // namespace bts::md::ice diff --git a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h index 1ea217a39..8c37f9476 100644 --- a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h +++ b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h @@ -1,21 +1,12 @@ #pragma once #include "ICEListener.h" -#include "ICESecurityIndex.h" -#include "lib/md/coconut2.h" -#include "lib/utils/bts_no_copy.h" -#include "onixs.h" #include "server/sparkd/md/platforms/IMDPlatform.h" -#include #include #include -namespace bts { -namespace md { -namespace ice { - -// void run_onixs_ice_server(UserSymbol* sym, size_t symcount); +namespace bts::md::ice { struct ICEChannel; @@ -28,27 +19,22 @@ struct ICEChannel; class ICEMain : public IMDPlatform { public: ICEMain(); + explicit ICEMain(ICEOptions options); + ICEMain(const ICEMain &) = delete; + ICEMain &operator=(const ICEMain &) = delete; ~ICEMain(); void start(const std::vector &symbols); void send_heartbeats() override; - std::unique_ptr - prepare_handler(const ICEChannel &channel, const std::string &license); - std::unique_ptr - prepare_listener(onixs_ice::Handler &handler, const ICEChannel &channel); + std::unique_ptr prepare_handler( + const ICEChannel &channel, const std::string &license); + std::unique_ptr prepare_listener( + onixs_ice::Handler &handler, const ICEChannel &channel); private: - BTS_NO_COPY(ICEMain); - - ICESecurityIndex _index; - SecurityIDRecorder _recorder; - MDProducer _producer; - std::vector>> - _handlers; - std::vector> _listeners; + struct PImpl; + std::unique_ptr me_; }; -} // namespace ice -} // namespace md -} // namespace bts +} // namespace bts::md::ice diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 9ad5c53ba..0a54040ad 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -1,24 +1,18 @@ #include "MDP3Main.h" - +#include "lib/api/md/GlbxMarketDataConfig.pb.h" #include "lib/config/config.h" +#include "lib/config/protos/deployment_type.pb.h" #include "lib/logging/logging.h" #include "lib/md/MBOProcessor.h" #include "lib/md/SecurityDB.h" #include "lib/md/cme/MDP3Utils.h" -#include "lib/md/coconut2.h" -#include "lib/md/protos/co2.pb.h" +#include "lib/md/cme/mdp3_data_directory.h" #include "lib/md/vendor/OnixsUtils.h" -#include "lib/os/os.h" -#include "lib/utils/Dispatcher.h" -#include "lib/utils/proto_helper.h" - -#include "lib/config/protos/market_data.pb.h" -#include "lib/config/protos/system_config.pb.h" +#include "lib/os/spark_data_directory.h" #include "lib/utils/thread_manager.h" #include #include -#include #include #include @@ -29,16 +23,33 @@ using std::size_t; namespace bts { namespace md { -MDP3Main::MDP3Main() - : _index(), _recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(_index.get_key(k.value())); - }}), - _producer(_recorder), _engines(), _handlers(), _listeners(), _recovery() { -} +#define TRADING_STATUS(x) bts::domain::protobufs::TRADINGSTATUS_##x -MDP3Main::~MDP3Main() {} +using Path = boost::filesystem::path; + +struct MDP3Main::PImpl { + Mdp3DataDirectory data_dir; + CMESecurityIndex index; + SecurityIDRecorder recorder; + MDProducer producer; + std::map> engines; + std::map> handlers; + std::vector> listeners; + std::vector> recovery; + PImpl() + : index(), recorder( + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(index.get_key(k.value())); + }, + &utils::Dispatcher::current()), + producer(recorder), engines(), handlers(), listeners(), recovery() {} +}; + +MDP3Main::MDP3Main() + : me_(std::make_unique()) {} + +MDP3Main::~MDP3Main() = default; const CmeChannelConfig channel_settings_for(const string &id) { const auto &config = config::get(); @@ -173,11 +184,11 @@ void MDP3Main::start( auto license = onixs_create_temporary_license(); initialize(license.path.string()); const auto &channels = parse_channels(symbols); - _engines = create_feed_engines(channels); + me_->engines = create_feed_engines(channels); for (const auto &channel : channels) { unique_ptr phandler = prepare_handler(channel.id, mbo); - _listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd)); - _handlers.insert(std::make_pair(channel.id, std::move(phandler))); + me_->listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd)); + me_->handlers.insert(std::make_pair(channel.id, std::move(phandler))); std::stringstream symstr; for (const auto &sym : channel.symbols) { symstr << sym.symbol() << "-" << SecurityType_Name(sym.type()) << " "; @@ -186,7 +197,7 @@ void MDP3Main::start( symstr.str()); } - for (auto &pair : _handlers) { + for (auto &pair : me_->handlers) { onixs::SessionSettings settings = session_settings_for_channel(pair.first, mbo); pair.second->start(settings); @@ -196,7 +207,7 @@ void MDP3Main::start( } void MDP3Main::send_heartbeats() { - for (const auto &listener : _listeners) { + for (const auto &listener : me_->listeners) { listener->heartbeat(); } } @@ -219,7 +230,8 @@ unique_ptr MDP3Main::prepare_handler( onixs::HandlerSettings settings; settings.channel(std::stoul(channel_id)); - settings.feeds().connectivityConfigurationFile(config_path()); + settings.feeds().connectivityConfigurationFile( + (me_->data_dir.config_file()).string()); // Market data options settings.bookManagement().directBooks().maintain(true); @@ -306,7 +318,8 @@ unique_ptr MDP3Main::prepare_handler( logger = new onixs::NullLogger(); } else { onixs::FileLoggerSettings loggerSettings; - loggerSettings.filename(log_path() + "/log_" + channel_id); + loggerSettings.filename( + (me_->data_dir.log_dir() / ("log_" + channel_id)).string()); loggerSettings.severityLevel( (onixs::LogSeverity::Enum)options.onixs_log_mode()); logger = new onixs::FileLogger(loggerSettings); @@ -338,19 +351,19 @@ unique_ptr MDP3Main::prepare_handler( try { auto handler = unique_ptr(new onixs::Handler(settings)); - auto iter = _engines.find(options.feed_engine()); - if (iter == _engines.end()) { + auto iter = me_->engines.find(options.feed_engine()); + if (iter == me_->engines.end()) { LOG(md, Fatal, "Missing feed engine definition '{}' for channel id '{}'. Add a " "feed_engine entry with name:'{}' to the CMEOptions config key.", options.feed_engine(), channel_id, options.feed_engine()); bts_assert_msg( - iter != _engines.end(), "Missing feed engine definition."); + iter != me_->engines.end(), "Missing feed engine definition."); } handler->bindFeedEngine(*(iter->second)); if (recovery) { handler->bindTcpRecovery(*recovery); - _recovery.push_back(std::move(recovery)); + me_->recovery.push_back(std::move(recovery)); } if (logger) { handler->bindLogger(*logger); @@ -375,12 +388,13 @@ unique_ptr MDP3Main::prepare_listener( // heartbeats vector producer_channels; for (auto &symbol : channel.symbols) { - producer_channels.push_back(_producer.get_channel( + producer_channels.push_back(me_->producer.get_channel( symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } unique_ptr listener(new MDP3Listener( - _index, _producer, mbo, producer_channels, trade_summary_dispatcher)); + me_->index, me_->producer, mbo, producer_channels, + trade_summary_dispatcher)); for (auto &symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index c67dc93d4..fc3043ac3 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -29,7 +29,9 @@ struct CMEChannel; */ class MDP3Main : public IMDPlatform { public: - MDP3Main(); + explicit MDP3Main(); + MDP3Main(const MDP3Main &) = delete; + MDP3Main &operator=(const MDP3Main &) = delete; ~MDP3Main(); void start( @@ -39,8 +41,8 @@ class MDP3Main : public IMDPlatform { void send_heartbeats() override; void initialize(const std::string &license_store); - std::unique_ptr - prepare_handler(const std::string &channel, MBOWriter *); + std::unique_ptr prepare_handler( + const std::string &channel, MBOWriter *); std::unique_ptr prepare_listener( onixs::Handler &handler, const CMEChannel &channel, @@ -48,16 +50,8 @@ class MDP3Main : public IMDPlatform { TradeSummarySignalDispatcher *); private: - BTS_NO_COPY(MDP3Main); - - CMESecurityIndex _index; - SecurityIDRecorder _recorder; - MDProducer _producer; - // co2::SettlementRecorder _settlement_recorder; - std::map> _engines; - std::map> _handlers; - std::vector> _listeners; - std::vector> _recovery; + struct PImpl; + std::unique_ptr me_; }; } // namespace md diff --git a/src/server/sparkd/position/PositionServiceServer.cpp b/src/server/sparkd/position/PositionServiceServer.cpp index c3e511409..26c95f74f 100644 --- a/src/server/sparkd/position/PositionServiceServer.cpp +++ b/src/server/sparkd/position/PositionServiceServer.cpp @@ -293,9 +293,9 @@ void PositionServiceServer::update_manual_fill( auto *this_resp = resp.add_response(); auto id = std::to_string(mf.record_id()); - auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id); - auto found = p.first; - if (!found) { + auto [exists, rowid, val] = + me->manual_fills_store->retrieve_one(manual_fills_ns, id); + if (!exists) { this_resp->set_ok(false); this_resp->set_msg("Could not find record"); } else { @@ -326,9 +326,9 @@ void PositionServiceServer::remove_manual_fill( auto *this_resp = resp.add_response(); auto id = std::to_string(mf.record_id()); - auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id); - auto found = p.first; - if (!found) { + auto [exists, rowid, val] = + me->manual_fills_store->retrieve_one(manual_fills_ns, id); + if (!exists) { this_resp->set_ok(false); this_resp->set_msg("Could not find record"); } else { @@ -367,10 +367,9 @@ void PositionServiceServer::update_security_parameters( req.op_params_size()); for (const auto &p : req.op_params()) { std::string key_str = std::to_string(p.security_key()); - auto db_p = + auto [exists, rowid, val] = me->security_params_store->retrieve_one(security_params_ns, key_str); - auto found = db_p.first; - if (!found) { + if (!exists) { me->security_params_store->insert( security_params_ns, key_str, utils::pb::to_string(p)); } else { diff --git a/src/server/sparkd/user/AdminServiceDB.cpp b/src/server/sparkd/user/AdminServiceDB.cpp index 755ff079d..c89407e0e 100644 --- a/src/server/sparkd/user/AdminServiceDB.cpp +++ b/src/server/sparkd/user/AdminServiceDB.cpp @@ -21,7 +21,8 @@ AdminServiceDB::AdminServiceDB() : me(new PImpl()) {} AdminServiceDB::~AdminServiceDB() {} std::optional AdminServiceDB::retrieve() { - auto [has_table, data] = me->store.retrieve_one(table_name, unique_key); + auto [has_table, rowid, data] = + me->store.retrieve_one(table_name, unique_key); if (!has_table) return {}; @@ -33,7 +34,8 @@ std::optional AdminServiceDB::retrieve() { } bool AdminServiceDB::update(api::admin::AdminData const &state) { - auto [has_table, data] = me->store.retrieve_one(table_name, unique_key); + auto [has_table, rowid, data] = + me->store.retrieve_one(table_name, unique_key); UNUSED(data); std::string ser_state;