From 2bc8cf6db52d9f32c9dd91c93a050a717efe87c8 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 10 Oct 2024 15:55:32 +0000 Subject: [PATCH] refactor: Return row id in IStore::retrieve_one This is a prerequisite to synchronizing security ids stored via IDGenerator (the "new" way to store ids, added when we implemented Deribit), and SecurityDB (the original way from spark days). The problem was that these 2 implementations were using separate databases. We need to unify the interfaces between these storage services to avoid breaking existing code. --- .../analytics/AnalyticsServiceServer.cpp | 8 +- .../exchange_sim/ExchangeSimApplication.cpp | 5 +- .../sparkd/md/emumba/EmumbaDeribitData.h | 9 +- .../sparkd/md/platforms/emumba/EmumbaMain.cpp | 9 +- .../sparkd/md/platforms/ice_onixs/ICEMain.cpp | 9 +- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 9 +- .../sparkd/position/PositionServiceServer.cpp | 17 +- src/app_sparkd/sparkd/user/AdminServiceDB.cpp | 6 +- src/lib_spark/spark/domain_model/Security.cpp | 2 +- src/lib_spark/spark/md/Diag/ShmDumper.cpp | 79 +--- src/lib_spark/spark/md/SecurityDB.cpp | 22 -- src/lib_spark/spark/md/SecurityIDRecorder.cpp | 201 ++++------ src/lib_spark/spark/md/SecurityIDRecorder.h | 22 +- src/lib_spark/spark/md/SymbolLoader.h | 26 +- .../spark/md/SyntheticSecurityService.cpp | 5 +- src/lib_spark/spark/md/shm/fixed_security.cpp | 24 +- src/lib_spark/spark/md/shm/fixed_security.h | 1 - src/lib_spark/spark/md/test/md2_tests.cpp | 29 +- .../spark/persistence/IDGenerator.cpp | 20 +- src/lib_spark/spark/persistence/IStore.h | 38 +- .../spark/persistence/SqliteStore.cpp | 351 +++++++++++------- src/lib_spark/spark/persistence/SqliteStore.h | 13 +- src/lib_spark/spark/persistence/store_row.h | 20 + .../system/math/checked_arithmetic.h | 37 ++ src/lib_system/system/time_span.h | 10 + 25 files changed, 495 insertions(+), 477 deletions(-) create mode 100644 src/lib_spark/spark/persistence/store_row.h create mode 100644 src/lib_system/system/math/checked_arithmetic.h diff --git a/src/app_sparkd/sparkd/analytics/AnalyticsServiceServer.cpp b/src/app_sparkd/sparkd/analytics/AnalyticsServiceServer.cpp index 2bee34c89..90e319c58 100644 --- a/src/app_sparkd/sparkd/analytics/AnalyticsServiceServer.cpp +++ b/src/app_sparkd/sparkd/analytics/AnalyticsServiceServer.cpp @@ -50,7 +50,7 @@ void add_or_update_to_table( const std::string &ns, const std::string &key, const std::string &val) { - auto [exists, existing_val] = store->retrieve_one(ns, key); + auto [exists, rowid, existing_val] = store->retrieve_one(ns, key); LOG(Gen, Info, "add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns, key, val, exists, existing_val); @@ -384,7 +384,8 @@ void AnalyticsServiceServer::save_report( const api::Response<::bts::api::ResultMessage> &resp) { Report report = report_arg; std::string key = std::to_string(report.id()); - auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key); + auto [present, rowid, val] = + me->reports_store->retrieve_one(me->reports_ns, key); if (!present) { int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1; report.set_id(new_id); @@ -406,7 +407,8 @@ void AnalyticsServiceServer::delete_report( const bts::api::analytics::IntWrapper &id, const api::Response<::bts::api::ResultMessage> &resp) { std::string key = std::to_string(id.data()); - auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key); + auto [present, rowid, val] = + me->reports_store->retrieve_one(me->reports_ns, key); api::ResultMessage res; res.set_success(false); if (!present) { diff --git a/src/app_sparkd/sparkd/exchange_sim/ExchangeSimApplication.cpp b/src/app_sparkd/sparkd/exchange_sim/ExchangeSimApplication.cpp index 1fd54bf43..01e879fa8 100644 --- a/src/app_sparkd/sparkd/exchange_sim/ExchangeSimApplication.cpp +++ b/src/app_sparkd/sparkd/exchange_sim/ExchangeSimApplication.cpp @@ -582,10 +582,11 @@ void negate_id(api::Security *sec) { } void ExchangeSimApplication::persist_book(OrderBook const &b) { - auto existing = me->store->retrieve_one("books", b.sec_desc()); + auto [exists, rowid, oldval] = + me->store->retrieve_one("books", b.sec_desc()); auto val = utils::pb::to_string(b); - if (existing.first) + if (exists) me->store->update("books", b.sec_desc(), val); else me->store->insert("books", b.sec_desc(), val); diff --git a/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitData.h b/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitData.h index 45ec10353..2705b24dc 100644 --- a/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitData.h +++ b/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitData.h @@ -18,10 +18,11 @@ struct EmumbaDeribitData { EmumbaDeribitData() : sec_index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(sec_index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(sec_index.get_key(k.value())); + }, + &utils::Dispatcher::current()), md_producer(recorder), listener(sec_index, md_producer) {} }; } // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/platforms/emumba/EmumbaMain.cpp b/src/app_sparkd/sparkd/md/platforms/emumba/EmumbaMain.cpp index 96533c9f4..55831df30 100644 --- a/src/app_sparkd/sparkd/md/platforms/emumba/EmumbaMain.cpp +++ b/src/app_sparkd/sparkd/md/platforms/emumba/EmumbaMain.cpp @@ -43,10 +43,11 @@ struct EmumbaMain::PImpl { PImpl() : sec_index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(sec_index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(sec_index.get_key(k.value())); + }, + &utils::Dispatcher::current()), producer(recorder), listener(sec_index, producer) { const auto config_path = Path{config::get().config_path()} / diff --git a/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp b/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp index 1996c47ed..87e895d8c 100644 --- a/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp +++ b/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp @@ -38,10 +38,11 @@ struct ICEMain::PImpl { "onixs_ice" / this->options.ice_config_file()) .string()), index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(index.get_key(k.value())); + }, + &utils::Dispatcher::current()), producer(recorder), handlers(), listeners() {} IceChannelConfig channel_settings_for(const ICEChannel &channel) { diff --git a/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index a785f2a2a..fa8fda27e 100644 --- a/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -43,10 +43,11 @@ struct MDP3Main::PImpl { : data_path(spark::SparkDataDirectory{}.root() / "cme_mdp3/"), log_path(spark::SparkDataDirectory{}.log_dir() / "onixs/"), index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(index.get_key(k.value())); + }, + &utils::Dispatcher::current()), producer(recorder), engines(), handlers(), listeners(), recovery() {} }; diff --git a/src/app_sparkd/sparkd/position/PositionServiceServer.cpp b/src/app_sparkd/sparkd/position/PositionServiceServer.cpp index aae09b1ff..45da18af1 100644 --- a/src/app_sparkd/sparkd/position/PositionServiceServer.cpp +++ b/src/app_sparkd/sparkd/position/PositionServiceServer.cpp @@ -293,9 +293,9 @@ void PositionServiceServer::update_manual_fill( auto *this_resp = resp.add_response(); auto id = std::to_string(mf.record_id()); - auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id); - auto found = p.first; - if (!found) { + auto [exists, rowid, val] = + me->manual_fills_store->retrieve_one(manual_fills_ns, id); + if (!exists) { this_resp->set_ok(false); this_resp->set_msg("Could not find record"); } else { @@ -326,9 +326,9 @@ void PositionServiceServer::remove_manual_fill( auto *this_resp = resp.add_response(); auto id = std::to_string(mf.record_id()); - auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id); - auto found = p.first; - if (!found) { + auto [exists, rowid, val] = + me->manual_fills_store->retrieve_one(manual_fills_ns, id); + if (!exists) { this_resp->set_ok(false); this_resp->set_msg("Could not find record"); } else { @@ -367,10 +367,9 @@ void PositionServiceServer::update_security_parameters( req.op_params_size()); for (const auto &p : req.op_params()) { std::string key_str = std::to_string(p.security_key()); - auto db_p = + auto [exists, rowid, val] = me->security_params_store->retrieve_one(security_params_ns, key_str); - auto found = db_p.first; - if (!found) { + if (!exists) { me->security_params_store->insert( security_params_ns, key_str, utils::pb::to_string(p)); } else { diff --git a/src/app_sparkd/sparkd/user/AdminServiceDB.cpp b/src/app_sparkd/sparkd/user/AdminServiceDB.cpp index 037554423..7786aa705 100644 --- a/src/app_sparkd/sparkd/user/AdminServiceDB.cpp +++ b/src/app_sparkd/sparkd/user/AdminServiceDB.cpp @@ -20,7 +20,8 @@ AdminServiceDB::AdminServiceDB() : me(new PImpl()) {} AdminServiceDB::~AdminServiceDB() {} std::optional AdminServiceDB::retrieve() { - auto [has_table, data] = me->store.retrieve_one(table_name, unique_key); + auto [has_table, rowid, data] = + me->store.retrieve_one(table_name, unique_key); if (!has_table) return {}; @@ -32,7 +33,8 @@ std::optional AdminServiceDB::retrieve() { } bool AdminServiceDB::update(api::admin::AdminData const &state) { - auto [has_table, data] = me->store.retrieve_one(table_name, unique_key); + auto [has_table, rowid, data] = + me->store.retrieve_one(table_name, unique_key); UNUSED(data); std::string ser_state; diff --git a/src/lib_spark/spark/domain_model/Security.cpp b/src/lib_spark/spark/domain_model/Security.cpp index 41f476616..8675730b7 100644 --- a/src/lib_spark/spark/domain_model/Security.cpp +++ b/src/lib_spark/spark/domain_model/Security.cpp @@ -2,7 +2,7 @@ #include "spark/config/config.h" #include "spark/config/protos/orders_fills_cleanup.pb.h" #include "spark/domain_model/Security.pb.h" -#include "spark/md/invalid_security_key.h" +#include "spark/md/common.h" #include "spark/utils/datetime.h" #include diff --git a/src/lib_spark/spark/md/Diag/ShmDumper.cpp b/src/lib_spark/spark/md/Diag/ShmDumper.cpp index 0d7f23d37..e089863c3 100644 --- a/src/lib_spark/spark/md/Diag/ShmDumper.cpp +++ b/src/lib_spark/spark/md/Diag/ShmDumper.cpp @@ -59,85 +59,8 @@ string dump_header(const ColdHeader &header) { return JsonDumper::dump(header); } -void write_date_string( - uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result); string ShmDumper::dump_security(const FixedSecurity &security) { - ostringstream res; - res << "{"; - res << "'key': " << security.key << ", "; - res << "'symbol': '" << security.symbol() << "', "; - res << "'type': '" << api::SecurityType_Name(security.type) << "', "; - res << "'exchange': '" << api::Venue_Name(security.exchange) << "', "; - res << "'alt_symbol': '" << security.alt_symbol() << "', "; - res << "'cfi_code': '" << security.cfi_code() << "', "; - res << "'security_desc': '" << security.security_desc() << "', "; - res << "'platform_id': '" << security.platform_id() << "', "; - res << "'display_factor': " << (int)security.display_factor << ", "; - res << "'numeric_id': " << security.numeric_id << ", "; - res << "'cboe_class_key': " << security.cboe_class_key << ", "; - res << "'main_frac': " << (int)security.cme_main_fraction << ", "; - res << "'sub_fract': " << (int)security.cme_sub_fraction << ", "; - res << "'price_format': " << (int)security.cme_price_display_format << ", "; - res << "'market_segment': " << (int)security.cme_market_segment_id << ", "; - - res << "'price_break': " << security.price_break << ", "; - res << "'low_tick': " << security.low_tick << ", "; - res << "'high_tick': " << security.high_tick << ", "; - res << "'cpp': " << security.cash_per_point << ", "; - - res << "'market_description ': " << security.ice_market_description() - << ", "; - res << "'cleared_alias ': " << security.ice_cleared_alias() << ", "; - res << "'product_description ': " << security.ice_product_name() << ", "; - res << "'hub_alias ': " << security.ice_hub_alias() << ", "; - res << "'strip_name': " << security.ice_strip_name() << ", "; - res << "'mic_code': " << security.mic_code() << ", "; - - switch (security.type) { - case SECURITYTYPE_SPOT: - break; - case SECURITYTYPE_FUTURE: { - auto *psec = static_cast(&security); - res << "'expiration': '"; - write_date_string( - psec->expiration_year, psec->expiration_month, psec->expiration_day, - res); - res << "' "; - break; - } - case SECURITYTYPE_OPTION: { - auto *psec = static_cast(&security); - res << "'expiration': '"; - write_date_string( - psec->expiration_year, psec->expiration_month, psec->expiration_day, - res); - res << "', "; - res << "'strike': " << psec->strike << ", "; - res << "'callput': '" << api::PutOrCall_Name(psec->call_put) << "' "; - break; - } - case SECURITYTYPE_SPREAD: { - auto *psec = static_cast(&security); - res << "'legs': ["; - for (size_t x = 0; x < psec->leg_count(); ++x) { - auto &leg = psec->legs[x]; - if (x > 0) - res << ","; - res << "{ 'key': " << leg.key << ", 'exkey': " << leg.exchange_key - << ", 'quantity': " << leg.leg_quantity - << ", 'leg_price': " << leg.leg_price - << ", 'delta': " << leg.leg_delta << ", 'side': '" - << api::Side_Name(leg.leg_side) << "' }"; - } - res << "]"; - break; - } - default: - break; - } - - res << "}"; - return res.str(); + return JsonDumper::dump(security); } string dump(datetime::Date date) { diff --git a/src/lib_spark/spark/md/SecurityDB.cpp b/src/lib_spark/spark/md/SecurityDB.cpp index 6ca69761f..8ba0e8e7f 100644 --- a/src/lib_spark/spark/md/SecurityDB.cpp +++ b/src/lib_spark/spark/md/SecurityDB.cpp @@ -63,26 +63,6 @@ datetime::Date sec_expiration_cut_off() { return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day()); } -void apply_compatibilit_mutations(api::Security &security) { - // Migrate from old tick stuff - if (security.has_glbx()) { - if (security.has_glbx()) { - if (security.contract_spec().main_fraction() == 0) { - security.mutable_contract_spec()->set_main_fraction( - security.glbx().do_not_use_main_fraction()); - } - if (security.contract_spec().sub_fraction() == 0) { - security.mutable_contract_spec()->set_sub_fraction( - security.glbx().do_not_use_sub_fraction()); - } - if (security.contract_spec().price_display_format() == 0) { - security.mutable_contract_spec()->set_price_display_format( - security.glbx().do_not_use_price_display_format()); - } - } - } -} - size_t load_securities( std::function const &sym_filter, const std::function &callback) { @@ -106,7 +86,6 @@ size_t load_securities( if (sym_filter(us)) { auto blob = select_stmt.getColumn("security_blob"); if (security.ParseFromArray(blob, blob.getBytes())) { - apply_compatibilit_mutations(security); bts_assert(security_key == security.id()); count++; callback(security); @@ -148,7 +127,6 @@ std::shared_ptr find_security(SecurityKey key) { auto result = std::make_shared(); auto blob = select.getColumn(0); if (result->ParseFromArray(blob, blob.getBytes())) { - apply_compatibilit_mutations(*result); return result; } } diff --git a/src/lib_spark/spark/md/SecurityIDRecorder.cpp b/src/lib_spark/spark/md/SecurityIDRecorder.cpp index 50471849a..205a1a208 100644 --- a/src/lib_spark/spark/md/SecurityIDRecorder.cpp +++ b/src/lib_spark/spark/md/SecurityIDRecorder.cpp @@ -2,22 +2,18 @@ #include "SecurityIDRecorder.h" #include "spark/domain_model/Security.h" #include "spark/domain_model/Security.pb.h" -#include "spark/md/invalid_security_key.h" -#include "spark/spark_data_directory.h" -#include - -#include "spark/config/config.h" +#include "spark/md/common.h" +#include "spark/persistence/SqliteStore.h" #include "spark/utils/Sqlite_utils.h" #include "system/logging/logging.h" +#include "system/math/checked_arithmetic.h" #include -#include - -#include "spark/config/protos/market_data.pb.h" -#include "system/protobufs/config/system_config.pb.h" - +#include #include +#include #include +#include using namespace std; using namespace bts::utils; @@ -29,10 +25,32 @@ namespace md { using bts::api::Security; using bts::api::SecurityType; using bts::api::Venue; -using bts::system::protobufs::SystemConfig; +using bts::persistence::IStore; +using bts::persistence::SqliteStore; uint64_t exchange_id(const api::Security &security); +static std::unique_ptr get_default_store() { + return std::make_unique("key_ids"); +} + +static persistence::StoreRow get_row( + Venue venue, + SecurityType type, + const std::string &name, + const std::string &prefix = "") { + return persistence::StoreRow{ + fmt::format( + "{}{}.{}", std::string{prefix}, Venue_Name(venue).substr(6), + SecurityType_Name(type).substr(13)), + name, ""}; +} + +static persistence::StoreRow get_row( + const api::Security &s, const std::string &prefix = "") { + return get_row(s.exchange(), bts::securities::security_type(s), prefix); +} + struct SecurityIdRequest { Venue venue; SecurityType type; @@ -42,14 +60,16 @@ struct SecurityIdRequest { }; struct SecurityIDRecorderState { - SecurityIDRecorderState(SecurityIDRecorderParams &p) - : pending_handle(), thread_id(), db(nullptr, sqlite3_close), timer(), - lookup_key(p.lookup_key), unresolved_legs() {} + SecurityIDRecorderState( + std::unique_ptr store, KeyLookupOption lookup_key) + : store(store == nullptr ? get_default_store() : std::move(store)), + pending_handle(), thread_id(), timer(), lookup_key(lookup_key), + unresolved_legs() {} + std::unique_ptr store{nullptr}; tbb::concurrent_queue> id_request_queue; tbb::concurrent_queue pending_handle; std::thread::id thread_id; - sqlite3_ptr db; unique_ptr timer; KeyLookupOption lookup_key; @@ -85,47 +105,28 @@ struct SecurityIDRecorderState { void check_access() { bts_assert(thread_id == std::this_thread::get_id()); } void add_unresolved_spread(MDProducerHandle &handle); - security_key search(const std::string &descriptor); - security_key insert(const std::string &descriptor, SecurityType); + security_key search(const persistence::StoreRow &row); + security_key insert(const persistence::StoreRow &row); }; -SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params) - : _self(new SecurityIDRecorderState(params)) { - if (params.db) { - _self->db = std::move(params.db); - } +SecurityIDRecorder::SecurityIDRecorder( + std::unique_ptr store, + KeyLookupOption lookup_key, + utils::Dispatcher *dispatcher) + : _self(std::make_unique( + store == nullptr + ? std::make_unique("key_ids") + : std::move(store), + lookup_key)) { init(); - if (params.dispatcher) { - _self->run(*params.dispatcher); + if (dispatcher != nullptr) { + _self->run(*dispatcher); } } SecurityIDRecorder::~SecurityIDRecorder() {} -void SecurityIDRecorder::init() { - LOG(md, Info, "SecurityIDRecorder::init"); - spark::SparkDataDirectory data_dir; - const auto db_dir = data_dir.database_dir("md"); - const auto db_path = data_dir.database_file("md", "security_keys"); - bts_assert(db_path.parent_path() == db_dir); - - if (!boost::filesystem::exists(db_dir)) { - boost::filesystem::create_directories(db_dir); - } - - if (!_self->db) { - _self->db = sqlite::open_db(db_path.c_str()); - bts_assert(_self->db); - } - - // Set a long busy timeout - sqlite3_busy_timeout(_self->db.get(), 9999); - sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); - sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0"); - sqlite::exec_nofail( - _self->db, "CREATE TABLE IF NOT EXISTS store (descriptor STRING " - "NOT NULL PRIMARY KEY, type int);"); -} +void SecurityIDRecorder::init() {} void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}", @@ -139,7 +140,7 @@ void SecurityIDRecorder::assign_key( assign_key( security.exchange(), bts::securities::security_type(security), bts::securities::descriptor(security), - [&callback, msec = security]( + [callback, msec = security]( api::Venue, api::SecurityType, const std::string &, uint32_t id) mutable { msec.set_id(id); @@ -274,27 +275,25 @@ void SecurityIDRecorderState::work_db() { // SPARK-1446 Need to try to add existing securities to the db in case we // deleted them by hand vector> existing_securities; - vector> defer; - vector new_descriptors; + vector new_rows; size_t count = 0; - string descriptor; while (id_request_queue.try_pop(temp)) { - descriptor = temp->name; - - // If we're unable to create a descriptor try again later - if (descriptor == "") { - defer.push_back(std::move(temp)); - continue; + security_key key = invalid_security_key; + auto row = get_row(temp->venue, temp->type, temp->name); + try { + key = search(row); + } catch (const std::overflow_error &ex) { + LOG(persist, Error, + "SecurityIDRecorderState::work_db. Row search threw ex: {}", + std::string{ex.what()}); } - - security_key key = search(descriptor); if (key != invalid_security_key) { temp->callback(temp->venue, temp->type, temp->name, key); existing_securities.push_back(std::move(temp)); } else { // always push in parallel new_entries.push_back(std::make_pair(key, std::move(temp))); - new_descriptors.push_back(descriptor); + new_rows.push_back(std::move(row)); ++count; // Limit the amount of time we spend writing to sqlite during startup // to avoid blocking the heartbeat @@ -304,22 +303,14 @@ void SecurityIDRecorderState::work_db() { } } - for (auto &d : defer) { - LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}", - d->name); - id_request_queue.push(std::move(d)); - } - if (!new_entries.empty()) { - sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION"); - + store->insert_many_resolve_ids(new_rows); for (size_t x = 0; x < new_entries.size(); ++x) { auto &entry = new_entries[x]; - entry.first = insert(new_descriptors[x], entry.second->type); + auto &row = new_rows[x]; + entry.first = row.id; bts_assert_d(entry.first != invalid_security_key); } - - sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } if (!new_entries.empty()) { @@ -332,65 +323,27 @@ void SecurityIDRecorderState::work_db() { void SecurityIDRecorder::bulk_insert( const std::vector &data, const char *prefix) { - LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size()); - if (data.size() == 0) - return; - sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); - std::string descriptor; - { - auto insert_stmt = sqlite::prepare_query( - _self->db, "INSERT INTO security_keys (rowid, descriptor, type) " - "values (?, ?, ?)"); - for (const api::Security *sec : data) { - descriptor = prefix + bts::securities::unique_descriptor(*sec); - auto type = securitytype_of(*sec); - - sqlite3_bind_int(insert_stmt.get(), 1, sec->id()); - sqlite3_bind_text( - insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - sqlite3_bind_int(insert_stmt.get(), 3, (int)type); - sqlite3_step(insert_stmt.get()); - sqlite3_reset(insert_stmt.get()); - } + std::vector rows; + rows.reserve(data.size()); + for (const auto *s : data) { + rows.push_back(get_row(*s, prefix)); } - sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + _self->store->insert_many_resolve_ids(rows); } -security_key SecurityIDRecorderState::search(const std::string &descriptor) { - LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor); - auto select_stmt = sqlite::prepare_query( - db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); - sqlite3_bind_text( - select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - int result = sqlite3_step(select_stmt.get()); - if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_column_int(select_stmt.get(), 0); +security_key SecurityIDRecorderState::search(const persistence::StoreRow &row) { + auto [exists, rowid, val] = store->retrieve_one(row.ns, row.key); + if (exists) { + if (!val.empty()) { + bts_assert(stol(val) == rowid); + } + return system::math::checked_ltou(rowid); } return invalid_security_key; } -security_key SecurityIDRecorderState::insert( - const std::string &descriptor, SecurityType type) { - LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor); - auto insert_stmt = sqlite::prepare_query( - db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); - sqlite3_bind_text( - insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - sqlite3_bind_int(insert_stmt.get(), 2, (int)type); - int result = sqlite3_step(insert_stmt.get()); - if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_last_insert_rowid(db.get()); - } - // Just in case our insert failed due to duplicate row - auto key = search(descriptor); - if (key == invalid_security_key) { - LOG(md, Error, "Failed to assign security key. SQLite return code: {}", - result); - } - return key; +security_key SecurityIDRecorderState::insert(const persistence::StoreRow &row) { + return store->insert(row.ns, row.key, row.value); } } // namespace md diff --git a/src/lib_spark/spark/md/SecurityIDRecorder.h b/src/lib_spark/spark/md/SecurityIDRecorder.h index 5e343ef54..78b382b79 100644 --- a/src/lib_spark/spark/md/SecurityIDRecorder.h +++ b/src/lib_spark/spark/md/SecurityIDRecorder.h @@ -1,4 +1,5 @@ #pragma once +#include "spark/persistence/IStore.h" #ifndef _WIN32 #include "MDProducerHandle.h" #include "common.h" @@ -22,26 +23,15 @@ namespace md { typedef const std::function KeyLookup; typedef std::optional KeyLookupOption; -struct SecurityIDRecorderParams { - SecurityIDRecorderParams() - : dispatcher(), db(sqlite::null_db()), lookup_key() {} - SecurityIDRecorderParams(utils::Dispatcher &d) - : dispatcher(&d), db(sqlite::null_db()), lookup_key() {} - SecurityIDRecorderParams(utils::Dispatcher &d, KeyLookup &get_key) - : dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {} - SecurityIDRecorderParams(sqlite3_ptr &db) - : dispatcher(), db(std::move(db)), lookup_key() {} - SecurityIDRecorderParams(sqlite3_ptr &db, KeyLookup &get_key) - : dispatcher(), db(std::move(db)), lookup_key(get_key) {} - utils::Dispatcher *dispatcher; - sqlite3_ptr db; - KeyLookupOption lookup_key; -}; + struct SecurityIDRecorderState; class SecurityIDRecorder { public: - SecurityIDRecorder(SecurityIDRecorderParams); + SecurityIDRecorder( + std::unique_ptr store = nullptr, + KeyLookupOption lookup_key = std::nullopt, + utils::Dispatcher *dispatcher = nullptr); ~SecurityIDRecorder(); typedef std::function Callback; diff --git a/src/lib_spark/spark/md/SymbolLoader.h b/src/lib_spark/spark/md/SymbolLoader.h index 66a778d8a..e8fb46493 100644 --- a/src/lib_spark/spark/md/SymbolLoader.h +++ b/src/lib_spark/spark/md/SymbolLoader.h @@ -40,9 +40,6 @@ class SymbolLoader { SymbolLoader(const SymbolLoader &); SymbolLoader &operator=(const SymbolLoader &); - static constexpr bool venue_has_synths(api::Venue venue); - static constexpr bool venue_has_synths(domain::protobufs::Venue venue); - static void add_symbols( const google::protobuf::RepeatedPtrField &symbols, Venue exchange, @@ -64,22 +61,6 @@ class SymbolLoader { // Definition ////////////////////////////////////////////////// -constexpr bool SymbolLoader::venue_has_synths(api::Venue venue) { - switch (venue) { - case api::VENUE_SIM: - case api::VENUE_SIMICE: - case api::VENUE_ICE: - case api::VENUE_GLBX: - return true; - default: - return false; - } -} - -constexpr bool SymbolLoader::venue_has_synths(domain::protobufs::Venue venue) { - return venue_has_synths(domain::DomainSecurityUtils::convert(venue)); -} - inline void SymbolLoader::add_symbols( const google::protobuf::RepeatedPtrField &symbols, Venue exchange, @@ -105,9 +86,7 @@ inline std::set SymbolLoader::load( const bts::domain::SecurityCriteria &include, const bts::domain::SecurityCriteria &exclude) { std::set result; - load(result, include); - if (exclude.is_defined()) { - } + load(result, include, exclude); return result; } @@ -142,8 +121,7 @@ inline std::size_t SymbolLoader::load( (exclude_none || !exclude.includes(venue, SEC_TYPE(SPREAD)))) { add_symbols(vcfg.spread(), venue, SEC_TYPE(SPREAD), result); } - if ((include_all || (venue_has_synths(venue) && - include.includes(venue, SEC_TYPE(SYNTHETIC)))) && + if ((include_all || (include.includes(venue, SEC_TYPE(SYNTHETIC)))) && (exclude_none || !exclude.includes(venue, SEC_TYPE(SYNTHETIC)))) { add_symbols(vcfg.spot(), VENUE(BTS), SEC_TYPE(SYNTHETIC), result); add_symbols(vcfg.future(), VENUE(BTS), SEC_TYPE(SYNTHETIC), result); diff --git a/src/lib_spark/spark/md/SyntheticSecurityService.cpp b/src/lib_spark/spark/md/SyntheticSecurityService.cpp index fe7416b84..f5df5cd2e 100644 --- a/src/lib_spark/spark/md/SyntheticSecurityService.cpp +++ b/src/lib_spark/spark/md/SyntheticSecurityService.cpp @@ -35,8 +35,9 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher struct SyntheticSecurityService::Pimpl { Pimpl(const MetadataStatefulPublisher &fn) - : recorder(utils::Dispatcher::background()), synthetics(), on_security(), - meta(), on_metadata(), metadata_publisher(fn) {} + : recorder(nullptr, std::nullopt, &utils::Dispatcher::background()), + synthetics(), on_security(), meta(), on_metadata(), + metadata_publisher(fn) {} SecurityIDRecorder recorder; std::unordered_map synthetics; utils::Event on_security; diff --git a/src/lib_spark/spark/md/shm/fixed_security.cpp b/src/lib_spark/spark/md/shm/fixed_security.cpp index ef62d21ae..6215c264c 100644 --- a/src/lib_spark/spark/md/shm/fixed_security.cpp +++ b/src/lib_spark/spark/md/shm/fixed_security.cpp @@ -219,26 +219,12 @@ void convert_security(const FixedSecurity &source, Security *presult) { get_contract_spec(source, *dest.mutable_contract_spec()); } -SecurityType securitytype_of_contractcase(Security::ContractCase cc) { - switch (cc) { - case Security::ContractCase::kFuture: - return SECURITYTYPE_FUTURE; - case Security::ContractCase::kOption: - return SECURITYTYPE_OPTION; - case Security::ContractCase::kSpot: - return api::SECURITYTYPE_SPOT; - case Security::ContractCase::kSpread: - return SECURITYTYPE_SPREAD; - case Security::ContractCase::kSynthetic: - return SECURITYTYPE_SYNTHETIC; - case Security::ContractCase::CONTRACT_NOT_SET: - bts_assert(cc != Security::ContractCase::CONTRACT_NOT_SET); - } - bts_assert(false); -} SecurityType securitytype_of(const Security &s) { - return securitytype_of_contractcase(s.Contract_case()); + auto type = bts::securities::security_type(s.Contract_case()); + bts_assert(type != api::SecurityType::SECURITYTYPE_UNSET); + return type; } + size_t num_legs(const Security &sec) { if (sec.Contract_case() == api::Security::ContractCase::kSpread) { return sec.spread().legs_size(); @@ -286,7 +272,7 @@ uint64_t exchange_id(const Security &security) { void convert_security( const Security &src, FixedSecurity &dest, bool set_security_key) { dest.key = set_security_key ? src.id() : invalid_security_key; - dest.type = securitytype_of_contractcase(src.Contract_case()); + dest.type = bts::securities::security_type(src.Contract_case()); dest.symbol(src.symbol()); dest.currency(src.currency()); dest.exchange = src.exchange(); diff --git a/src/lib_spark/spark/md/shm/fixed_security.h b/src/lib_spark/spark/md/shm/fixed_security.h index 29dbbfc82..ee9bb5448 100644 --- a/src/lib_spark/spark/md/shm/fixed_security.h +++ b/src/lib_spark/spark/md/shm/fixed_security.h @@ -215,7 +215,6 @@ void convert_security( FixedSecurity &dest, bool set_security_key = false); -SecurityType securitytype_of_contractcase(api::Security::ContractCase cc); SecurityType securitytype_of(const api::Security &); size_t num_legs(const api::Security &); uint64_t exchange_id(const api::Security &security); diff --git a/src/lib_spark/spark/md/test/md2_tests.cpp b/src/lib_spark/spark/md/test/md2_tests.cpp index bea27c3b7..d3b4a954d 100644 --- a/src/lib_spark/spark/md/test/md2_tests.cpp +++ b/src/lib_spark/spark/md/test/md2_tests.cpp @@ -8,6 +8,7 @@ #include "spark/md/PriceSize.h" #include "spark/md/relay/query_server.h" #include "spark/metrics/metrics.h" +#include "spark/persistence/SqliteStore.h" #include "spark/utils/Dispatcher.h" #include "spark/utils/Shuffle.h" #include "spark/utils/bts_chrono.h" @@ -159,12 +160,10 @@ class Co2Tests { #ifndef _WIN32 void test_security_id_recorder(const std::string &db_name) { // Verify callbacks happen after work complete - auto db1 = sqlite::open_db(db_name.c_str()); - REQUIRE(db1); - auto db2 = sqlite::open_db(db_name.c_str()); - REQUIRE(db2); - - SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}}; + SecurityIDRecorder idsvc{ + std::make_unique(db_name)}; + SecurityIDRecorder idsvc2{ + std::make_unique(db_name)}; security_key result = invalid_security_key; Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850); Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20); @@ -214,18 +213,18 @@ class Co2Tests { std::make_unique(s1, cold_header, hot_header); // Add spread to SecurityIDRecorder and work it - auto db = sqlite::open_db(":memory:"); + auto db = std::make_unique(":memory:"); REQUIRE(db); SecurityIDRecorder idsvc{ - {db, [&](ExchangeKey k) -> SecurityKey { - for (const auto &fut : - {f1handle->producer(), f2handle->producer()}) { - if (k.value() == fut.security().numeric_id) - return SecurityKey(fut.key()); - } - return SecurityKey(invalid_security_key); - }}}; + std::move(db), [&](ExchangeKey k) -> SecurityKey { + for (const auto &fut : + {f1handle->producer(), f2handle->producer()}) { + if (k.value() == fut.security().numeric_id) + return SecurityKey(fut.key()); + } + return SecurityKey(invalid_security_key); + }}; auto workwork = [&]() { idsvc.do_work(); diff --git a/src/lib_spark/spark/persistence/IDGenerator.cpp b/src/lib_spark/spark/persistence/IDGenerator.cpp index 0244c655e..c3eee227b 100644 --- a/src/lib_spark/spark/persistence/IDGenerator.cpp +++ b/src/lib_spark/spark/persistence/IDGenerator.cpp @@ -7,6 +7,7 @@ #include "spark/spark_data_directory.h" #include "spark/utils/bts_assert.h" #include "system/logging/logging.h" +#include "system/math/checked_arithmetic.h" #include #include @@ -66,24 +67,22 @@ std::optional IDGenerator::get_id( bool add_if_new) { std::string ns = fmt::format( "{}.{}", Venue_Name(venue).substr(6), - SecurityType_Name(type).substr(12)); + SecurityType_Name(type).substr(14)); return get_id(ns, name, add_if_new); } std::optional IDGenerator::get_id( const std::string &ns, const std::string &key, bool add_if_new) { std::lock_guard myLock(_instance_mx); - auto [exists, val] = me->kv_store->retrieve_one(ns, key); + auto [exists, rowid, val] = me->kv_store->retrieve_one(ns, key); if (exists) { - uint64_t rv = str_to_id(val); - LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rv); - return std::optional(rv); + LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rowid); + return std::optional(rowid); } else { if (add_if_new) { auto all = me->kv_store->retrieve_all(); - uint64_t new_id = get_next_id(all); - me->kv_store->insert(ns, key, std::to_string(new_id)); + auto new_id = me->kv_store->insert(ns, key, ""); LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, new_id); return std::optional(new_id); @@ -94,13 +93,6 @@ std::optional IDGenerator::get_id( } } -std::uint64_t IDGenerator::get_or_add( - const std::string &ns, const std::string &key) { - auto id = get_id(ns, key, true); - bts_assert(id.has_value()); - return *id; -} - std::uint64_t IDGenerator::get_or_add( domain::protobufs::Venue venue, domain::protobufs::SecurityType type, diff --git a/src/lib_spark/spark/persistence/IStore.h b/src/lib_spark/spark/persistence/IStore.h index 1f33ddf28..04f32a6f5 100644 --- a/src/lib_spark/spark/persistence/IStore.h +++ b/src/lib_spark/spark/persistence/IStore.h @@ -1,8 +1,12 @@ #pragma once +#include "spark/persistence/store_row.h" +#include #include #include #include +#include +#include namespace bts { namespace persistence { @@ -11,17 +15,45 @@ class IStore { public: virtual std::map> retrieve_all() = 0; - virtual std::pair - retrieve_one(std::string const &ns, std::string const &k) = 0; + + /** + * Finds the row with the specified namespace and key. + * + * @return std::tuple The first item indicates + * whether or not a row was found. For successful queries, the second and third + * items are the rowid and value, respectively. + */ + virtual std::tuple retrieve_one( + std::string const &ns, std::string const &k) = 0; virtual void remove(std::string const &ns, std::set const &keys) = 0; virtual void remove_all(std::string const &ns) = 0; - virtual void insert( + virtual std::int64_t insert( std::string const &ns, std::string const &k, std::string const &v) = 0; + + void insert_many(const std::vector &rows); + void insert_many_resolve_ids(std::vector &rows); + virtual void update( std::string const &ns, std::string const &k, std::string const &v) = 0; virtual ~IStore(); }; +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline void IStore::insert_many(const std::vector &rows) { + for (const auto &row : rows) { + insert(row.ns, row.key, row.value); + } +} + +inline void IStore::insert_many_resolve_ids(std::vector &rows) { + for (auto &row : rows) { + row.id = insert(row.ns, row.key, row.value); + } +} + } // namespace persistence } // namespace bts diff --git a/src/lib_spark/spark/persistence/SqliteStore.cpp b/src/lib_spark/spark/persistence/SqliteStore.cpp index a14c41f57..e4c21cd52 100644 --- a/src/lib_spark/spark/persistence/SqliteStore.cpp +++ b/src/lib_spark/spark/persistence/SqliteStore.cpp @@ -1,16 +1,10 @@ #include "SqliteStore.h" -#include "spark/config/config.h" #include "spark/persistence/persistence_utils.h" #include "spark/spark_data_directory.h" #include "spark/utils/Sqlite_utils.h" #include "spark/utils/bts_assert.h" -#include "spark/utils/bts_chrono.h" -#include "spark/utils/bts_no_copy.h" -#include "spark/utils/defer.h" -#include "spark/utils/file_utils.h" #include "system/logging/logging.h" -#include "system/os.h" #include "system/protobufs/config/system_config.pb.h" #include @@ -36,82 +30,95 @@ static boost::filesystem::path get_db_path(const std::string &str) { } struct SqliteStore::PImpl { + std::string const db; + boost::filesystem::path db_path; + sqlite3_ptr conn{nullptr, sqlite3_close}; + sqlite3_stmt_ptr insert{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr insert_many{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr insert_with_id{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr remove{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr remove_all{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr update{nullptr, sqlite3_finalize}; + PImpl(const boost::filesystem::path &db_path) - : db(db_path.stem().string()), db_path(db_path), - conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize), - remove(nullptr, sqlite3_finalize), - remove_all(nullptr, sqlite3_finalize), - update(nullptr, sqlite3_finalize) { + : db(db_path.stem().string()), db_path(db_path) { bts_assert(db.size()); } PImpl(std::string const &store_root, std::string const &db_name) : PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {} - std::string const db; - boost::filesystem::path db_path; - sqlite3_ptr conn; - sqlite3_stmt_ptr insert; - sqlite3_stmt_ptr remove; - sqlite3_stmt_ptr remove_all; - sqlite3_stmt_ptr update; - BTS_NO_COPY(PImpl); + explicit PImpl(sqlite3_ptr conn) : db(), db_path(), conn(std::move(conn)) {} + + PImpl(const PImpl &) = delete; + PImpl &operator=(const PImpl &) = delete; }; SqliteStore::SqliteStore(std::string const &db) : me(std::make_unique(get_db_path(db))) { - LOG(persist, Info, "SqliteStore ctr. db: {}; db_path: {}", db, me->db_path); + LOG(persist, Info, "SqliteStore ctr. db: {}; db_path: {}", db, + me->db_path.string()); + init_db(); +} + +SqliteStore::SqliteStore(sqlite3_ptr db) + : me(std::make_unique(std::move(db))) { + LOG(persist, Info, "SqliteStore ctr. db: "); init_db(); } void SqliteStore::init_db() { - auto path = me->db_path; - auto parent = path.parent_path(); + const auto path = me->db_path; - if (!boost::filesystem::is_directory(parent)) - bts_assert(boost::filesystem::create_directories(parent)); - - if (!boost::filesystem::exists(path)) { - LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", - me->db, path.string()); - - me->conn = sqlite::open_db(path.c_str()); - bts_assert_msg(me->conn, path.c_str()); - - const char *create_schema_sql = - "create table store ( ns text, key text, value text)"; - sqlite::exec_nofail(me->conn, create_schema_sql); - - const char *key_index_sql = - "create unique index key_index on store(ns,key)"; - sqlite::exec_nofail(me->conn, key_index_sql); - - const char *ns_index_sql = "create index ns_index on store(ns)"; - sqlite::exec_nofail(me->conn, ns_index_sql); - } else { - me->conn = sqlite::open_db(path.c_str()); + if (!path.empty()) { + if (!boost::filesystem::is_directory(path.parent_path())) { + bts_assert(boost::filesystem::create_directories(path.parent_path())); + } + if (!boost::filesystem::exists(path)) { + LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", + me->db, path.string()); + } } + auto &conn = me->conn; + if (conn == nullptr) { + conn = sqlite::open_db(path.c_str()); + bts_assert(conn != nullptr); + } + sqlite::exec_nofail( + conn, + "CREATE TABLE IF NOT EXISTS store (ns TEXT, key TEXT, value TEXT)"); + sqlite::exec_nofail( + conn, "CREATE UNIQUE INDEX IF NOT EXISTS key_index on store(ns, key)"); + sqlite::exec_nofail( + conn, "CREATE INDEX IF NOT EXISTS ns_index on store(ns)"); + conn = sqlite::open_db(path.c_str()); + // Asynchronous for speed - sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;"); - sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL"); + sqlite::exec_nofail(conn, "PRAGMA synchronous = 0;"); + sqlite::exec_nofail(conn, "PRAGMA journal_mode=WAL"); // prepare insert statement const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)"; - me->insert = sqlite::prepare_query(me->conn, insert_sql); + me->insert = sqlite::prepare_query(conn, insert_sql); + + // prepare insert_with_id statement + const char *insert_with_id_sql = + "insert into store (rowid, ns, key, value) values (?,?,?,?)"; + me->insert_with_id = sqlite::prepare_query(conn, insert_with_id_sql); // prepare update statement const char *update_sql = "update store set value = ? where ns = ? and key = ?"; - me->update = sqlite::prepare_query(me->conn, update_sql); + me->update = sqlite::prepare_query(conn, update_sql); // prepare remove statement const char *remove_all_sql = "delete from store where ns = ?"; - me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql); + me->remove_all = sqlite::prepare_query(conn, remove_all_sql); // prepare remove statement const char *remove_sql = "delete from store where ns = ? and key = ?"; - me->remove = sqlite::prepare_query(me->conn, remove_sql); + me->remove = sqlite::prepare_query(conn, remove_sql); } SqliteStore::~SqliteStore() {} @@ -140,25 +147,27 @@ SqliteStore::Data SqliteStore::retrieve_all() { return result; } -std::pair SqliteStore::retrieve_one( +std::tuple SqliteStore::retrieve_one( std::string const &ns, std::string const &k) { auto query_str = fmt::format( - "select ns,key,value from store where ns='{}' and key='{}'", ns, k); + "select rowid,ns,key,value from store where ns='{}' and key='{}'", ns, + k); sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str()); - std::pair result{false, ""}; - + auto result = std::make_tuple(false, 0, ""); auto pq = query.get(); auto sqlite_result = sqlite3_step(pq); if (sqlite_result == SQLITE_ROW) { + std::int64_t rowid = sqlite3_column_int64(pq, 0); std::string ns( - (char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0)); - std::string key( (char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1)); + std::string key( + (char *)sqlite3_column_text(pq, 2), sqlite3_column_bytes(pq, 2)); std::string val( - (char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2)); - result.first = true; - result.second = std::move(val); + (char *)sqlite3_column_blob(pq, 3), sqlite3_column_bytes(pq, 3)); + std::get<0>(result) = true; + std::get<1>(result) = rowid; + std::get<2>(result) = std::move(val); } return result; } @@ -167,91 +176,185 @@ void SqliteStore::remove( std::string const &ns, std::set const &keys) { bts_assert(ns.size()); - for (auto &k : keys) { - SQLITE_CHK( - sqlite3_bind_text( - me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->remove.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not delete row in store. Will retry once. error_code: {} " - "ns:{}, k: {}", - r, ns, k); - sleep_for(2s); - r = sqlite3_step(me->remove.get()); + try { + for (auto &k : keys) { + SQLITE_CHK( + sqlite3_bind_text( + me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->remove.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to delete row. Bailing out. error_code: " - "{}", - r); - bts_assert(r == SQLITE_DONE); + "Could not delete row in store. Will retry once. error_code: " + "{} " + "ns:{}, k: {}", + r, ns, k); + sleep_for(2s); + r = sqlite3_step(me->remove.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to delete row. Bailing out. " + "error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); } - SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::remove threw. what: {}", + std::string{ex.what()}); } } void SqliteStore::remove_all(std::string const &ns) { bts_assert(ns.size()); - SQLITE_CHK( - sqlite3_bind_text( - me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->remove_all.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not delete all rows in store. Will retry once. error_code: {} " - "ns:{}", - r, ns); - sleep_for(2s); - r = sqlite3_step(me->remove_all.get()); + try { + SQLITE_CHK( + sqlite3_bind_text( + me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->remove_all.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to delete all rows. Bailing out. " - "error_code: " - "{}", - r); - bts_assert(r == SQLITE_DONE); + "Could not delete all rows in store. Will retry once. error_code: " + "{} " + "ns:{}", + r, ns); + sleep_for(2s); + r = sqlite3_step(me->remove_all.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to delete all rows. Bailing out. " + "error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::remove_all threw. what: {}", + std::string{ex.what()}); } - SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); } -void SqliteStore::insert( +std::int64_t SqliteStore::insert( std::string const &ns, std::string const &k, std::string const &v) { - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->insert.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not insert row in store. Will retry once. error_code: {} ns: " - "{}, k: {}, v: {}", - r, ns, k, v); - sleep_for(2s); - r = sqlite3_step(me->insert.get()); + try { + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->insert.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to insert row. Bailing out. error_code: {}", - r); - bts_assert(r == SQLITE_DONE); + "Could not insert row in store. Will retry once. error_code: {} " + "ns: " + "{}, k: {}, v: {}", + r, ns, k, v); + sleep_for(2s); + r = sqlite3_step(me->insert.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to insert row. Bailing out. error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + const auto rowid = sqlite3_last_insert_rowid(me->conn.get()); + SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); + return rowid; + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::insert threw. what: {}", + std::string{ex.what()}); } - SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); + return 0; +} + +void SqliteStore::insert_many(const std::vector &rows) { + LOG(md, Info, "SqliteStore::insert_many size: {}", rows.size()); + if (rows.empty()) { + return; + } + try { + sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION"); + sqlite3 *conn_rawp = me->conn.get(); + sqlite3_stmt *insert_rawp = me->insert.get(); + sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get(); + for (const auto &row : rows) { + int col_idx = 1; + sqlite3_stmt *stmt; + if (row.id > 0) { + stmt = insert_with_id_rawp; + sqlite3_bind_int(stmt, col_idx++, row.id); + } else { + stmt = insert_rawp; + } + sqlite3_bind_text( + stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC); + sqlite3_bind_text( + stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC); + int result = sqlite3_step(stmt); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + sqlite3_last_insert_rowid(conn_rawp); + } + sqlite3_reset(stmt); + } + sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION"); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::insert_many threw. what: {}", + std::string{ex.what()}); + } +} + +void SqliteStore::insert_many_resolve_ids(std::vector &rows) { + LOG(md, Info, "SqliteStore::insert_many_resolve_ids size: {}", rows.size()); + if (rows.empty()) { + return; + } + sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION"); + sqlite3 *conn_rawp = me->conn.get(); + sqlite3_stmt *insert_rawp = me->insert.get(); + sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get(); + for (auto &row : rows) { + int col_idx = 1; + sqlite3_stmt *stmt; + if (row.id > 0) { + stmt = insert_with_id_rawp; + sqlite3_bind_int(stmt, col_idx++, row.id); + } else { + stmt = insert_rawp; + } + sqlite3_bind_text( + stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC); + sqlite3_bind_text( + stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC); + int result = sqlite3_step(stmt); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + if (row.id > 0) { + bts_assert(sqlite3_last_insert_rowid(conn_rawp) == row.id); + } else { + row.id = sqlite3_last_insert_rowid(conn_rawp); + } + } + sqlite3_reset(stmt); + } + sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION"); } void SqliteStore::update( diff --git a/src/lib_spark/spark/persistence/SqliteStore.h b/src/lib_spark/spark/persistence/SqliteStore.h index 4f0caf725..96fc0fb66 100644 --- a/src/lib_spark/spark/persistence/SqliteStore.h +++ b/src/lib_spark/spark/persistence/SqliteStore.h @@ -1,7 +1,12 @@ #pragma once #include "IStore.h" +#include "spark/persistence/store_row.h" +#include "spark/utils/Sqlite_utils.h" + +#include #include +#include namespace bts { namespace persistence { @@ -9,22 +14,26 @@ namespace persistence { class SqliteStore : public IStore { public: SqliteStore(std::string const &db); + explicit SqliteStore(sqlite3_ptr db); ~SqliteStore(); static SqliteStore from_path(const std::string &full_path); typedef std::map> Data; Data retrieve_all() override; - std::pair retrieve_one( + + std::tuple retrieve_one( std::string const &ns, std::string const &k) override; void remove( std::string const &ns, std::set const &keys) override; void remove_all(std::string const &ns) override; - void insert( + std::int64_t insert( std::string const &ns, std::string const &k, std::string const &v) override; + void insert_many(const std::vector &rows); + void insert_many_resolve_ids(std::vector &rows); void update( std::string const &ns, std::string const &k, diff --git a/src/lib_spark/spark/persistence/store_row.h b/src/lib_spark/spark/persistence/store_row.h new file mode 100644 index 000000000..1c7b7f079 --- /dev/null +++ b/src/lib_spark/spark/persistence/store_row.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace bts::persistence { + +struct StoreRow { + std::string ns; + std::string key; + std::string value; + /** + * Represents the unique 'rowid' in the sqlite db. A value of 0 indicates that + * the rowid isn't known, and should either be generated by the db (for inserts), + * or resolved by finding the row with matching ns, key, value (for retrieves). + */ + std::int64_t id{0}; +}; + +} // namespace bts::persistence diff --git a/src/lib_system/system/math/checked_arithmetic.h b/src/lib_system/system/math/checked_arithmetic.h new file mode 100644 index 000000000..91e60a6b9 --- /dev/null +++ b/src/lib_system/system/math/checked_arithmetic.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +namespace bts::system::math { + +static std::uint32_t checked_ltou(std::int64_t l); +static std::uint64_t checked_ltoul(std::int64_t l); + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::uint32_t checked_ltou(std::int64_t l) { + if (l < 0) { + throw std::overflow_error{ + "Failed to convert int64 to uint32 (value < 0)"}; + } + if (l > + static_cast(std::numeric_limits::max())) { + throw std::overflow_error{ + "Failed to convert int64 to uint32. value > max)"}; + } + return static_cast(l); +} + +inline std::uint64_t checked_ltoul(std::int64_t l) { + if (l < 0) { + throw std::overflow_error{ + "Failed to convert int64 to uint64 (value < 0)"}; + } + return static_cast(l); +} + +} // namespace bts::system::math diff --git a/src/lib_system/system/time_span.h b/src/lib_system/system/time_span.h index 3b8c8c03c..9b345082b 100644 --- a/src/lib_system/system/time_span.h +++ b/src/lib_system/system/time_span.h @@ -50,7 +50,9 @@ class TimeSpan { constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept; +constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept; +constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept; std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept; @@ -116,10 +118,18 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept { return lhs.nanos() < rhs.nanos(); } +constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept { + return lhs.nanos() <= rhs.nanos(); +} + constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept { return lhs.nanos() > rhs.nanos(); } +constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept { + return lhs.nanos() >= rhs.nanos(); +} + } // namespace bts::core FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string());