diff --git a/src/app_sparkd/sparkd/analytics/AnalyticsServiceServer.cpp b/src/app_sparkd/sparkd/analytics/AnalyticsServiceServer.cpp index 2bee34c89..90e319c58 100644 --- a/src/app_sparkd/sparkd/analytics/AnalyticsServiceServer.cpp +++ b/src/app_sparkd/sparkd/analytics/AnalyticsServiceServer.cpp @@ -50,7 +50,7 @@ void add_or_update_to_table( const std::string &ns, const std::string &key, const std::string &val) { - auto [exists, existing_val] = store->retrieve_one(ns, key); + auto [exists, rowid, existing_val] = store->retrieve_one(ns, key); LOG(Gen, Info, "add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns, key, val, exists, existing_val); @@ -384,7 +384,8 @@ void AnalyticsServiceServer::save_report( const api::Response<::bts::api::ResultMessage> &resp) { Report report = report_arg; std::string key = std::to_string(report.id()); - auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key); + auto [present, rowid, val] = + me->reports_store->retrieve_one(me->reports_ns, key); if (!present) { int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1; report.set_id(new_id); @@ -406,7 +407,8 @@ void AnalyticsServiceServer::delete_report( const bts::api::analytics::IntWrapper &id, const api::Response<::bts::api::ResultMessage> &resp) { std::string key = std::to_string(id.data()); - auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key); + auto [present, rowid, val] = + me->reports_store->retrieve_one(me->reports_ns, key); api::ResultMessage res; res.set_success(false); if (!present) { diff --git a/src/app_sparkd/sparkd/exchange_sim/ExchangeSimApplication.cpp b/src/app_sparkd/sparkd/exchange_sim/ExchangeSimApplication.cpp index 1fd54bf43..01e879fa8 100644 --- a/src/app_sparkd/sparkd/exchange_sim/ExchangeSimApplication.cpp +++ b/src/app_sparkd/sparkd/exchange_sim/ExchangeSimApplication.cpp @@ -582,10 +582,11 @@ void negate_id(api::Security *sec) { } void ExchangeSimApplication::persist_book(OrderBook const &b) { - auto existing = me->store->retrieve_one("books", b.sec_desc()); + auto [exists, rowid, oldval] = + me->store->retrieve_one("books", b.sec_desc()); auto val = utils::pb::to_string(b); - if (existing.first) + if (exists) me->store->update("books", b.sec_desc(), val); else me->store->insert("books", b.sec_desc(), val); diff --git a/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitData.h b/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitData.h index 45ec10353..2705b24dc 100644 --- a/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitData.h +++ b/src/app_sparkd/sparkd/md/emumba/EmumbaDeribitData.h @@ -18,10 +18,11 @@ struct EmumbaDeribitData { EmumbaDeribitData() : sec_index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(sec_index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(sec_index.get_key(k.value())); + }, + &utils::Dispatcher::current()), md_producer(recorder), listener(sec_index, md_producer) {} }; } // namespace bts::md diff --git a/src/app_sparkd/sparkd/md/platforms/emumba/EmumbaMain.cpp b/src/app_sparkd/sparkd/md/platforms/emumba/EmumbaMain.cpp index 96533c9f4..55831df30 100644 --- a/src/app_sparkd/sparkd/md/platforms/emumba/EmumbaMain.cpp +++ b/src/app_sparkd/sparkd/md/platforms/emumba/EmumbaMain.cpp @@ -43,10 +43,11 @@ struct EmumbaMain::PImpl { PImpl() : sec_index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(sec_index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(sec_index.get_key(k.value())); + }, + &utils::Dispatcher::current()), producer(recorder), listener(sec_index, producer) { const auto config_path = Path{config::get().config_path()} / diff --git a/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp b/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp index 1996c47ed..87e895d8c 100644 --- a/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp +++ b/src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp @@ -38,10 +38,11 @@ struct ICEMain::PImpl { "onixs_ice" / this->options.ice_config_file()) .string()), index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(index.get_key(k.value())); + }, + &utils::Dispatcher::current()), producer(recorder), handlers(), listeners() {} IceChannelConfig channel_settings_for(const ICEChannel &channel) { diff --git a/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index a785f2a2a..fa8fda27e 100644 --- a/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -43,10 +43,11 @@ struct MDP3Main::PImpl { : data_path(spark::SparkDataDirectory{}.root() / "cme_mdp3/"), log_path(spark::SparkDataDirectory{}.log_dir() / "onixs/"), index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(index.get_key(k.value())); + }, + &utils::Dispatcher::current()), producer(recorder), engines(), handlers(), listeners(), recovery() {} }; diff --git a/src/app_sparkd/sparkd/position/PositionServiceServer.cpp b/src/app_sparkd/sparkd/position/PositionServiceServer.cpp index aae09b1ff..45da18af1 100644 --- a/src/app_sparkd/sparkd/position/PositionServiceServer.cpp +++ b/src/app_sparkd/sparkd/position/PositionServiceServer.cpp @@ -293,9 +293,9 @@ void PositionServiceServer::update_manual_fill( auto *this_resp = resp.add_response(); auto id = std::to_string(mf.record_id()); - auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id); - auto found = p.first; - if (!found) { + auto [exists, rowid, val] = + me->manual_fills_store->retrieve_one(manual_fills_ns, id); + if (!exists) { this_resp->set_ok(false); this_resp->set_msg("Could not find record"); } else { @@ -326,9 +326,9 @@ void PositionServiceServer::remove_manual_fill( auto *this_resp = resp.add_response(); auto id = std::to_string(mf.record_id()); - auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id); - auto found = p.first; - if (!found) { + auto [exists, rowid, val] = + me->manual_fills_store->retrieve_one(manual_fills_ns, id); + if (!exists) { this_resp->set_ok(false); this_resp->set_msg("Could not find record"); } else { @@ -367,10 +367,9 @@ void PositionServiceServer::update_security_parameters( req.op_params_size()); for (const auto &p : req.op_params()) { std::string key_str = std::to_string(p.security_key()); - auto db_p = + auto [exists, rowid, val] = me->security_params_store->retrieve_one(security_params_ns, key_str); - auto found = db_p.first; - if (!found) { + if (!exists) { me->security_params_store->insert( security_params_ns, key_str, utils::pb::to_string(p)); } else { diff --git a/src/app_sparkd/sparkd/user/AdminServiceDB.cpp b/src/app_sparkd/sparkd/user/AdminServiceDB.cpp index 037554423..7786aa705 100644 --- a/src/app_sparkd/sparkd/user/AdminServiceDB.cpp +++ b/src/app_sparkd/sparkd/user/AdminServiceDB.cpp @@ -20,7 +20,8 @@ AdminServiceDB::AdminServiceDB() : me(new PImpl()) {} AdminServiceDB::~AdminServiceDB() {} std::optional AdminServiceDB::retrieve() { - auto [has_table, data] = me->store.retrieve_one(table_name, unique_key); + auto [has_table, rowid, data] = + me->store.retrieve_one(table_name, unique_key); if (!has_table) return {}; @@ -32,7 +33,8 @@ std::optional AdminServiceDB::retrieve() { } bool AdminServiceDB::update(api::admin::AdminData const &state) { - auto [has_table, data] = me->store.retrieve_one(table_name, unique_key); + auto [has_table, rowid, data] = + me->store.retrieve_one(table_name, unique_key); UNUSED(data); std::string ser_state; diff --git a/src/lib_spark/spark/domain_model/Security.cpp b/src/lib_spark/spark/domain_model/Security.cpp index 41f476616..8675730b7 100644 --- a/src/lib_spark/spark/domain_model/Security.cpp +++ b/src/lib_spark/spark/domain_model/Security.cpp @@ -2,7 +2,7 @@ #include "spark/config/config.h" #include "spark/config/protos/orders_fills_cleanup.pb.h" #include "spark/domain_model/Security.pb.h" -#include "spark/md/invalid_security_key.h" +#include "spark/md/common.h" #include "spark/utils/datetime.h" #include diff --git a/src/lib_spark/spark/md/Diag/ShmDumper.cpp b/src/lib_spark/spark/md/Diag/ShmDumper.cpp index 0d7f23d37..e089863c3 100644 --- a/src/lib_spark/spark/md/Diag/ShmDumper.cpp +++ b/src/lib_spark/spark/md/Diag/ShmDumper.cpp @@ -59,85 +59,8 @@ string dump_header(const ColdHeader &header) { return JsonDumper::dump(header); } -void write_date_string( - uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result); string ShmDumper::dump_security(const FixedSecurity &security) { - ostringstream res; - res << "{"; - res << "'key': " << security.key << ", "; - res << "'symbol': '" << security.symbol() << "', "; - res << "'type': '" << api::SecurityType_Name(security.type) << "', "; - res << "'exchange': '" << api::Venue_Name(security.exchange) << "', "; - res << "'alt_symbol': '" << security.alt_symbol() << "', "; - res << "'cfi_code': '" << security.cfi_code() << "', "; - res << "'security_desc': '" << security.security_desc() << "', "; - res << "'platform_id': '" << security.platform_id() << "', "; - res << "'display_factor': " << (int)security.display_factor << ", "; - res << "'numeric_id': " << security.numeric_id << ", "; - res << "'cboe_class_key': " << security.cboe_class_key << ", "; - res << "'main_frac': " << (int)security.cme_main_fraction << ", "; - res << "'sub_fract': " << (int)security.cme_sub_fraction << ", "; - res << "'price_format': " << (int)security.cme_price_display_format << ", "; - res << "'market_segment': " << (int)security.cme_market_segment_id << ", "; - - res << "'price_break': " << security.price_break << ", "; - res << "'low_tick': " << security.low_tick << ", "; - res << "'high_tick': " << security.high_tick << ", "; - res << "'cpp': " << security.cash_per_point << ", "; - - res << "'market_description ': " << security.ice_market_description() - << ", "; - res << "'cleared_alias ': " << security.ice_cleared_alias() << ", "; - res << "'product_description ': " << security.ice_product_name() << ", "; - res << "'hub_alias ': " << security.ice_hub_alias() << ", "; - res << "'strip_name': " << security.ice_strip_name() << ", "; - res << "'mic_code': " << security.mic_code() << ", "; - - switch (security.type) { - case SECURITYTYPE_SPOT: - break; - case SECURITYTYPE_FUTURE: { - auto *psec = static_cast(&security); - res << "'expiration': '"; - write_date_string( - psec->expiration_year, psec->expiration_month, psec->expiration_day, - res); - res << "' "; - break; - } - case SECURITYTYPE_OPTION: { - auto *psec = static_cast(&security); - res << "'expiration': '"; - write_date_string( - psec->expiration_year, psec->expiration_month, psec->expiration_day, - res); - res << "', "; - res << "'strike': " << psec->strike << ", "; - res << "'callput': '" << api::PutOrCall_Name(psec->call_put) << "' "; - break; - } - case SECURITYTYPE_SPREAD: { - auto *psec = static_cast(&security); - res << "'legs': ["; - for (size_t x = 0; x < psec->leg_count(); ++x) { - auto &leg = psec->legs[x]; - if (x > 0) - res << ","; - res << "{ 'key': " << leg.key << ", 'exkey': " << leg.exchange_key - << ", 'quantity': " << leg.leg_quantity - << ", 'leg_price': " << leg.leg_price - << ", 'delta': " << leg.leg_delta << ", 'side': '" - << api::Side_Name(leg.leg_side) << "' }"; - } - res << "]"; - break; - } - default: - break; - } - - res << "}"; - return res.str(); + return JsonDumper::dump(security); } string dump(datetime::Date date) { diff --git a/src/lib_spark/spark/md/SecurityDB.cpp b/src/lib_spark/spark/md/SecurityDB.cpp index 6ca69761f..8ba0e8e7f 100644 --- a/src/lib_spark/spark/md/SecurityDB.cpp +++ b/src/lib_spark/spark/md/SecurityDB.cpp @@ -63,26 +63,6 @@ datetime::Date sec_expiration_cut_off() { return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day()); } -void apply_compatibilit_mutations(api::Security &security) { - // Migrate from old tick stuff - if (security.has_glbx()) { - if (security.has_glbx()) { - if (security.contract_spec().main_fraction() == 0) { - security.mutable_contract_spec()->set_main_fraction( - security.glbx().do_not_use_main_fraction()); - } - if (security.contract_spec().sub_fraction() == 0) { - security.mutable_contract_spec()->set_sub_fraction( - security.glbx().do_not_use_sub_fraction()); - } - if (security.contract_spec().price_display_format() == 0) { - security.mutable_contract_spec()->set_price_display_format( - security.glbx().do_not_use_price_display_format()); - } - } - } -} - size_t load_securities( std::function const &sym_filter, const std::function &callback) { @@ -106,7 +86,6 @@ size_t load_securities( if (sym_filter(us)) { auto blob = select_stmt.getColumn("security_blob"); if (security.ParseFromArray(blob, blob.getBytes())) { - apply_compatibilit_mutations(security); bts_assert(security_key == security.id()); count++; callback(security); @@ -148,7 +127,6 @@ std::shared_ptr find_security(SecurityKey key) { auto result = std::make_shared(); auto blob = select.getColumn(0); if (result->ParseFromArray(blob, blob.getBytes())) { - apply_compatibilit_mutations(*result); return result; } } diff --git a/src/lib_spark/spark/md/SecurityIDRecorder.cpp b/src/lib_spark/spark/md/SecurityIDRecorder.cpp index 50471849a..205a1a208 100644 --- a/src/lib_spark/spark/md/SecurityIDRecorder.cpp +++ b/src/lib_spark/spark/md/SecurityIDRecorder.cpp @@ -2,22 +2,18 @@ #include "SecurityIDRecorder.h" #include "spark/domain_model/Security.h" #include "spark/domain_model/Security.pb.h" -#include "spark/md/invalid_security_key.h" -#include "spark/spark_data_directory.h" -#include - -#include "spark/config/config.h" +#include "spark/md/common.h" +#include "spark/persistence/SqliteStore.h" #include "spark/utils/Sqlite_utils.h" #include "system/logging/logging.h" +#include "system/math/checked_arithmetic.h" #include -#include - -#include "spark/config/protos/market_data.pb.h" -#include "system/protobufs/config/system_config.pb.h" - +#include #include +#include #include +#include using namespace std; using namespace bts::utils; @@ -29,10 +25,32 @@ namespace md { using bts::api::Security; using bts::api::SecurityType; using bts::api::Venue; -using bts::system::protobufs::SystemConfig; +using bts::persistence::IStore; +using bts::persistence::SqliteStore; uint64_t exchange_id(const api::Security &security); +static std::unique_ptr get_default_store() { + return std::make_unique("key_ids"); +} + +static persistence::StoreRow get_row( + Venue venue, + SecurityType type, + const std::string &name, + const std::string &prefix = "") { + return persistence::StoreRow{ + fmt::format( + "{}{}.{}", std::string{prefix}, Venue_Name(venue).substr(6), + SecurityType_Name(type).substr(13)), + name, ""}; +} + +static persistence::StoreRow get_row( + const api::Security &s, const std::string &prefix = "") { + return get_row(s.exchange(), bts::securities::security_type(s), prefix); +} + struct SecurityIdRequest { Venue venue; SecurityType type; @@ -42,14 +60,16 @@ struct SecurityIdRequest { }; struct SecurityIDRecorderState { - SecurityIDRecorderState(SecurityIDRecorderParams &p) - : pending_handle(), thread_id(), db(nullptr, sqlite3_close), timer(), - lookup_key(p.lookup_key), unresolved_legs() {} + SecurityIDRecorderState( + std::unique_ptr store, KeyLookupOption lookup_key) + : store(store == nullptr ? get_default_store() : std::move(store)), + pending_handle(), thread_id(), timer(), lookup_key(lookup_key), + unresolved_legs() {} + std::unique_ptr store{nullptr}; tbb::concurrent_queue> id_request_queue; tbb::concurrent_queue pending_handle; std::thread::id thread_id; - sqlite3_ptr db; unique_ptr timer; KeyLookupOption lookup_key; @@ -85,47 +105,28 @@ struct SecurityIDRecorderState { void check_access() { bts_assert(thread_id == std::this_thread::get_id()); } void add_unresolved_spread(MDProducerHandle &handle); - security_key search(const std::string &descriptor); - security_key insert(const std::string &descriptor, SecurityType); + security_key search(const persistence::StoreRow &row); + security_key insert(const persistence::StoreRow &row); }; -SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params) - : _self(new SecurityIDRecorderState(params)) { - if (params.db) { - _self->db = std::move(params.db); - } +SecurityIDRecorder::SecurityIDRecorder( + std::unique_ptr store, + KeyLookupOption lookup_key, + utils::Dispatcher *dispatcher) + : _self(std::make_unique( + store == nullptr + ? std::make_unique("key_ids") + : std::move(store), + lookup_key)) { init(); - if (params.dispatcher) { - _self->run(*params.dispatcher); + if (dispatcher != nullptr) { + _self->run(*dispatcher); } } SecurityIDRecorder::~SecurityIDRecorder() {} -void SecurityIDRecorder::init() { - LOG(md, Info, "SecurityIDRecorder::init"); - spark::SparkDataDirectory data_dir; - const auto db_dir = data_dir.database_dir("md"); - const auto db_path = data_dir.database_file("md", "security_keys"); - bts_assert(db_path.parent_path() == db_dir); - - if (!boost::filesystem::exists(db_dir)) { - boost::filesystem::create_directories(db_dir); - } - - if (!_self->db) { - _self->db = sqlite::open_db(db_path.c_str()); - bts_assert(_self->db); - } - - // Set a long busy timeout - sqlite3_busy_timeout(_self->db.get(), 9999); - sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); - sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0"); - sqlite::exec_nofail( - _self->db, "CREATE TABLE IF NOT EXISTS store (descriptor STRING " - "NOT NULL PRIMARY KEY, type int);"); -} +void SecurityIDRecorder::init() {} void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}", @@ -139,7 +140,7 @@ void SecurityIDRecorder::assign_key( assign_key( security.exchange(), bts::securities::security_type(security), bts::securities::descriptor(security), - [&callback, msec = security]( + [callback, msec = security]( api::Venue, api::SecurityType, const std::string &, uint32_t id) mutable { msec.set_id(id); @@ -274,27 +275,25 @@ void SecurityIDRecorderState::work_db() { // SPARK-1446 Need to try to add existing securities to the db in case we // deleted them by hand vector> existing_securities; - vector> defer; - vector new_descriptors; + vector new_rows; size_t count = 0; - string descriptor; while (id_request_queue.try_pop(temp)) { - descriptor = temp->name; - - // If we're unable to create a descriptor try again later - if (descriptor == "") { - defer.push_back(std::move(temp)); - continue; + security_key key = invalid_security_key; + auto row = get_row(temp->venue, temp->type, temp->name); + try { + key = search(row); + } catch (const std::overflow_error &ex) { + LOG(persist, Error, + "SecurityIDRecorderState::work_db. Row search threw ex: {}", + std::string{ex.what()}); } - - security_key key = search(descriptor); if (key != invalid_security_key) { temp->callback(temp->venue, temp->type, temp->name, key); existing_securities.push_back(std::move(temp)); } else { // always push in parallel new_entries.push_back(std::make_pair(key, std::move(temp))); - new_descriptors.push_back(descriptor); + new_rows.push_back(std::move(row)); ++count; // Limit the amount of time we spend writing to sqlite during startup // to avoid blocking the heartbeat @@ -304,22 +303,14 @@ void SecurityIDRecorderState::work_db() { } } - for (auto &d : defer) { - LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}", - d->name); - id_request_queue.push(std::move(d)); - } - if (!new_entries.empty()) { - sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION"); - + store->insert_many_resolve_ids(new_rows); for (size_t x = 0; x < new_entries.size(); ++x) { auto &entry = new_entries[x]; - entry.first = insert(new_descriptors[x], entry.second->type); + auto &row = new_rows[x]; + entry.first = row.id; bts_assert_d(entry.first != invalid_security_key); } - - sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } if (!new_entries.empty()) { @@ -332,65 +323,27 @@ void SecurityIDRecorderState::work_db() { void SecurityIDRecorder::bulk_insert( const std::vector &data, const char *prefix) { - LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size()); - if (data.size() == 0) - return; - sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); - std::string descriptor; - { - auto insert_stmt = sqlite::prepare_query( - _self->db, "INSERT INTO security_keys (rowid, descriptor, type) " - "values (?, ?, ?)"); - for (const api::Security *sec : data) { - descriptor = prefix + bts::securities::unique_descriptor(*sec); - auto type = securitytype_of(*sec); - - sqlite3_bind_int(insert_stmt.get(), 1, sec->id()); - sqlite3_bind_text( - insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - sqlite3_bind_int(insert_stmt.get(), 3, (int)type); - sqlite3_step(insert_stmt.get()); - sqlite3_reset(insert_stmt.get()); - } + std::vector rows; + rows.reserve(data.size()); + for (const auto *s : data) { + rows.push_back(get_row(*s, prefix)); } - sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + _self->store->insert_many_resolve_ids(rows); } -security_key SecurityIDRecorderState::search(const std::string &descriptor) { - LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor); - auto select_stmt = sqlite::prepare_query( - db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); - sqlite3_bind_text( - select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - int result = sqlite3_step(select_stmt.get()); - if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_column_int(select_stmt.get(), 0); +security_key SecurityIDRecorderState::search(const persistence::StoreRow &row) { + auto [exists, rowid, val] = store->retrieve_one(row.ns, row.key); + if (exists) { + if (!val.empty()) { + bts_assert(stol(val) == rowid); + } + return system::math::checked_ltou(rowid); } return invalid_security_key; } -security_key SecurityIDRecorderState::insert( - const std::string &descriptor, SecurityType type) { - LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor); - auto insert_stmt = sqlite::prepare_query( - db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); - sqlite3_bind_text( - insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - sqlite3_bind_int(insert_stmt.get(), 2, (int)type); - int result = sqlite3_step(insert_stmt.get()); - if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_last_insert_rowid(db.get()); - } - // Just in case our insert failed due to duplicate row - auto key = search(descriptor); - if (key == invalid_security_key) { - LOG(md, Error, "Failed to assign security key. SQLite return code: {}", - result); - } - return key; +security_key SecurityIDRecorderState::insert(const persistence::StoreRow &row) { + return store->insert(row.ns, row.key, row.value); } } // namespace md diff --git a/src/lib_spark/spark/md/SecurityIDRecorder.h b/src/lib_spark/spark/md/SecurityIDRecorder.h index 5e343ef54..78b382b79 100644 --- a/src/lib_spark/spark/md/SecurityIDRecorder.h +++ b/src/lib_spark/spark/md/SecurityIDRecorder.h @@ -1,4 +1,5 @@ #pragma once +#include "spark/persistence/IStore.h" #ifndef _WIN32 #include "MDProducerHandle.h" #include "common.h" @@ -22,26 +23,15 @@ namespace md { typedef const std::function KeyLookup; typedef std::optional KeyLookupOption; -struct SecurityIDRecorderParams { - SecurityIDRecorderParams() - : dispatcher(), db(sqlite::null_db()), lookup_key() {} - SecurityIDRecorderParams(utils::Dispatcher &d) - : dispatcher(&d), db(sqlite::null_db()), lookup_key() {} - SecurityIDRecorderParams(utils::Dispatcher &d, KeyLookup &get_key) - : dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {} - SecurityIDRecorderParams(sqlite3_ptr &db) - : dispatcher(), db(std::move(db)), lookup_key() {} - SecurityIDRecorderParams(sqlite3_ptr &db, KeyLookup &get_key) - : dispatcher(), db(std::move(db)), lookup_key(get_key) {} - utils::Dispatcher *dispatcher; - sqlite3_ptr db; - KeyLookupOption lookup_key; -}; + struct SecurityIDRecorderState; class SecurityIDRecorder { public: - SecurityIDRecorder(SecurityIDRecorderParams); + SecurityIDRecorder( + std::unique_ptr store = nullptr, + KeyLookupOption lookup_key = std::nullopt, + utils::Dispatcher *dispatcher = nullptr); ~SecurityIDRecorder(); typedef std::function Callback; diff --git a/src/lib_spark/spark/md/SymbolLoader.h b/src/lib_spark/spark/md/SymbolLoader.h index 66a778d8a..e8fb46493 100644 --- a/src/lib_spark/spark/md/SymbolLoader.h +++ b/src/lib_spark/spark/md/SymbolLoader.h @@ -40,9 +40,6 @@ class SymbolLoader { SymbolLoader(const SymbolLoader &); SymbolLoader &operator=(const SymbolLoader &); - static constexpr bool venue_has_synths(api::Venue venue); - static constexpr bool venue_has_synths(domain::protobufs::Venue venue); - static void add_symbols( const google::protobuf::RepeatedPtrField &symbols, Venue exchange, @@ -64,22 +61,6 @@ class SymbolLoader { // Definition ////////////////////////////////////////////////// -constexpr bool SymbolLoader::venue_has_synths(api::Venue venue) { - switch (venue) { - case api::VENUE_SIM: - case api::VENUE_SIMICE: - case api::VENUE_ICE: - case api::VENUE_GLBX: - return true; - default: - return false; - } -} - -constexpr bool SymbolLoader::venue_has_synths(domain::protobufs::Venue venue) { - return venue_has_synths(domain::DomainSecurityUtils::convert(venue)); -} - inline void SymbolLoader::add_symbols( const google::protobuf::RepeatedPtrField &symbols, Venue exchange, @@ -105,9 +86,7 @@ inline std::set SymbolLoader::load( const bts::domain::SecurityCriteria &include, const bts::domain::SecurityCriteria &exclude) { std::set result; - load(result, include); - if (exclude.is_defined()) { - } + load(result, include, exclude); return result; } @@ -142,8 +121,7 @@ inline std::size_t SymbolLoader::load( (exclude_none || !exclude.includes(venue, SEC_TYPE(SPREAD)))) { add_symbols(vcfg.spread(), venue, SEC_TYPE(SPREAD), result); } - if ((include_all || (venue_has_synths(venue) && - include.includes(venue, SEC_TYPE(SYNTHETIC)))) && + if ((include_all || (include.includes(venue, SEC_TYPE(SYNTHETIC)))) && (exclude_none || !exclude.includes(venue, SEC_TYPE(SYNTHETIC)))) { add_symbols(vcfg.spot(), VENUE(BTS), SEC_TYPE(SYNTHETIC), result); add_symbols(vcfg.future(), VENUE(BTS), SEC_TYPE(SYNTHETIC), result); diff --git a/src/lib_spark/spark/md/SyntheticSecurityService.cpp b/src/lib_spark/spark/md/SyntheticSecurityService.cpp index fe7416b84..f5df5cd2e 100644 --- a/src/lib_spark/spark/md/SyntheticSecurityService.cpp +++ b/src/lib_spark/spark/md/SyntheticSecurityService.cpp @@ -35,8 +35,9 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher struct SyntheticSecurityService::Pimpl { Pimpl(const MetadataStatefulPublisher &fn) - : recorder(utils::Dispatcher::background()), synthetics(), on_security(), - meta(), on_metadata(), metadata_publisher(fn) {} + : recorder(nullptr, std::nullopt, &utils::Dispatcher::background()), + synthetics(), on_security(), meta(), on_metadata(), + metadata_publisher(fn) {} SecurityIDRecorder recorder; std::unordered_map synthetics; utils::Event on_security; diff --git a/src/lib_spark/spark/md/shm/fixed_security.cpp b/src/lib_spark/spark/md/shm/fixed_security.cpp index ef62d21ae..6215c264c 100644 --- a/src/lib_spark/spark/md/shm/fixed_security.cpp +++ b/src/lib_spark/spark/md/shm/fixed_security.cpp @@ -219,26 +219,12 @@ void convert_security(const FixedSecurity &source, Security *presult) { get_contract_spec(source, *dest.mutable_contract_spec()); } -SecurityType securitytype_of_contractcase(Security::ContractCase cc) { - switch (cc) { - case Security::ContractCase::kFuture: - return SECURITYTYPE_FUTURE; - case Security::ContractCase::kOption: - return SECURITYTYPE_OPTION; - case Security::ContractCase::kSpot: - return api::SECURITYTYPE_SPOT; - case Security::ContractCase::kSpread: - return SECURITYTYPE_SPREAD; - case Security::ContractCase::kSynthetic: - return SECURITYTYPE_SYNTHETIC; - case Security::ContractCase::CONTRACT_NOT_SET: - bts_assert(cc != Security::ContractCase::CONTRACT_NOT_SET); - } - bts_assert(false); -} SecurityType securitytype_of(const Security &s) { - return securitytype_of_contractcase(s.Contract_case()); + auto type = bts::securities::security_type(s.Contract_case()); + bts_assert(type != api::SecurityType::SECURITYTYPE_UNSET); + return type; } + size_t num_legs(const Security &sec) { if (sec.Contract_case() == api::Security::ContractCase::kSpread) { return sec.spread().legs_size(); @@ -286,7 +272,7 @@ uint64_t exchange_id(const Security &security) { void convert_security( const Security &src, FixedSecurity &dest, bool set_security_key) { dest.key = set_security_key ? src.id() : invalid_security_key; - dest.type = securitytype_of_contractcase(src.Contract_case()); + dest.type = bts::securities::security_type(src.Contract_case()); dest.symbol(src.symbol()); dest.currency(src.currency()); dest.exchange = src.exchange(); diff --git a/src/lib_spark/spark/md/shm/fixed_security.h b/src/lib_spark/spark/md/shm/fixed_security.h index 29dbbfc82..ee9bb5448 100644 --- a/src/lib_spark/spark/md/shm/fixed_security.h +++ b/src/lib_spark/spark/md/shm/fixed_security.h @@ -215,7 +215,6 @@ void convert_security( FixedSecurity &dest, bool set_security_key = false); -SecurityType securitytype_of_contractcase(api::Security::ContractCase cc); SecurityType securitytype_of(const api::Security &); size_t num_legs(const api::Security &); uint64_t exchange_id(const api::Security &security); diff --git a/src/lib_spark/spark/md/test/md2_tests.cpp b/src/lib_spark/spark/md/test/md2_tests.cpp index bea27c3b7..d3b4a954d 100644 --- a/src/lib_spark/spark/md/test/md2_tests.cpp +++ b/src/lib_spark/spark/md/test/md2_tests.cpp @@ -8,6 +8,7 @@ #include "spark/md/PriceSize.h" #include "spark/md/relay/query_server.h" #include "spark/metrics/metrics.h" +#include "spark/persistence/SqliteStore.h" #include "spark/utils/Dispatcher.h" #include "spark/utils/Shuffle.h" #include "spark/utils/bts_chrono.h" @@ -159,12 +160,10 @@ class Co2Tests { #ifndef _WIN32 void test_security_id_recorder(const std::string &db_name) { // Verify callbacks happen after work complete - auto db1 = sqlite::open_db(db_name.c_str()); - REQUIRE(db1); - auto db2 = sqlite::open_db(db_name.c_str()); - REQUIRE(db2); - - SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}}; + SecurityIDRecorder idsvc{ + std::make_unique(db_name)}; + SecurityIDRecorder idsvc2{ + std::make_unique(db_name)}; security_key result = invalid_security_key; Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850); Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20); @@ -214,18 +213,18 @@ class Co2Tests { std::make_unique(s1, cold_header, hot_header); // Add spread to SecurityIDRecorder and work it - auto db = sqlite::open_db(":memory:"); + auto db = std::make_unique(":memory:"); REQUIRE(db); SecurityIDRecorder idsvc{ - {db, [&](ExchangeKey k) -> SecurityKey { - for (const auto &fut : - {f1handle->producer(), f2handle->producer()}) { - if (k.value() == fut.security().numeric_id) - return SecurityKey(fut.key()); - } - return SecurityKey(invalid_security_key); - }}}; + std::move(db), [&](ExchangeKey k) -> SecurityKey { + for (const auto &fut : + {f1handle->producer(), f2handle->producer()}) { + if (k.value() == fut.security().numeric_id) + return SecurityKey(fut.key()); + } + return SecurityKey(invalid_security_key); + }}; auto workwork = [&]() { idsvc.do_work(); diff --git a/src/lib_spark/spark/persistence/IDGenerator.cpp b/src/lib_spark/spark/persistence/IDGenerator.cpp index 0244c655e..c3eee227b 100644 --- a/src/lib_spark/spark/persistence/IDGenerator.cpp +++ b/src/lib_spark/spark/persistence/IDGenerator.cpp @@ -7,6 +7,7 @@ #include "spark/spark_data_directory.h" #include "spark/utils/bts_assert.h" #include "system/logging/logging.h" +#include "system/math/checked_arithmetic.h" #include #include @@ -66,24 +67,22 @@ std::optional IDGenerator::get_id( bool add_if_new) { std::string ns = fmt::format( "{}.{}", Venue_Name(venue).substr(6), - SecurityType_Name(type).substr(12)); + SecurityType_Name(type).substr(14)); return get_id(ns, name, add_if_new); } std::optional IDGenerator::get_id( const std::string &ns, const std::string &key, bool add_if_new) { std::lock_guard myLock(_instance_mx); - auto [exists, val] = me->kv_store->retrieve_one(ns, key); + auto [exists, rowid, val] = me->kv_store->retrieve_one(ns, key); if (exists) { - uint64_t rv = str_to_id(val); - LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rv); - return std::optional(rv); + LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rowid); + return std::optional(rowid); } else { if (add_if_new) { auto all = me->kv_store->retrieve_all(); - uint64_t new_id = get_next_id(all); - me->kv_store->insert(ns, key, std::to_string(new_id)); + auto new_id = me->kv_store->insert(ns, key, ""); LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, new_id); return std::optional(new_id); @@ -94,13 +93,6 @@ std::optional IDGenerator::get_id( } } -std::uint64_t IDGenerator::get_or_add( - const std::string &ns, const std::string &key) { - auto id = get_id(ns, key, true); - bts_assert(id.has_value()); - return *id; -} - std::uint64_t IDGenerator::get_or_add( domain::protobufs::Venue venue, domain::protobufs::SecurityType type, diff --git a/src/lib_spark/spark/persistence/IStore.h b/src/lib_spark/spark/persistence/IStore.h index 1f33ddf28..04f32a6f5 100644 --- a/src/lib_spark/spark/persistence/IStore.h +++ b/src/lib_spark/spark/persistence/IStore.h @@ -1,8 +1,12 @@ #pragma once +#include "spark/persistence/store_row.h" +#include #include #include #include +#include +#include namespace bts { namespace persistence { @@ -11,17 +15,45 @@ class IStore { public: virtual std::map> retrieve_all() = 0; - virtual std::pair - retrieve_one(std::string const &ns, std::string const &k) = 0; + + /** + * Finds the row with the specified namespace and key. + * + * @return std::tuple The first item indicates + * whether or not a row was found. For successful queries, the second and third + * items are the rowid and value, respectively. + */ + virtual std::tuple retrieve_one( + std::string const &ns, std::string const &k) = 0; virtual void remove(std::string const &ns, std::set const &keys) = 0; virtual void remove_all(std::string const &ns) = 0; - virtual void insert( + virtual std::int64_t insert( std::string const &ns, std::string const &k, std::string const &v) = 0; + + void insert_many(const std::vector &rows); + void insert_many_resolve_ids(std::vector &rows); + virtual void update( std::string const &ns, std::string const &k, std::string const &v) = 0; virtual ~IStore(); }; +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline void IStore::insert_many(const std::vector &rows) { + for (const auto &row : rows) { + insert(row.ns, row.key, row.value); + } +} + +inline void IStore::insert_many_resolve_ids(std::vector &rows) { + for (auto &row : rows) { + row.id = insert(row.ns, row.key, row.value); + } +} + } // namespace persistence } // namespace bts diff --git a/src/lib_spark/spark/persistence/SqliteStore.cpp b/src/lib_spark/spark/persistence/SqliteStore.cpp index a14c41f57..e4c21cd52 100644 --- a/src/lib_spark/spark/persistence/SqliteStore.cpp +++ b/src/lib_spark/spark/persistence/SqliteStore.cpp @@ -1,16 +1,10 @@ #include "SqliteStore.h" -#include "spark/config/config.h" #include "spark/persistence/persistence_utils.h" #include "spark/spark_data_directory.h" #include "spark/utils/Sqlite_utils.h" #include "spark/utils/bts_assert.h" -#include "spark/utils/bts_chrono.h" -#include "spark/utils/bts_no_copy.h" -#include "spark/utils/defer.h" -#include "spark/utils/file_utils.h" #include "system/logging/logging.h" -#include "system/os.h" #include "system/protobufs/config/system_config.pb.h" #include @@ -36,82 +30,95 @@ static boost::filesystem::path get_db_path(const std::string &str) { } struct SqliteStore::PImpl { + std::string const db; + boost::filesystem::path db_path; + sqlite3_ptr conn{nullptr, sqlite3_close}; + sqlite3_stmt_ptr insert{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr insert_many{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr insert_with_id{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr remove{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr remove_all{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr update{nullptr, sqlite3_finalize}; + PImpl(const boost::filesystem::path &db_path) - : db(db_path.stem().string()), db_path(db_path), - conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize), - remove(nullptr, sqlite3_finalize), - remove_all(nullptr, sqlite3_finalize), - update(nullptr, sqlite3_finalize) { + : db(db_path.stem().string()), db_path(db_path) { bts_assert(db.size()); } PImpl(std::string const &store_root, std::string const &db_name) : PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {} - std::string const db; - boost::filesystem::path db_path; - sqlite3_ptr conn; - sqlite3_stmt_ptr insert; - sqlite3_stmt_ptr remove; - sqlite3_stmt_ptr remove_all; - sqlite3_stmt_ptr update; - BTS_NO_COPY(PImpl); + explicit PImpl(sqlite3_ptr conn) : db(), db_path(), conn(std::move(conn)) {} + + PImpl(const PImpl &) = delete; + PImpl &operator=(const PImpl &) = delete; }; SqliteStore::SqliteStore(std::string const &db) : me(std::make_unique(get_db_path(db))) { - LOG(persist, Info, "SqliteStore ctr. db: {}; db_path: {}", db, me->db_path); + LOG(persist, Info, "SqliteStore ctr. db: {}; db_path: {}", db, + me->db_path.string()); + init_db(); +} + +SqliteStore::SqliteStore(sqlite3_ptr db) + : me(std::make_unique(std::move(db))) { + LOG(persist, Info, "SqliteStore ctr. db: "); init_db(); } void SqliteStore::init_db() { - auto path = me->db_path; - auto parent = path.parent_path(); + const auto path = me->db_path; - if (!boost::filesystem::is_directory(parent)) - bts_assert(boost::filesystem::create_directories(parent)); - - if (!boost::filesystem::exists(path)) { - LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", - me->db, path.string()); - - me->conn = sqlite::open_db(path.c_str()); - bts_assert_msg(me->conn, path.c_str()); - - const char *create_schema_sql = - "create table store ( ns text, key text, value text)"; - sqlite::exec_nofail(me->conn, create_schema_sql); - - const char *key_index_sql = - "create unique index key_index on store(ns,key)"; - sqlite::exec_nofail(me->conn, key_index_sql); - - const char *ns_index_sql = "create index ns_index on store(ns)"; - sqlite::exec_nofail(me->conn, ns_index_sql); - } else { - me->conn = sqlite::open_db(path.c_str()); + if (!path.empty()) { + if (!boost::filesystem::is_directory(path.parent_path())) { + bts_assert(boost::filesystem::create_directories(path.parent_path())); + } + if (!boost::filesystem::exists(path)) { + LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", + me->db, path.string()); + } } + auto &conn = me->conn; + if (conn == nullptr) { + conn = sqlite::open_db(path.c_str()); + bts_assert(conn != nullptr); + } + sqlite::exec_nofail( + conn, + "CREATE TABLE IF NOT EXISTS store (ns TEXT, key TEXT, value TEXT)"); + sqlite::exec_nofail( + conn, "CREATE UNIQUE INDEX IF NOT EXISTS key_index on store(ns, key)"); + sqlite::exec_nofail( + conn, "CREATE INDEX IF NOT EXISTS ns_index on store(ns)"); + conn = sqlite::open_db(path.c_str()); + // Asynchronous for speed - sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;"); - sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL"); + sqlite::exec_nofail(conn, "PRAGMA synchronous = 0;"); + sqlite::exec_nofail(conn, "PRAGMA journal_mode=WAL"); // prepare insert statement const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)"; - me->insert = sqlite::prepare_query(me->conn, insert_sql); + me->insert = sqlite::prepare_query(conn, insert_sql); + + // prepare insert_with_id statement + const char *insert_with_id_sql = + "insert into store (rowid, ns, key, value) values (?,?,?,?)"; + me->insert_with_id = sqlite::prepare_query(conn, insert_with_id_sql); // prepare update statement const char *update_sql = "update store set value = ? where ns = ? and key = ?"; - me->update = sqlite::prepare_query(me->conn, update_sql); + me->update = sqlite::prepare_query(conn, update_sql); // prepare remove statement const char *remove_all_sql = "delete from store where ns = ?"; - me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql); + me->remove_all = sqlite::prepare_query(conn, remove_all_sql); // prepare remove statement const char *remove_sql = "delete from store where ns = ? and key = ?"; - me->remove = sqlite::prepare_query(me->conn, remove_sql); + me->remove = sqlite::prepare_query(conn, remove_sql); } SqliteStore::~SqliteStore() {} @@ -140,25 +147,27 @@ SqliteStore::Data SqliteStore::retrieve_all() { return result; } -std::pair SqliteStore::retrieve_one( +std::tuple SqliteStore::retrieve_one( std::string const &ns, std::string const &k) { auto query_str = fmt::format( - "select ns,key,value from store where ns='{}' and key='{}'", ns, k); + "select rowid,ns,key,value from store where ns='{}' and key='{}'", ns, + k); sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str()); - std::pair result{false, ""}; - + auto result = std::make_tuple(false, 0, ""); auto pq = query.get(); auto sqlite_result = sqlite3_step(pq); if (sqlite_result == SQLITE_ROW) { + std::int64_t rowid = sqlite3_column_int64(pq, 0); std::string ns( - (char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0)); - std::string key( (char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1)); + std::string key( + (char *)sqlite3_column_text(pq, 2), sqlite3_column_bytes(pq, 2)); std::string val( - (char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2)); - result.first = true; - result.second = std::move(val); + (char *)sqlite3_column_blob(pq, 3), sqlite3_column_bytes(pq, 3)); + std::get<0>(result) = true; + std::get<1>(result) = rowid; + std::get<2>(result) = std::move(val); } return result; } @@ -167,91 +176,185 @@ void SqliteStore::remove( std::string const &ns, std::set const &keys) { bts_assert(ns.size()); - for (auto &k : keys) { - SQLITE_CHK( - sqlite3_bind_text( - me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->remove.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not delete row in store. Will retry once. error_code: {} " - "ns:{}, k: {}", - r, ns, k); - sleep_for(2s); - r = sqlite3_step(me->remove.get()); + try { + for (auto &k : keys) { + SQLITE_CHK( + sqlite3_bind_text( + me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->remove.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to delete row. Bailing out. error_code: " - "{}", - r); - bts_assert(r == SQLITE_DONE); + "Could not delete row in store. Will retry once. error_code: " + "{} " + "ns:{}, k: {}", + r, ns, k); + sleep_for(2s); + r = sqlite3_step(me->remove.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to delete row. Bailing out. " + "error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); } - SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::remove threw. what: {}", + std::string{ex.what()}); } } void SqliteStore::remove_all(std::string const &ns) { bts_assert(ns.size()); - SQLITE_CHK( - sqlite3_bind_text( - me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->remove_all.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not delete all rows in store. Will retry once. error_code: {} " - "ns:{}", - r, ns); - sleep_for(2s); - r = sqlite3_step(me->remove_all.get()); + try { + SQLITE_CHK( + sqlite3_bind_text( + me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->remove_all.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to delete all rows. Bailing out. " - "error_code: " - "{}", - r); - bts_assert(r == SQLITE_DONE); + "Could not delete all rows in store. Will retry once. error_code: " + "{} " + "ns:{}", + r, ns); + sleep_for(2s); + r = sqlite3_step(me->remove_all.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to delete all rows. Bailing out. " + "error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::remove_all threw. what: {}", + std::string{ex.what()}); } - SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); } -void SqliteStore::insert( +std::int64_t SqliteStore::insert( std::string const &ns, std::string const &k, std::string const &v) { - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->insert.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not insert row in store. Will retry once. error_code: {} ns: " - "{}, k: {}, v: {}", - r, ns, k, v); - sleep_for(2s); - r = sqlite3_step(me->insert.get()); + try { + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->insert.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to insert row. Bailing out. error_code: {}", - r); - bts_assert(r == SQLITE_DONE); + "Could not insert row in store. Will retry once. error_code: {} " + "ns: " + "{}, k: {}, v: {}", + r, ns, k, v); + sleep_for(2s); + r = sqlite3_step(me->insert.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to insert row. Bailing out. error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + const auto rowid = sqlite3_last_insert_rowid(me->conn.get()); + SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); + return rowid; + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::insert threw. what: {}", + std::string{ex.what()}); } - SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); + return 0; +} + +void SqliteStore::insert_many(const std::vector &rows) { + LOG(md, Info, "SqliteStore::insert_many size: {}", rows.size()); + if (rows.empty()) { + return; + } + try { + sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION"); + sqlite3 *conn_rawp = me->conn.get(); + sqlite3_stmt *insert_rawp = me->insert.get(); + sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get(); + for (const auto &row : rows) { + int col_idx = 1; + sqlite3_stmt *stmt; + if (row.id > 0) { + stmt = insert_with_id_rawp; + sqlite3_bind_int(stmt, col_idx++, row.id); + } else { + stmt = insert_rawp; + } + sqlite3_bind_text( + stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC); + sqlite3_bind_text( + stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC); + int result = sqlite3_step(stmt); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + sqlite3_last_insert_rowid(conn_rawp); + } + sqlite3_reset(stmt); + } + sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION"); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::insert_many threw. what: {}", + std::string{ex.what()}); + } +} + +void SqliteStore::insert_many_resolve_ids(std::vector &rows) { + LOG(md, Info, "SqliteStore::insert_many_resolve_ids size: {}", rows.size()); + if (rows.empty()) { + return; + } + sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION"); + sqlite3 *conn_rawp = me->conn.get(); + sqlite3_stmt *insert_rawp = me->insert.get(); + sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get(); + for (auto &row : rows) { + int col_idx = 1; + sqlite3_stmt *stmt; + if (row.id > 0) { + stmt = insert_with_id_rawp; + sqlite3_bind_int(stmt, col_idx++, row.id); + } else { + stmt = insert_rawp; + } + sqlite3_bind_text( + stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC); + sqlite3_bind_text( + stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC); + int result = sqlite3_step(stmt); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + if (row.id > 0) { + bts_assert(sqlite3_last_insert_rowid(conn_rawp) == row.id); + } else { + row.id = sqlite3_last_insert_rowid(conn_rawp); + } + } + sqlite3_reset(stmt); + } + sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION"); } void SqliteStore::update( diff --git a/src/lib_spark/spark/persistence/SqliteStore.h b/src/lib_spark/spark/persistence/SqliteStore.h index 4f0caf725..96fc0fb66 100644 --- a/src/lib_spark/spark/persistence/SqliteStore.h +++ b/src/lib_spark/spark/persistence/SqliteStore.h @@ -1,7 +1,12 @@ #pragma once #include "IStore.h" +#include "spark/persistence/store_row.h" +#include "spark/utils/Sqlite_utils.h" + +#include #include +#include namespace bts { namespace persistence { @@ -9,22 +14,26 @@ namespace persistence { class SqliteStore : public IStore { public: SqliteStore(std::string const &db); + explicit SqliteStore(sqlite3_ptr db); ~SqliteStore(); static SqliteStore from_path(const std::string &full_path); typedef std::map> Data; Data retrieve_all() override; - std::pair retrieve_one( + + std::tuple retrieve_one( std::string const &ns, std::string const &k) override; void remove( std::string const &ns, std::set const &keys) override; void remove_all(std::string const &ns) override; - void insert( + std::int64_t insert( std::string const &ns, std::string const &k, std::string const &v) override; + void insert_many(const std::vector &rows); + void insert_many_resolve_ids(std::vector &rows); void update( std::string const &ns, std::string const &k, diff --git a/src/lib_spark/spark/persistence/store_row.h b/src/lib_spark/spark/persistence/store_row.h new file mode 100644 index 000000000..1c7b7f079 --- /dev/null +++ b/src/lib_spark/spark/persistence/store_row.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace bts::persistence { + +struct StoreRow { + std::string ns; + std::string key; + std::string value; + /** + * Represents the unique 'rowid' in the sqlite db. A value of 0 indicates that + * the rowid isn't known, and should either be generated by the db (for inserts), + * or resolved by finding the row with matching ns, key, value (for retrieves). + */ + std::int64_t id{0}; +}; + +} // namespace bts::persistence diff --git a/src/lib_system/system/math/checked_arithmetic.h b/src/lib_system/system/math/checked_arithmetic.h new file mode 100644 index 000000000..91e60a6b9 --- /dev/null +++ b/src/lib_system/system/math/checked_arithmetic.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +namespace bts::system::math { + +static std::uint32_t checked_ltou(std::int64_t l); +static std::uint64_t checked_ltoul(std::int64_t l); + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::uint32_t checked_ltou(std::int64_t l) { + if (l < 0) { + throw std::overflow_error{ + "Failed to convert int64 to uint32 (value < 0)"}; + } + if (l > + static_cast(std::numeric_limits::max())) { + throw std::overflow_error{ + "Failed to convert int64 to uint32. value > max)"}; + } + return static_cast(l); +} + +inline std::uint64_t checked_ltoul(std::int64_t l) { + if (l < 0) { + throw std::overflow_error{ + "Failed to convert int64 to uint64 (value < 0)"}; + } + return static_cast(l); +} + +} // namespace bts::system::math diff --git a/src/lib_system/system/time_span.h b/src/lib_system/system/time_span.h index 3b8c8c03c..9b345082b 100644 --- a/src/lib_system/system/time_span.h +++ b/src/lib_system/system/time_span.h @@ -50,7 +50,9 @@ class TimeSpan { constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept; +constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept; +constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept; std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept; @@ -116,10 +118,18 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept { return lhs.nanos() < rhs.nanos(); } +constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept { + return lhs.nanos() <= rhs.nanos(); +} + constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept { return lhs.nanos() > rhs.nanos(); } +constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept { + return lhs.nanos() >= rhs.nanos(); +} + } // namespace bts::core FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string());