diff --git a/src/lib/core/TimeSpan.h b/src/lib/core/TimeSpan.h index 23970e616..5e0d14458 100644 --- a/src/lib/core/TimeSpan.h +++ b/src/lib/core/TimeSpan.h @@ -50,7 +50,9 @@ class TimeSpan { constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept; +constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept; constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept; +constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept; std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept; @@ -116,10 +118,18 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept { return lhs.nanos() < rhs.nanos(); } +constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept { + return lhs.nanos() <= rhs.nanos(); +} + constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept { return lhs.nanos() > rhs.nanos(); } +constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept { + return lhs.nanos() >= rhs.nanos(); +} + } // namespace bts::core FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string()); diff --git a/src/lib/md/Diag/ShmDumper.cpp b/src/lib/md/Diag/ShmDumper.cpp index 4dbb10599..dc04f67f7 100644 --- a/src/lib/md/Diag/ShmDumper.cpp +++ b/src/lib/md/Diag/ShmDumper.cpp @@ -53,82 +53,7 @@ string dump_header(const ColdHeader &header) { } string ShmDumper::dump_security(const FixedSecurity &security) { - ostringstream res; - res << "{"; - res << "'key': " << security.key << ", "; - res << "'symbol': '" << security.symbol() << "', "; - res << "'type': '" << api::SecurityType_Name(security.type) << "', "; - res << "'exchange': '" << api::Venue_Name(security.exchange) << "', "; - res << "'alt_symbol': '" << security.alt_symbol() << "', "; - res << "'cfi_code': '" << security.cfi_code() << "', "; - res << "'security_desc': '" << security.security_desc() << "', "; - res << "'platform_id': '" << security.platform_id() << "', "; - res << "'display_factor': " << (int)security.display_factor << ", "; - res << "'numeric_id': " << security.numeric_id << ", "; - res << "'cboe_class_key': " << security.cboe_class_key << ", "; - res << "'main_frac': " << (int)security.cme_main_fraction << ", "; - res << "'sub_fract': " << (int)security.cme_sub_fraction << ", "; - res << "'price_format': " << (int)security.cme_price_display_format << ", "; - res << "'market_segment': " << (int)security.cme_market_segment_id << ", "; - - res << "'price_break': " << security.price_break << ", "; - res << "'low_tick': " << security.low_tick << ", "; - res << "'high_tick': " << security.high_tick << ", "; - res << "'cpp': " << security.cash_per_point << ", "; - - res << "'market_description ': " << security.ice_market_description() - << ", "; - res << "'cleared_alias ': " << security.ice_cleared_alias() << ", "; - res << "'product_description ': " << security.ice_product_name() << ", "; - res << "'hub_alias ': " << security.ice_hub_alias() << ", "; - res << "'strip_name': " << security.ice_strip_name() << ", "; - res << "'mic_code': " << security.mic_code() << ", "; - - switch (security.type) { - case SECURITYTYPE_SPOT: - break; - case SECURITYTYPE_FUTURE: { - auto *psec = static_cast(&security); - res << "'expiration': '"; - securities::write_date_string( - psec->expiration_year, psec->expiration_month, psec->expiration_day, - res); - res << "' "; - break; - } - case SECURITYTYPE_OPTION: { - auto *psec = static_cast(&security); - res << "'expiration': '"; - securities::write_date_string( - psec->expiration_year, psec->expiration_month, psec->expiration_day, - res); - res << "', "; - res << "'strike': " << psec->strike << ", "; - res << "'callput': '" << api::PutOrCall_Name(psec->call_put) << "' "; - break; - } - case SECURITYTYPE_SPREAD: { - auto *psec = static_cast(&security); - res << "'legs': ["; - for (size_t x = 0; x < psec->leg_count(); ++x) { - auto &leg = psec->legs[x]; - if (x > 0) - res << ","; - res << "{ 'key': " << leg.key << ", 'exkey': " << leg.exchange_key - << ", 'quantity': " << leg.leg_quantity - << ", 'leg_price': " << leg.leg_price - << ", 'delta': " << leg.leg_delta << ", 'side': '" - << api::Side_Name(leg.leg_side) << "' }"; - } - res << "]"; - break; - } - default: - break; - } - - res << "}"; - return res.str(); + return JsonDumper::dump(security); } string dump(datetime::Date date) { diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index 27b84e073..c33e5814d 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -61,26 +61,6 @@ datetime::Date sec_expiration_cut_off() { return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day()); } -void apply_compatibilit_mutations(Security &security) { - // Migrate from old tick stuff - if (security.has_glbx()) { - if (security.has_glbx()) { - if (security.contract_spec().main_fraction() == 0) { - security.mutable_contract_spec()->set_main_fraction( - security.glbx().do_not_use_main_fraction()); - } - if (security.contract_spec().sub_fraction() == 0) { - security.mutable_contract_spec()->set_sub_fraction( - security.glbx().do_not_use_sub_fraction()); - } - if (security.contract_spec().price_display_format() == 0) { - security.mutable_contract_spec()->set_price_display_format( - security.glbx().do_not_use_price_display_format()); - } - } - } -} - size_t load_securities( std::function const &sym_filter, const std::function &callback) { @@ -104,7 +84,6 @@ size_t load_securities( if (sym_filter(us)) { auto blob = select_stmt.getColumn("security_blob"); if (security.ParseFromArray(blob, blob.getBytes())) { - apply_compatibilit_mutations(security); bts_assert(security_key == security.id()); count++; callback(security); @@ -146,7 +125,6 @@ SecurityPtr find_security(SecurityKey key) { auto result = std::make_shared(); auto blob = select.getColumn(0); if (result->ParseFromArray(blob, blob.getBytes())) { - apply_compatibilit_mutations(*result); return result; } } diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 3313a037f..f0abccdf8 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -1,19 +1,25 @@ #ifndef _WIN32 #include "SecurityIDRecorder.h" #include "lib/domain_model/Security.h" +#include "lib/persistence/IStore.h" +#include "lib/persistence/SqliteStore.h" #include #include "lib/config/config.h" #include "lib/logging/logging.h" #include "lib/utils/Sqlite_utils.h" +#include "lib/utils/checked_arithmetic.h" #include #include #include "lib/config/protos/market_data.pb.h" #include "lib/config/protos/system_config.pb.h" +#include #include +#include #include +#include using namespace std; using namespace bts::utils; @@ -26,9 +32,32 @@ using bts::api::Security; using bts::api::SecurityType; using bts::api::Venue; using bts::config::SystemConfig; +using bts::persistence::IStore; +using bts::persistence::SqliteStore; uint64_t exchange_id(const api::Security &security); +static std::unique_ptr get_default_store() { + return std::make_unique("key_ids"); +} + +static persistence::StoreRow get_row( + Venue venue, + SecurityType type, + const std::string &name, + const std::string &prefix = "") { + return persistence::StoreRow{ + fmt::format( + "{}{}.{}", std::string{prefix}, Venue_Name(venue).substr(6), + SecurityType_Name(type).substr(13)), + name, ""}; +} + +static persistence::StoreRow get_row( + const api::Security &s, const std::string &prefix = "") { + return get_row(s.exchange(), bts::securities::security_type(s), prefix); +} + struct SecurityIdRequest { Venue venue; SecurityType type; @@ -38,14 +67,16 @@ struct SecurityIdRequest { }; struct SecurityIDRecorderState { - SecurityIDRecorderState(SecurityIDRecorderParams &p) - : pending_handle(), thread_id(), db(nullptr, sqlite3_close), timer(), - lookup_key(p.lookup_key), unresolved_legs() {} + SecurityIDRecorderState( + std::unique_ptr store, KeyLookupOption lookup_key) + : store(store == nullptr ? get_default_store() : std::move(store)), + pending_handle(), thread_id(), timer(), lookup_key(lookup_key), + unresolved_legs() {} + std::unique_ptr store{nullptr}; tbb::concurrent_queue> id_request_queue; tbb::concurrent_queue pending_handle; std::thread::id thread_id; - sqlite3_ptr db; unique_ptr timer; KeyLookupOption lookup_key; @@ -81,46 +112,28 @@ struct SecurityIDRecorderState { void check_access() { bts_assert(thread_id == std::this_thread::get_id()); } void add_unresolved_spread(MDProducerHandle &handle); - security_key search(const std::string &descriptor); - security_key insert(const std::string &descriptor, SecurityType); + security_key search(const persistence::StoreRow &row); + security_key insert(const persistence::StoreRow &row); }; -SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params) - : _self(new SecurityIDRecorderState(params)) { - if (params.db) { - _self->db = std::move(params.db); - } +SecurityIDRecorder::SecurityIDRecorder( + std::unique_ptr store, + KeyLookupOption lookup_key, + utils::Dispatcher *dispatcher) + : _self(std::make_unique( + store == nullptr + ? std::make_unique("key_ids") + : std::move(store), + lookup_key)) { init(); - if (params.dispatcher) { - _self->run(*params.dispatcher); + if (dispatcher != nullptr) { + _self->run(*dispatcher); } } SecurityIDRecorder::~SecurityIDRecorder() {} -void SecurityIDRecorder::init() { - LOG(md, Info, "SecurityIDRecorder::init"); - string db_dir = - config::get().data_path() + "databases/md/"; - string db_path = db_dir + "/security_keys.sqlite"; - - if (!boost::filesystem::exists(db_dir)) { - boost::filesystem::create_directories(db_dir); - } - - if (!_self->db) { - _self->db = sqlite::open_db(db_path.c_str()); - bts_assert(_self->db); - } - - // Set a long busy timeout - sqlite3_busy_timeout(_self->db.get(), 9999); - sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); - sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0"); - sqlite::exec_nofail( - _self->db, "CREATE TABLE IF NOT EXISTS store (descriptor STRING " - "NOT NULL PRIMARY KEY, type int);"); -} +void SecurityIDRecorder::init() {} void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}", @@ -134,7 +147,7 @@ void SecurityIDRecorder::assign_key( assign_key( security.exchange(), bts::securities::security_type(security), bts::securities::descriptor(security), - [&callback, msec = security]( + [callback, msec = security]( api::Venue, api::SecurityType, const std::string &, uint32_t id) mutable { msec.set_id(id); @@ -269,27 +282,25 @@ void SecurityIDRecorderState::work_db() { // SPARK-1446 Need to try to add existing securities to the db in case we // deleted them by hand vector> existing_securities; - vector> defer; - vector new_descriptors; + vector new_rows; size_t count = 0; - string descriptor; while (id_request_queue.try_pop(temp)) { - descriptor = temp->name; - - // If we're unable to create a descriptor try again later - if (descriptor == "") { - defer.push_back(std::move(temp)); - continue; + security_key key = invalid_security_key; + auto row = get_row(temp->venue, temp->type, temp->name); + try { + key = search(row); + } catch (const std::overflow_error &ex) { + LOG(persist, Error, + "SecurityIDRecorderState::work_db. Row search threw ex: {}", + std::string{ex.what()}); } - - security_key key = search(descriptor); if (key != invalid_security_key) { temp->callback(temp->venue, temp->type, temp->name, key); existing_securities.push_back(std::move(temp)); } else { // always push in parallel new_entries.push_back(std::make_pair(key, std::move(temp))); - new_descriptors.push_back(descriptor); + new_rows.push_back(std::move(row)); ++count; // Limit the amount of time we spend writing to sqlite during startup // to avoid blocking the heartbeat @@ -299,22 +310,14 @@ void SecurityIDRecorderState::work_db() { } } - for (auto &d : defer) { - LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}", - d->name); - id_request_queue.push(std::move(d)); - } - if (!new_entries.empty()) { - sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION"); - + store->insert_many_resolve_ids(new_rows); for (size_t x = 0; x < new_entries.size(); ++x) { auto &entry = new_entries[x]; - entry.first = insert(new_descriptors[x], entry.second->type); + auto &row = new_rows[x]; + entry.first = row.id; bts_assert_d(entry.first != invalid_security_key); } - - sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } if (!new_entries.empty()) { @@ -327,65 +330,27 @@ void SecurityIDRecorderState::work_db() { void SecurityIDRecorder::bulk_insert( const std::vector &data, const char *prefix) { - LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size()); - if (data.size() == 0) - return; - sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); - std::string descriptor; - { - auto insert_stmt = sqlite::prepare_query( - _self->db, "INSERT INTO security_keys (rowid, descriptor, type) " - "values (?, ?, ?)"); - for (const api::Security *sec : data) { - descriptor = prefix + bts::securities::unique_descriptor(*sec); - auto type = securitytype_of(*sec); - - sqlite3_bind_int(insert_stmt.get(), 1, sec->id()); - sqlite3_bind_text( - insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - sqlite3_bind_int(insert_stmt.get(), 3, (int)type); - sqlite3_step(insert_stmt.get()); - sqlite3_reset(insert_stmt.get()); - } + std::vector rows; + rows.reserve(data.size()); + for (const auto *s : data) { + rows.push_back(get_row(*s, prefix)); } - sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + _self->store->insert_many_resolve_ids(rows); } -security_key SecurityIDRecorderState::search(const std::string &descriptor) { - LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor); - auto select_stmt = sqlite::prepare_query( - db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); - sqlite3_bind_text( - select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - int result = sqlite3_step(select_stmt.get()); - if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_column_int(select_stmt.get(), 0); +security_key SecurityIDRecorderState::search(const persistence::StoreRow &row) { + auto [exists, rowid, val] = store->retrieve_one(row.ns, row.key); + if (exists) { + if (!val.empty()) { + bts_assert(stol(val) == rowid); + } + return system::math::checked_ltou(rowid); } return invalid_security_key; } -security_key SecurityIDRecorderState::insert( - const std::string &descriptor, SecurityType type) { - LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor); - auto insert_stmt = sqlite::prepare_query( - db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); - sqlite3_bind_text( - insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), - SQLITE_STATIC); - sqlite3_bind_int(insert_stmt.get(), 2, (int)type); - int result = sqlite3_step(insert_stmt.get()); - if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_last_insert_rowid(db.get()); - } - // Just in case our insert failed due to duplicate row - auto key = search(descriptor); - if (key == invalid_security_key) { - LOG(md, Error, "Failed to assign security key. SQLite return code: {}", - result); - } - return key; +security_key SecurityIDRecorderState::insert(const persistence::StoreRow &row) { + return store->insert(row.ns, row.key, row.value); } } // namespace md diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index 862ba79b8..2925e0c20 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -2,7 +2,7 @@ #ifndef _WIN32 #include "MDProducerHandle.h" #include "common.h" - +#include "lib/persistence/IStore.h" #include "lib/utils/Sqlite_utils.h" #include #include @@ -22,26 +22,15 @@ namespace md { typedef const std::function KeyLookup; typedef std::optional KeyLookupOption; -struct SecurityIDRecorderParams { - SecurityIDRecorderParams() - : dispatcher(), db(sqlite::null_db()), lookup_key() {} - SecurityIDRecorderParams(utils::Dispatcher &d) - : dispatcher(&d), db(sqlite::null_db()), lookup_key() {} - SecurityIDRecorderParams(utils::Dispatcher &d, KeyLookup &get_key) - : dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {} - SecurityIDRecorderParams(sqlite3_ptr &db) - : dispatcher(), db(std::move(db)), lookup_key() {} - SecurityIDRecorderParams(sqlite3_ptr &db, KeyLookup &get_key) - : dispatcher(), db(std::move(db)), lookup_key(get_key) {} - utils::Dispatcher *dispatcher; - sqlite3_ptr db; - KeyLookupOption lookup_key; -}; + struct SecurityIDRecorderState; class SecurityIDRecorder { public: - SecurityIDRecorder(SecurityIDRecorderParams); + SecurityIDRecorder( + std::unique_ptr store = nullptr, + KeyLookupOption lookup_key = std::nullopt, + utils::Dispatcher *dispatcher = nullptr); ~SecurityIDRecorder(); typedef std::function Callback; diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index 252edf917..cd2ebc9ee 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -217,26 +217,12 @@ void convert_security(const FixedSecurity &source, Security *presult) { get_contract_spec(source, *dest.mutable_contract_spec()); } -SecurityType securitytype_of_contractcase(Security::ContractCase cc) { - switch (cc) { - case Security::ContractCase::kFuture: - return SECURITYTYPE_FUTURE; - case Security::ContractCase::kOption: - return SECURITYTYPE_OPTION; - case Security::ContractCase::kSpot: - return api::SECURITYTYPE_SPOT; - case Security::ContractCase::kSpread: - return SECURITYTYPE_SPREAD; - case Security::ContractCase::kSynthetic: - return SECURITYTYPE_SYNTHETIC; - case Security::ContractCase::CONTRACT_NOT_SET: - bts_assert(cc != Security::ContractCase::CONTRACT_NOT_SET); - } - bts_assert(false); -} SecurityType securitytype_of(const Security &s) { - return securitytype_of_contractcase(s.Contract_case()); + auto type = bts::securities::security_type(s.Contract_case()); + bts_assert(type != api::SecurityType::SECURITYTYPE_UNSET); + return type; } + size_t num_legs(const Security &sec) { if (sec.Contract_case() == api::Security::ContractCase::kSpread) { return sec.spread().legs_size(); @@ -284,7 +270,7 @@ uint64_t exchange_id(const Security &security) { void convert_security( const Security &src, FixedSecurity &dest, bool set_security_key) { dest.key = set_security_key ? src.id() : invalid_security_key; - dest.type = securitytype_of_contractcase(src.Contract_case()); + dest.type = bts::securities::security_type(src.Contract_case()); dest.symbol(src.symbol()); dest.currency(src.currency()); dest.exchange = src.exchange(); diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index 3afb6c79e..4b39e6c51 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -215,10 +215,9 @@ void convert_security(const FixedSecurity &, Security *); void convert_security( const Security &src, FixedSecurity &dest, bool set_security_key = false); -SecurityType securitytype_of_contractcase(Security::ContractCase cc); -SecurityType securitytype_of(const Security &); -size_t num_legs(const Security &); -uint64_t exchange_id(const Security &security); +SecurityType securitytype_of(const api::Security &); +size_t num_legs(const api::Security &); +uint64_t exchange_id(const api::Security &security); void to_json(nlohmann::json &j, const FixedSpreadLeg &f); diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp index c0301e8f4..6daca8b4c 100644 --- a/src/lib/md/SyntheticSecurityService.cpp +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -35,8 +35,9 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher struct SyntheticSecurityService::Pimpl { Pimpl(const MetadataStatefulPublisher &fn) - : recorder(utils::Dispatcher::background()), synthetics(), on_security(), - meta(), on_metadata(), metadata_publisher(fn) {} + : recorder(nullptr, std::nullopt, &utils::Dispatcher::background()), + synthetics(), on_security(), meta(), on_metadata(), + metadata_publisher(fn) {} SecurityIDRecorder recorder; std::unordered_map synthetics; utils::Event on_security; diff --git a/src/lib/md/cme/mdp3_data_directory.h b/src/lib/md/cme/mdp3_data_directory.h new file mode 100644 index 000000000..740d46dbb --- /dev/null +++ b/src/lib/md/cme/mdp3_data_directory.h @@ -0,0 +1,32 @@ +#pragma once + +#include "lib/os/spark_data_directory.h" + +#include +#include + +namespace bts::md { + +struct Mdp3DataDirectory { + boost::filesystem::path log_dir() const { return root.log_dir() / "onixs"; } + + boost::filesystem::path data_dir() const { return root.root() / "cme_mdp3"; } + + boost::filesystem::path config_file() const { + return data_dir() / "config.xml"; + } + + boost::filesystem::path templates_file() const { + return data_dir() / "templates.xml"; + } + + void create_dirs() { + boost::filesystem::create_directories(data_dir()); + boost::filesystem::create_directories(log_dir()); + } + + private: + spark::SparkDataDirectory root{}; +}; + +} // namespace bts::md diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 13cdbb323..f4cff0acc 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -9,6 +9,7 @@ #include "lib/config/config.h" #include "lib/metrics/metrics.h" #include "lib/os/os.h" +#include "lib/persistence/SqliteStore.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/Shuffle.h" #include "lib/utils/bts_chrono.h" @@ -160,12 +161,10 @@ class Co2Tests { #ifndef _WIN32 void test_security_id_recorder(const std::string &db_name) { // Verify callbacks happen after work complete - auto db1 = sqlite::open_db(db_name.c_str()); - REQUIRE(db1); - auto db2 = sqlite::open_db(db_name.c_str()); - REQUIRE(db2); - - SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}}; + SecurityIDRecorder idsvc{ + std::make_unique(db_name)}; + SecurityIDRecorder idsvc2{ + std::make_unique(db_name)}; security_key result = invalid_security_key; Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850); Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20); @@ -215,18 +214,18 @@ class Co2Tests { std::make_unique(s1, cold_header, hot_header); // Add spread to SecurityIDRecorder and work it - auto db = sqlite::open_db(":memory:"); + auto db = std::make_unique(":memory:"); REQUIRE(db); SecurityIDRecorder idsvc{ - {db, [&](ExchangeKey k) -> SecurityKey { - for (const auto &fut : - {f1handle->producer(), f2handle->producer()}) { - if (k.value() == fut.security().numeric_id) - return SecurityKey(fut.key()); - } - return SecurityKey(invalid_security_key); - }}}; + std::move(db), [&](ExchangeKey k) -> SecurityKey { + for (const auto &fut : + {f1handle->producer(), f2handle->producer()}) { + if (k.value() == fut.security().numeric_id) + return SecurityKey(fut.key()); + } + return SecurityKey(invalid_security_key); + }}; auto workwork = [&]() { idsvc.do_work(); diff --git a/src/lib/os/spark_data_directory.cpp b/src/lib/os/spark_data_directory.cpp new file mode 100644 index 000000000..fea551a9e --- /dev/null +++ b/src/lib/os/spark_data_directory.cpp @@ -0,0 +1,45 @@ +#include "spark_data_directory.h" +#include "lib/config/config.h" +#include "lib/config/protos/system_config.pb.h" + +#include +#include +#include + +namespace bts::spark { + +SparkDataDirectory::SparkDataDirectory() + : SparkDataDirectory(config::get().data_path()) {} + +SparkDataDirectory::SparkDataDirectory(const std::string &root) : root_(root) {} + +boost::filesystem::path SparkDataDirectory::root() const { return root_; } + +boost::filesystem::path SparkDataDirectory::log_dir() const { + return root_ / "logging"; +} + +boost::filesystem::path SparkDataDirectory::log_file() const { + return log_dir() / "log"; +} + +boost::filesystem::path SparkDataDirectory::log_cache_file() const { + return log_dir() / "log_cache"; +} + +boost::filesystem::path SparkDataDirectory::database_dir( + const std::string &name) const { + return root_ / "databases" / (name + "/"); +} + +boost::filesystem::path SparkDataDirectory::database_file( + const std::string &name) const { + return root_ / "databases" / name / (name + ".sqlite"); +} + +boost::filesystem::path SparkDataDirectory::database_file( + const std::string &dir_name, const std::string &db_name) const { + return root_ / "databases" / dir_name / (db_name + ".sqlite"); +} + +} // namespace bts::spark diff --git a/src/lib/os/spark_data_directory.h b/src/lib/os/spark_data_directory.h new file mode 100644 index 000000000..e33248f85 --- /dev/null +++ b/src/lib/os/spark_data_directory.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +namespace bts::spark { + +class SparkDataDirectory { + public: + SparkDataDirectory(); + + explicit SparkDataDirectory(const std::string &root); + + boost::filesystem::path root() const; + boost::filesystem::path log_dir() const; + boost::filesystem::path log_file() const; + boost::filesystem::path log_cache_file() const; + + boost::filesystem::path database_dir(const std::string &name) const; + boost::filesystem::path database_file(const std::string &name) const; + boost::filesystem::path database_file( + const std::string &dir_name, const std::string &db_name) const; + + private: + boost::filesystem::path root_; +}; + +} // namespace bts::spark diff --git a/src/lib/persistence/IDGenerator.cpp b/src/lib/persistence/IDGenerator.cpp index 2ee855258..7cf38724e 100644 --- a/src/lib/persistence/IDGenerator.cpp +++ b/src/lib/persistence/IDGenerator.cpp @@ -60,24 +60,21 @@ std::optional IDGenerator::get_id( bool add_if_new) { std::string ns = fmt::format( "{}.{}", Venue_Name(venue).substr(6), - SecurityType_Name(type).substr(12)); + SecurityType_Name(type).substr(14)); return get_id(ns, name, add_if_new); } std::optional IDGenerator::get_id( const std::string &ns, const std::string &key, bool add_if_new) { std::lock_guard myLock(_instance_mx); - auto [exists, val] = me->kv_store->retrieve_one(ns, key); + auto [exists, rowid, val] = me->kv_store->retrieve_one(ns, key); if (exists) { - uint64_t rv = str_to_id(val); - LOG(Gen, Diag, "IDGenerator::get_id ns: {} key:{} id: {}", ns, key, rv); - return std::optional(rv); - + LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rowid); + return std::optional(rowid); } else { if (add_if_new) { auto all = me->kv_store->retrieve_all(); - uint64_t new_id = get_next_id(all); - me->kv_store->insert(ns, key, std::to_string(new_id)); + auto new_id = me->kv_store->insert(ns, key, ""); LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, new_id); return std::optional(new_id); @@ -88,13 +85,6 @@ std::optional IDGenerator::get_id( } } -std::uint64_t IDGenerator::get_or_add( - const std::string &ns, const std::string &key) { - auto id = get_id(ns, key, true); - bts_assert(id.has_value()); - return *id; -} - std::uint64_t IDGenerator::get_or_add( api::domain::Venue venue, api::domain::SecurityType type, diff --git a/src/lib/persistence/IStore.h b/src/lib/persistence/IStore.h index 1f33ddf28..29e38eb7a 100644 --- a/src/lib/persistence/IStore.h +++ b/src/lib/persistence/IStore.h @@ -1,8 +1,13 @@ #pragma once +#include "lib/persistence/store_row.h" + +#include #include #include #include +#include +#include namespace bts { namespace persistence { @@ -11,17 +16,45 @@ class IStore { public: virtual std::map> retrieve_all() = 0; - virtual std::pair - retrieve_one(std::string const &ns, std::string const &k) = 0; + + /** + * Finds the row with the specified namespace and key. + * + * @return std::tuple The first item indicates + * whether or not a row was found. For successful queries, the second and third + * items are the rowid and value, respectively. + */ + virtual std::tuple retrieve_one( + std::string const &ns, std::string const &k) = 0; virtual void remove(std::string const &ns, std::set const &keys) = 0; virtual void remove_all(std::string const &ns) = 0; - virtual void insert( + virtual std::int64_t insert( std::string const &ns, std::string const &k, std::string const &v) = 0; + + void insert_many(const std::vector &rows); + void insert_many_resolve_ids(std::vector &rows); + virtual void update( std::string const &ns, std::string const &k, std::string const &v) = 0; virtual ~IStore(); }; +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline void IStore::insert_many(const std::vector &rows) { + for (const auto &row : rows) { + insert(row.ns, row.key, row.value); + } +} + +inline void IStore::insert_many_resolve_ids(std::vector &rows) { + for (auto &row : rows) { + row.id = insert(row.ns, row.key, row.value); + } +} + } // namespace persistence } // namespace bts diff --git a/src/lib/persistence/SqliteStore.cpp b/src/lib/persistence/SqliteStore.cpp index 84e741ea3..50bf7cbf6 100644 --- a/src/lib/persistence/SqliteStore.cpp +++ b/src/lib/persistence/SqliteStore.cpp @@ -35,26 +35,28 @@ static boost::filesystem::path get_db_path(const std::string &str) { } struct SqliteStore::PImpl { + std::string const db; + boost::filesystem::path db_path; + sqlite3_ptr conn{nullptr, sqlite3_close}; + sqlite3_stmt_ptr insert{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr insert_many{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr insert_with_id{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr remove{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr remove_all{nullptr, sqlite3_finalize}; + sqlite3_stmt_ptr update{nullptr, sqlite3_finalize}; + PImpl(const boost::filesystem::path &db_path) - : db(db_path.stem().string()), db_path(db_path), - conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize), - remove(nullptr, sqlite3_finalize), - remove_all(nullptr, sqlite3_finalize), - update(nullptr, sqlite3_finalize) { + : db(db_path.stem().string()), db_path(db_path) { bts_assert(db.size()); } PImpl(std::string const &store_root, std::string const &db_name) : PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {} - std::string const db; - boost::filesystem::path db_path; - sqlite3_ptr conn; - sqlite3_stmt_ptr insert; - sqlite3_stmt_ptr remove; - sqlite3_stmt_ptr remove_all; - sqlite3_stmt_ptr update; - BTS_NO_COPY(PImpl); + explicit PImpl(sqlite3_ptr conn) : db(), db_path(), conn(std::move(conn)) {} + + PImpl(const PImpl &) = delete; + PImpl &operator=(const PImpl &) = delete; }; SqliteStore::SqliteStore(std::string const &db) @@ -64,54 +66,64 @@ SqliteStore::SqliteStore(std::string const &db) init_db(); } +SqliteStore::SqliteStore(sqlite3_ptr db) + : me(std::make_unique(std::move(db))) { + LOG(persist, Info, "SqliteStore ctr. db: "); + init_db(); +} + void SqliteStore::init_db() { - auto path = me->db_path; - auto parent = path.parent_path(); + const auto path = me->db_path; - if (!boost::filesystem::is_directory(parent)) - bts_assert(boost::filesystem::create_directories(parent)); - - if (!boost::filesystem::exists(path)) { - LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", - me->db, path.string()); - - me->conn = sqlite::open_db(path.c_str()); - bts_assert_msg(me->conn, path.c_str()); - - const char *create_schema_sql = - "create table store ( ns text, key text, value text)"; - sqlite::exec_nofail(me->conn, create_schema_sql); - - const char *key_index_sql = - "create unique index key_index on store(ns,key)"; - sqlite::exec_nofail(me->conn, key_index_sql); - - const char *ns_index_sql = "create index ns_index on store(ns)"; - sqlite::exec_nofail(me->conn, ns_index_sql); - } else { - me->conn = sqlite::open_db(path.c_str()); + if (!path.empty()) { + if (!boost::filesystem::is_directory(path.parent_path())) { + bts_assert(boost::filesystem::create_directories(path.parent_path())); + } + if (!boost::filesystem::exists(path)) { + LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", + me->db, path.string()); + } } + auto &conn = me->conn; + if (conn == nullptr) { + conn = sqlite::open_db(path.c_str()); + bts_assert(conn != nullptr); + } + sqlite::exec_nofail( + conn, + "CREATE TABLE IF NOT EXISTS store (ns TEXT, key TEXT, value TEXT)"); + sqlite::exec_nofail( + conn, "CREATE UNIQUE INDEX IF NOT EXISTS key_index on store(ns, key)"); + sqlite::exec_nofail( + conn, "CREATE INDEX IF NOT EXISTS ns_index on store(ns)"); + conn = sqlite::open_db(path.c_str()); + // Asynchronous for speed - sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;"); - sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL"); + sqlite::exec_nofail(conn, "PRAGMA synchronous = 0;"); + sqlite::exec_nofail(conn, "PRAGMA journal_mode=WAL"); // prepare insert statement const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)"; - me->insert = sqlite::prepare_query(me->conn, insert_sql); + me->insert = sqlite::prepare_query(conn, insert_sql); + + // prepare insert_with_id statement + const char *insert_with_id_sql = + "insert into store (rowid, ns, key, value) values (?,?,?,?)"; + me->insert_with_id = sqlite::prepare_query(conn, insert_with_id_sql); // prepare update statement const char *update_sql = "update store set value = ? where ns = ? and key = ?"; - me->update = sqlite::prepare_query(me->conn, update_sql); + me->update = sqlite::prepare_query(conn, update_sql); // prepare remove statement const char *remove_all_sql = "delete from store where ns = ?"; - me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql); + me->remove_all = sqlite::prepare_query(conn, remove_all_sql); // prepare remove statement const char *remove_sql = "delete from store where ns = ? and key = ?"; - me->remove = sqlite::prepare_query(me->conn, remove_sql); + me->remove = sqlite::prepare_query(conn, remove_sql); } SqliteStore::~SqliteStore() {} @@ -140,25 +152,27 @@ SqliteStore::Data SqliteStore::retrieve_all() { return result; } -std::pair SqliteStore::retrieve_one( +std::tuple SqliteStore::retrieve_one( std::string const &ns, std::string const &k) { auto query_str = fmt::format( - "select ns,key,value from store where ns='{}' and key='{}'", ns, k); + "select rowid,ns,key,value from store where ns='{}' and key='{}'", ns, + k); sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str()); - std::pair result{false, ""}; - + auto result = std::make_tuple(false, 0, ""); auto pq = query.get(); auto sqlite_result = sqlite3_step(pq); if (sqlite_result == SQLITE_ROW) { + std::int64_t rowid = sqlite3_column_int64(pq, 0); std::string ns( - (char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0)); - std::string key( (char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1)); + std::string key( + (char *)sqlite3_column_text(pq, 2), sqlite3_column_bytes(pq, 2)); std::string val( - (char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2)); - result.first = true; - result.second = std::move(val); + (char *)sqlite3_column_blob(pq, 3), sqlite3_column_bytes(pq, 3)); + std::get<0>(result) = true; + std::get<1>(result) = rowid; + std::get<2>(result) = std::move(val); } return result; } @@ -167,91 +181,185 @@ void SqliteStore::remove( std::string const &ns, std::set const &keys) { bts_assert(ns.size()); - for (auto &k : keys) { - SQLITE_CHK( - sqlite3_bind_text( - me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->remove.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not delete row in store. Will retry once. error_code: {} " - "ns:{}, k: {}", - r, ns, k); - sleep_for(2s); - r = sqlite3_step(me->remove.get()); + try { + for (auto &k : keys) { + SQLITE_CHK( + sqlite3_bind_text( + me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->remove.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to delete row. Bailing out. error_code: " - "{}", - r); - bts_assert(r == SQLITE_DONE); + "Could not delete row in store. Will retry once. error_code: " + "{} " + "ns:{}, k: {}", + r, ns, k); + sleep_for(2s); + r = sqlite3_step(me->remove.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to delete row. Bailing out. " + "error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); } - SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::remove threw. what: {}", + std::string{ex.what()}); } } void SqliteStore::remove_all(std::string const &ns) { bts_assert(ns.size()); - SQLITE_CHK( - sqlite3_bind_text( - me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->remove_all.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not delete all rows in store. Will retry once. error_code: {} " - "ns:{}", - r, ns); - sleep_for(2s); - r = sqlite3_step(me->remove_all.get()); + try { + SQLITE_CHK( + sqlite3_bind_text( + me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->remove_all.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to delete all rows. Bailing out. " - "error_code: " - "{}", - r); - bts_assert(r == SQLITE_DONE); + "Could not delete all rows in store. Will retry once. error_code: " + "{} " + "ns:{}", + r, ns); + sleep_for(2s); + r = sqlite3_step(me->remove_all.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to delete all rows. Bailing out. " + "error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::remove_all threw. what: {}", + std::string{ex.what()}); } - SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); } -void SqliteStore::insert( +std::int64_t SqliteStore::insert( std::string const &ns, std::string const &k, std::string const &v) { - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), - me->conn); - SQLITE_CHK( - sqlite3_bind_text( - me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), - me->conn); - auto r = sqlite3_step(me->insert.get()); - if (r != SQLITE_DONE) { - LOG(persist, Error, - "Could not insert row in store. Will retry once. error_code: {} ns: " - "{}, k: {}, v: {}", - r, ns, k, v); - sleep_for(2s); - r = sqlite3_step(me->insert.get()); + try { + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK( + sqlite3_bind_text( + me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->insert.get()); if (r != SQLITE_DONE) { LOG(persist, Error, - "Failed a second time to insert row. Bailing out. error_code: {}", - r); - bts_assert(r == SQLITE_DONE); + "Could not insert row in store. Will retry once. error_code: {} " + "ns: " + "{}, k: {}, v: {}", + r, ns, k, v); + sleep_for(2s); + r = sqlite3_step(me->insert.get()); + if (r != SQLITE_DONE) { + LOG(persist, Error, + "Failed a second time to insert row. Bailing out. error_code: " + "{}", + r); + bts_assert(r == SQLITE_DONE); + } } + const auto rowid = sqlite3_last_insert_rowid(me->conn.get()); + SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); + return rowid; + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::insert threw. what: {}", + std::string{ex.what()}); } - SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); + return 0; +} + +void SqliteStore::insert_many(const std::vector &rows) { + LOG(md, Info, "SqliteStore::insert_many size: {}", rows.size()); + if (rows.empty()) { + return; + } + try { + sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION"); + sqlite3 *conn_rawp = me->conn.get(); + sqlite3_stmt *insert_rawp = me->insert.get(); + sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get(); + for (const auto &row : rows) { + int col_idx = 1; + sqlite3_stmt *stmt; + if (row.id > 0) { + stmt = insert_with_id_rawp; + sqlite3_bind_int(stmt, col_idx++, row.id); + } else { + stmt = insert_rawp; + } + sqlite3_bind_text( + stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC); + sqlite3_bind_text( + stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC); + int result = sqlite3_step(stmt); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + sqlite3_last_insert_rowid(conn_rawp); + } + sqlite3_reset(stmt); + } + sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION"); + } catch (const std::exception &ex) { + LOG(persist, Error, "SqliteStore::insert_many threw. what: {}", + std::string{ex.what()}); + } +} + +void SqliteStore::insert_many_resolve_ids(std::vector &rows) { + LOG(md, Info, "SqliteStore::insert_many_resolve_ids size: {}", rows.size()); + if (rows.empty()) { + return; + } + sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION"); + sqlite3 *conn_rawp = me->conn.get(); + sqlite3_stmt *insert_rawp = me->insert.get(); + sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get(); + for (auto &row : rows) { + int col_idx = 1; + sqlite3_stmt *stmt; + if (row.id > 0) { + stmt = insert_with_id_rawp; + sqlite3_bind_int(stmt, col_idx++, row.id); + } else { + stmt = insert_rawp; + } + sqlite3_bind_text( + stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC); + sqlite3_bind_text( + stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC); + int result = sqlite3_step(stmt); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + if (row.id > 0) { + bts_assert(sqlite3_last_insert_rowid(conn_rawp) == row.id); + } else { + row.id = sqlite3_last_insert_rowid(conn_rawp); + } + } + sqlite3_reset(stmt); + } + sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION"); } void SqliteStore::update( diff --git a/src/lib/persistence/SqliteStore.h b/src/lib/persistence/SqliteStore.h index 4f0caf725..6a833df80 100644 --- a/src/lib/persistence/SqliteStore.h +++ b/src/lib/persistence/SqliteStore.h @@ -1,7 +1,12 @@ #pragma once #include "IStore.h" +#include "lib/persistence/store_row.h" +#include "lib/utils/Sqlite_utils.h" + +#include #include +#include namespace bts { namespace persistence { @@ -9,22 +14,26 @@ namespace persistence { class SqliteStore : public IStore { public: SqliteStore(std::string const &db); + explicit SqliteStore(sqlite3_ptr db); ~SqliteStore(); static SqliteStore from_path(const std::string &full_path); typedef std::map> Data; Data retrieve_all() override; - std::pair retrieve_one( + + std::tuple retrieve_one( std::string const &ns, std::string const &k) override; void remove( std::string const &ns, std::set const &keys) override; void remove_all(std::string const &ns) override; - void insert( + std::int64_t insert( std::string const &ns, std::string const &k, std::string const &v) override; + void insert_many(const std::vector &rows); + void insert_many_resolve_ids(std::vector &rows); void update( std::string const &ns, std::string const &k, diff --git a/src/lib/persistence/store_row.h b/src/lib/persistence/store_row.h new file mode 100644 index 000000000..1c7b7f079 --- /dev/null +++ b/src/lib/persistence/store_row.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace bts::persistence { + +struct StoreRow { + std::string ns; + std::string key; + std::string value; + /** + * Represents the unique 'rowid' in the sqlite db. A value of 0 indicates that + * the rowid isn't known, and should either be generated by the db (for inserts), + * or resolved by finding the row with matching ns, key, value (for retrieves). + */ + std::int64_t id{0}; +}; + +} // namespace bts::persistence diff --git a/src/lib/utils/checked_arithmetic.h b/src/lib/utils/checked_arithmetic.h new file mode 100644 index 000000000..91e60a6b9 --- /dev/null +++ b/src/lib/utils/checked_arithmetic.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +namespace bts::system::math { + +static std::uint32_t checked_ltou(std::int64_t l); +static std::uint64_t checked_ltoul(std::int64_t l); + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::uint32_t checked_ltou(std::int64_t l) { + if (l < 0) { + throw std::overflow_error{ + "Failed to convert int64 to uint32 (value < 0)"}; + } + if (l > + static_cast(std::numeric_limits::max())) { + throw std::overflow_error{ + "Failed to convert int64 to uint32. value > max)"}; + } + return static_cast(l); +} + +inline std::uint64_t checked_ltoul(std::int64_t l) { + if (l < 0) { + throw std::overflow_error{ + "Failed to convert int64 to uint64 (value < 0)"}; + } + return static_cast(l); +} + +} // namespace bts::system::math diff --git a/src/server/sparkd/analytics/AnalyticsServiceServer.cpp b/src/server/sparkd/analytics/AnalyticsServiceServer.cpp index 7cd0cc84a..83ff900fd 100644 --- a/src/server/sparkd/analytics/AnalyticsServiceServer.cpp +++ b/src/server/sparkd/analytics/AnalyticsServiceServer.cpp @@ -52,7 +52,7 @@ void add_or_update_to_table( const std::string &ns, const std::string &key, const std::string &val) { - auto [exists, existing_val] = store->retrieve_one(ns, key); + auto [exists, rowid, existing_val] = store->retrieve_one(ns, key); LOG(Gen, Info, "add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns, key, val, exists, existing_val); @@ -386,7 +386,8 @@ void AnalyticsServiceServer::save_report( const api::Response<::bts::api::ResultMessage> &resp) { Report report = report_arg; std::string key = std::to_string(report.id()); - auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key); + auto [present, rowid, val] = + me->reports_store->retrieve_one(me->reports_ns, key); if (!present) { int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1; report.set_id(new_id); @@ -408,7 +409,8 @@ void AnalyticsServiceServer::delete_report( const bts::api::analytics::IntWrapper &id, const api::Response<::bts::api::ResultMessage> &resp) { std::string key = std::to_string(id.data()); - auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key); + auto [present, rowid, val] = + me->reports_store->retrieve_one(me->reports_ns, key); api::ResultMessage res; res.set_success(false); if (!present) { diff --git a/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp b/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp index 055e718b4..5acc34f92 100644 --- a/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp +++ b/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp @@ -581,10 +581,11 @@ void negate_id(api::Security *sec) { } void ExchangeSimApplication::persist_book(OrderBook const &b) { - auto existing = me->store->retrieve_one("books", b.sec_desc()); + auto [exists, rowid, oldval] = + me->store->retrieve_one("books", b.sec_desc()); auto val = utils::pb::to_string(b); - if (existing.first) + if (exists) me->store->update("books", b.sec_desc(), val); else me->store->insert("books", b.sec_desc(), val); diff --git a/src/server/sparkd/md/emumba/EmumbaDeribitData.h b/src/server/sparkd/md/emumba/EmumbaDeribitData.h index 81295af80..c35d44075 100644 --- a/src/server/sparkd/md/emumba/EmumbaDeribitData.h +++ b/src/server/sparkd/md/emumba/EmumbaDeribitData.h @@ -18,10 +18,11 @@ struct EmumbaDeribitData { EmumbaDeribitData() : sec_index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(sec_index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(sec_index.get_key(k.value())); + }, + &utils::Dispatcher::current()), md_producer(recorder), listener(sec_index, md_producer) {} }; } // namespace bts::md diff --git a/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp b/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp index 748a89251..70e8e0eee 100644 --- a/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp +++ b/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp @@ -43,10 +43,11 @@ struct EmumbaMain::PImpl { PImpl() : sec_index(), recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(sec_index.get_key(k.value())); - }}), + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(sec_index.get_key(k.value())); + }, + &utils::Dispatcher::current()), producer(recorder), listener(sec_index, producer) { const auto config_path = Path{config::get().config_path()} / diff --git a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp index 4bcd0c9d4..0dc491f5d 100644 --- a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp +++ b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.cpp @@ -1,70 +1,66 @@ #include "ICEMain.h" #include "lib/config/config.h" -#include "lib/md/SecurityDB.h" -#include "lib/md/coconut2.h" -#include "lib/md/ice/ICEUtils.h" -#include "lib/md/protos/co2.pb.h" -#include "lib/md/vendor/OnixsUtils.h" -#include "lib/os/os.h" -#include "lib/utils/Dispatcher.h" -#include "lib/utils/bts_chrono.h" -#include "lib/utils/helper_macros.h" -#include "lib/utils/proto_helper.h" - +#include "lib/config/protos/deployment_type.pb.h" #include "lib/config/protos/market_data.pb.h" #include "lib/config/protos/system_config.pb.h" -#include "lib/utils/thread_manager.h" +#include "lib/md/ice/ICEUtils.h" +#include "lib/md/vendor/OnixsUtils.h" +#include #include #include -#include - -void recursive_copy( - const boost::filesystem::path &src, const boost::filesystem::path &dst) { - if (boost::filesystem::exists(dst)) - throw std::runtime_error(dst.generic_string() + " exists"); - if (boost::filesystem::is_directory(src)) { - boost::filesystem::create_directories(dst); - for (boost::filesystem::directory_entry &item : - boost::filesystem::directory_iterator(src)) - recursive_copy(item.path(), dst / item.path().filename()); - } else if (boost::filesystem::is_regular_file(src)) - boost::filesystem::copy(src, dst); - else - throw std::runtime_error(dst.generic_string() + " not dir or file"); -} using namespace std; using namespace bts::co2; using std::size_t; -namespace bts { -namespace md { -namespace ice { +namespace bts::md::ice { using bts::config::SystemConfig; using Path = boost::filesystem::path; -ICEMain::ICEMain() - : _index(), _recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(_index.get_key(k.value())); - }}), - _producer(_recorder), _handlers(), _listeners() {} +struct ICEMain::PImpl { + ICEOptions options; + std::string config_file; + ICESecurityIndex index; + SecurityIDRecorder recorder; + MDProducer producer; + std::vector>> + handlers; + std::vector> listeners; -ICEMain::~ICEMain() {} + PImpl(ICEOptions options) + : options(options), + config_file((Path{config::get().data_path()} / + "onixs_ice" / this->options.ice_config_file()) + .string()), + index(), recorder( + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(index.get_key(k.value())); + }, + &utils::Dispatcher::current()), + producer(recorder), handlers(), listeners() {} -const IceChannelConfig channel_settings_for(const ICEChannel &channel) { - const auto &config = config::get(); - for (const auto &ch : config.channel()) { - if (ch.name() == channel.name) - return ch; + IceChannelConfig channel_settings_for(const ICEChannel &channel) { + for (const auto &ch : options.channel()) { + if (ch.name() == channel.name) + return ch; + } + return {}; } - return {}; +}; + +ICEMain::ICEMain() : ICEMain(config::get()) {} + +ICEMain::ICEMain(ICEOptions options) + : me_(std::make_unique(std::move(options))) { + me_->options = std::move(options); } +ICEMain::~ICEMain() = default; + onixs_ice::MarketSubscriptions market_subcription_for_channel( const ICEChannel &chan) { onixs_ice::MarketSubscriptions subs; @@ -90,23 +86,30 @@ onixs_ice::MarketSubscriptions market_subcription_for_channel( return subs; } -void ICEMain::start(const std::vector &symbols) { - auto ice_conf_location = - Path{config::get().data_path()} / "onixs_ice/"; - if (!boost::filesystem::is_directory(ice_conf_location)) { - auto conf_path = Path{os::sparkd_wd()} / "data_files" / "onixs_ice/"; - recursive_copy(Path{conf_path}, ice_conf_location); +static void validate_config_file(const std::string &config_file) { + if (!boost::filesystem::exists(config_file)) { + const std::string config_url = + config::get().type() == "dev" + ? "https://community.theice.com/docs/DOC-19989" + : "https://drive.google.com/" + "open?id=13_eyF2pKtn3F7chgsa_75ftUdf7emMbp"; + bts_assert_msgf( + false, "Ice config file not found '{}'. Please download from {}", + config_file, config_url); } +} +void ICEMain::start(const std::vector &symbols) { await_ice_config(); + validate_config_file(me_->config_file); auto license = onixs_create_temporary_license(); const auto &channels = parse_ice_channels(symbols); for (const auto &channel : channels) { unique_ptr phandler = prepare_handler(channel, license.path.string()); - _listeners.push_back(prepare_listener(*phandler, channel)); - _handlers.push_back(std::make_pair(channel, std::move(phandler))); + me_->listeners.push_back(prepare_listener(*phandler, channel)); + me_->handlers.push_back(std::make_pair(channel, std::move(phandler))); std::stringstream symstr; for (const auto &sym : channel.symbols) @@ -115,7 +118,7 @@ void ICEMain::start(const std::vector &symbols) { symstr.str()); } - for (auto &pair : _handlers) { + for (auto &pair : me_->handlers) { LOG(md, Info, "ICE: Init Handler {}", pair.first.to_string()); sleep_for(16s); onixs_ice::MarketSubscriptions subs = @@ -132,7 +135,7 @@ void ICEMain::start(const std::vector &symbols) { } void ICEMain::send_heartbeats() { - for (const auto &listener : _listeners) + for (const auto &listener : me_->listeners) listener->heartbeat(); } @@ -140,20 +143,18 @@ void ICEMain::send_heartbeats() { // Initialize HandlerSettings and allocate the channel Handler unique_ptr ICEMain::prepare_handler( const ICEChannel &channel, const std::string &license) { - const auto &options = channel_settings_for(channel); + const auto &options = me_->channel_settings_for(channel); LOG(md, Info, "Loaded channel options for channel {}:\n{}", channel.to_string(), options.DebugString()); onixs_ice::HandlerSettings settings; settings.licenseDirectory = license; - // - // Logging settings.logLevel = (onixs_ice::LogLevels::Enum)options.onixs_log_mode(); - settings.logDirectory = ice_log_path(); - - // - // Config file - settings.connectivityConfiguration = ice_config_path(); + settings.logDirectory = + (boost::filesystem::path{config::get().data_path()} / + "logging" / "onixs_ice/") + .string(); + settings.connectivityConfiguration = me_->config_file; // // Network settings @@ -231,12 +232,12 @@ unique_ptr ICEMain::prepare_listener( // heartbeats vector producer_channels; for (auto &symbol : channel.symbols) { - producer_channels.push_back(_producer.get_channel( + producer_channels.push_back(me_->producer.get_channel( symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } unique_ptr listener(new ICEListener( - _index, _producer, /*_settlement_recorder,*/ producer_channels)); + me_->index, me_->producer, /*_settlement_recorder,*/ producer_channels)); for (auto &symbol : channel.symbols) listener->add_filter_symbol(symbol.symbol()); @@ -254,6 +255,4 @@ unique_ptr ICEMain::prepare_listener( return listener; } -} // namespace ice -} // namespace md -} // namespace bts +} // namespace bts::md::ice diff --git a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h index 1ea217a39..8c37f9476 100644 --- a/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h +++ b/src/server/sparkd/md/platforms/ice_onixs/ICEMain.h @@ -1,21 +1,12 @@ #pragma once #include "ICEListener.h" -#include "ICESecurityIndex.h" -#include "lib/md/coconut2.h" -#include "lib/utils/bts_no_copy.h" -#include "onixs.h" #include "server/sparkd/md/platforms/IMDPlatform.h" -#include #include #include -namespace bts { -namespace md { -namespace ice { - -// void run_onixs_ice_server(UserSymbol* sym, size_t symcount); +namespace bts::md::ice { struct ICEChannel; @@ -28,27 +19,22 @@ struct ICEChannel; class ICEMain : public IMDPlatform { public: ICEMain(); + explicit ICEMain(ICEOptions options); + ICEMain(const ICEMain &) = delete; + ICEMain &operator=(const ICEMain &) = delete; ~ICEMain(); void start(const std::vector &symbols); void send_heartbeats() override; - std::unique_ptr - prepare_handler(const ICEChannel &channel, const std::string &license); - std::unique_ptr - prepare_listener(onixs_ice::Handler &handler, const ICEChannel &channel); + std::unique_ptr prepare_handler( + const ICEChannel &channel, const std::string &license); + std::unique_ptr prepare_listener( + onixs_ice::Handler &handler, const ICEChannel &channel); private: - BTS_NO_COPY(ICEMain); - - ICESecurityIndex _index; - SecurityIDRecorder _recorder; - MDProducer _producer; - std::vector>> - _handlers; - std::vector> _listeners; + struct PImpl; + std::unique_ptr me_; }; -} // namespace ice -} // namespace md -} // namespace bts +} // namespace bts::md::ice diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 9ad5c53ba..0a54040ad 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -1,24 +1,18 @@ #include "MDP3Main.h" - +#include "lib/api/md/GlbxMarketDataConfig.pb.h" #include "lib/config/config.h" +#include "lib/config/protos/deployment_type.pb.h" #include "lib/logging/logging.h" #include "lib/md/MBOProcessor.h" #include "lib/md/SecurityDB.h" #include "lib/md/cme/MDP3Utils.h" -#include "lib/md/coconut2.h" -#include "lib/md/protos/co2.pb.h" +#include "lib/md/cme/mdp3_data_directory.h" #include "lib/md/vendor/OnixsUtils.h" -#include "lib/os/os.h" -#include "lib/utils/Dispatcher.h" -#include "lib/utils/proto_helper.h" - -#include "lib/config/protos/market_data.pb.h" -#include "lib/config/protos/system_config.pb.h" +#include "lib/os/spark_data_directory.h" #include "lib/utils/thread_manager.h" #include #include -#include #include #include @@ -29,16 +23,33 @@ using std::size_t; namespace bts { namespace md { -MDP3Main::MDP3Main() - : _index(), _recorder( - {utils::Dispatcher::current(), - [=, this](ExchangeKey k) { - return SecurityKey(_index.get_key(k.value())); - }}), - _producer(_recorder), _engines(), _handlers(), _listeners(), _recovery() { -} +#define TRADING_STATUS(x) bts::domain::protobufs::TRADINGSTATUS_##x -MDP3Main::~MDP3Main() {} +using Path = boost::filesystem::path; + +struct MDP3Main::PImpl { + Mdp3DataDirectory data_dir; + CMESecurityIndex index; + SecurityIDRecorder recorder; + MDProducer producer; + std::map> engines; + std::map> handlers; + std::vector> listeners; + std::vector> recovery; + PImpl() + : index(), recorder( + nullptr, + [=, this](ExchangeKey k) { + return SecurityKey(index.get_key(k.value())); + }, + &utils::Dispatcher::current()), + producer(recorder), engines(), handlers(), listeners(), recovery() {} +}; + +MDP3Main::MDP3Main() + : me_(std::make_unique()) {} + +MDP3Main::~MDP3Main() = default; const CmeChannelConfig channel_settings_for(const string &id) { const auto &config = config::get(); @@ -173,11 +184,11 @@ void MDP3Main::start( auto license = onixs_create_temporary_license(); initialize(license.path.string()); const auto &channels = parse_channels(symbols); - _engines = create_feed_engines(channels); + me_->engines = create_feed_engines(channels); for (const auto &channel : channels) { unique_ptr phandler = prepare_handler(channel.id, mbo); - _listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd)); - _handlers.insert(std::make_pair(channel.id, std::move(phandler))); + me_->listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd)); + me_->handlers.insert(std::make_pair(channel.id, std::move(phandler))); std::stringstream symstr; for (const auto &sym : channel.symbols) { symstr << sym.symbol() << "-" << SecurityType_Name(sym.type()) << " "; @@ -186,7 +197,7 @@ void MDP3Main::start( symstr.str()); } - for (auto &pair : _handlers) { + for (auto &pair : me_->handlers) { onixs::SessionSettings settings = session_settings_for_channel(pair.first, mbo); pair.second->start(settings); @@ -196,7 +207,7 @@ void MDP3Main::start( } void MDP3Main::send_heartbeats() { - for (const auto &listener : _listeners) { + for (const auto &listener : me_->listeners) { listener->heartbeat(); } } @@ -219,7 +230,8 @@ unique_ptr MDP3Main::prepare_handler( onixs::HandlerSettings settings; settings.channel(std::stoul(channel_id)); - settings.feeds().connectivityConfigurationFile(config_path()); + settings.feeds().connectivityConfigurationFile( + (me_->data_dir.config_file()).string()); // Market data options settings.bookManagement().directBooks().maintain(true); @@ -306,7 +318,8 @@ unique_ptr MDP3Main::prepare_handler( logger = new onixs::NullLogger(); } else { onixs::FileLoggerSettings loggerSettings; - loggerSettings.filename(log_path() + "/log_" + channel_id); + loggerSettings.filename( + (me_->data_dir.log_dir() / ("log_" + channel_id)).string()); loggerSettings.severityLevel( (onixs::LogSeverity::Enum)options.onixs_log_mode()); logger = new onixs::FileLogger(loggerSettings); @@ -338,19 +351,19 @@ unique_ptr MDP3Main::prepare_handler( try { auto handler = unique_ptr(new onixs::Handler(settings)); - auto iter = _engines.find(options.feed_engine()); - if (iter == _engines.end()) { + auto iter = me_->engines.find(options.feed_engine()); + if (iter == me_->engines.end()) { LOG(md, Fatal, "Missing feed engine definition '{}' for channel id '{}'. Add a " "feed_engine entry with name:'{}' to the CMEOptions config key.", options.feed_engine(), channel_id, options.feed_engine()); bts_assert_msg( - iter != _engines.end(), "Missing feed engine definition."); + iter != me_->engines.end(), "Missing feed engine definition."); } handler->bindFeedEngine(*(iter->second)); if (recovery) { handler->bindTcpRecovery(*recovery); - _recovery.push_back(std::move(recovery)); + me_->recovery.push_back(std::move(recovery)); } if (logger) { handler->bindLogger(*logger); @@ -375,12 +388,13 @@ unique_ptr MDP3Main::prepare_listener( // heartbeats vector producer_channels; for (auto &symbol : channel.symbols) { - producer_channels.push_back(_producer.get_channel( + producer_channels.push_back(me_->producer.get_channel( symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } unique_ptr listener(new MDP3Listener( - _index, _producer, mbo, producer_channels, trade_summary_dispatcher)); + me_->index, me_->producer, mbo, producer_channels, + trade_summary_dispatcher)); for (auto &symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index c67dc93d4..fc3043ac3 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -29,7 +29,9 @@ struct CMEChannel; */ class MDP3Main : public IMDPlatform { public: - MDP3Main(); + explicit MDP3Main(); + MDP3Main(const MDP3Main &) = delete; + MDP3Main &operator=(const MDP3Main &) = delete; ~MDP3Main(); void start( @@ -39,8 +41,8 @@ class MDP3Main : public IMDPlatform { void send_heartbeats() override; void initialize(const std::string &license_store); - std::unique_ptr - prepare_handler(const std::string &channel, MBOWriter *); + std::unique_ptr prepare_handler( + const std::string &channel, MBOWriter *); std::unique_ptr prepare_listener( onixs::Handler &handler, const CMEChannel &channel, @@ -48,16 +50,8 @@ class MDP3Main : public IMDPlatform { TradeSummarySignalDispatcher *); private: - BTS_NO_COPY(MDP3Main); - - CMESecurityIndex _index; - SecurityIDRecorder _recorder; - MDProducer _producer; - // co2::SettlementRecorder _settlement_recorder; - std::map> _engines; - std::map> _handlers; - std::vector> _listeners; - std::vector> _recovery; + struct PImpl; + std::unique_ptr me_; }; } // namespace md diff --git a/src/server/sparkd/position/PositionServiceServer.cpp b/src/server/sparkd/position/PositionServiceServer.cpp index c3e511409..26c95f74f 100644 --- a/src/server/sparkd/position/PositionServiceServer.cpp +++ b/src/server/sparkd/position/PositionServiceServer.cpp @@ -293,9 +293,9 @@ void PositionServiceServer::update_manual_fill( auto *this_resp = resp.add_response(); auto id = std::to_string(mf.record_id()); - auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id); - auto found = p.first; - if (!found) { + auto [exists, rowid, val] = + me->manual_fills_store->retrieve_one(manual_fills_ns, id); + if (!exists) { this_resp->set_ok(false); this_resp->set_msg("Could not find record"); } else { @@ -326,9 +326,9 @@ void PositionServiceServer::remove_manual_fill( auto *this_resp = resp.add_response(); auto id = std::to_string(mf.record_id()); - auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id); - auto found = p.first; - if (!found) { + auto [exists, rowid, val] = + me->manual_fills_store->retrieve_one(manual_fills_ns, id); + if (!exists) { this_resp->set_ok(false); this_resp->set_msg("Could not find record"); } else { @@ -367,10 +367,9 @@ void PositionServiceServer::update_security_parameters( req.op_params_size()); for (const auto &p : req.op_params()) { std::string key_str = std::to_string(p.security_key()); - auto db_p = + auto [exists, rowid, val] = me->security_params_store->retrieve_one(security_params_ns, key_str); - auto found = db_p.first; - if (!found) { + if (!exists) { me->security_params_store->insert( security_params_ns, key_str, utils::pb::to_string(p)); } else { diff --git a/src/server/sparkd/user/AdminServiceDB.cpp b/src/server/sparkd/user/AdminServiceDB.cpp index 755ff079d..c89407e0e 100644 --- a/src/server/sparkd/user/AdminServiceDB.cpp +++ b/src/server/sparkd/user/AdminServiceDB.cpp @@ -21,7 +21,8 @@ AdminServiceDB::AdminServiceDB() : me(new PImpl()) {} AdminServiceDB::~AdminServiceDB() {} std::optional AdminServiceDB::retrieve() { - auto [has_table, data] = me->store.retrieve_one(table_name, unique_key); + auto [has_table, rowid, data] = + me->store.retrieve_one(table_name, unique_key); if (!has_table) return {}; @@ -33,7 +34,8 @@ std::optional AdminServiceDB::retrieve() { } bool AdminServiceDB::update(api::admin::AdminData const &state) { - auto [has_table, data] = me->store.retrieve_one(table_name, unique_key); + auto [has_table, rowid, data] = + me->store.retrieve_one(table_name, unique_key); UNUSED(data); std::string ser_state;