refactor: Return row id in IStore::retrieve_one

This is a prerequisite to synchronizing security ids stored via IDGenerator
(the "new" way to store ids, added when we implemented Deribit), and
SecurityDB (the original way from spark days). The problem was that these
2 implementations were using separate databases. We need to unify the
interfaces between these storage services to avoid breaking existing code.
This commit is contained in:
John Colvin committed 2024-10-17 05:47:47 +00:00
1 parent 881651e3d6
commit 2bc8cf6db5
25 files changed
+495 -477

No files matched your search

@@ -50,7 +50,7 @@ void add_or_update_to_table(
const std::string &ns,
const std::string &key,
const std::string &val) {
auto [exists, existing_val] = store->retrieve_one(ns, key);
auto [exists, rowid, existing_val] = store->retrieve_one(ns, key);
LOG(Gen, Info,
"add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns,
key, val, exists, existing_val);
@@ -384,7 +384,8 @@ void AnalyticsServiceServer::save_report(
const api::Response<::bts::api::ResultMessage> &resp) {
Report report = report_arg;
std::string key = std::to_string(report.id());
auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key);
auto [present, rowid, val] =
me->reports_store->retrieve_one(me->reports_ns, key);
if (!present) {
int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1;
report.set_id(new_id);
@@ -406,7 +407,8 @@ void AnalyticsServiceServer::delete_report(
const bts::api::analytics::IntWrapper &id,
const api::Response<::bts::api::ResultMessage> &resp) {
std::string key = std::to_string(id.data());
auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key);
auto [present, rowid, val] =
me->reports_store->retrieve_one(me->reports_ns, key);
api::ResultMessage res;
res.set_success(false);
if (!present) {
@@ -582,10 +582,11 @@ void negate_id(api::Security *sec) {
}
void ExchangeSimApplication::persist_book(OrderBook const &b) {
auto existing = me->store->retrieve_one("books", b.sec_desc());
auto [exists, rowid, oldval] =
me->store->retrieve_one("books", b.sec_desc());
auto val = utils::pb::to_string(b);
if (existing.first)
if (exists)
me->store->update("books", b.sec_desc(), val);
else
me->store->insert("books", b.sec_desc(), val);
@@ -18,10 +18,11 @@ struct EmumbaDeribitData {
EmumbaDeribitData()
: sec_index(), recorder(
{utils::Dispatcher::current(),
[=, this](ExchangeKey k) {
return SecurityKey(sec_index.get_key(k.value()));
}}),
nullptr,
[=, this](ExchangeKey k) {
return SecurityKey(sec_index.get_key(k.value()));
},
&utils::Dispatcher::current()),
md_producer(recorder), listener(sec_index, md_producer) {}
};
} // namespace bts::md
@@ -43,10 +43,11 @@ struct EmumbaMain::PImpl {
PImpl()
: sec_index(), recorder(
{utils::Dispatcher::current(),
[=, this](ExchangeKey k) {
return SecurityKey(sec_index.get_key(k.value()));
}}),
nullptr,
[=, this](ExchangeKey k) {
return SecurityKey(sec_index.get_key(k.value()));
},
&utils::Dispatcher::current()),
producer(recorder), listener(sec_index, producer) {
const auto config_path = Path{config::get<SystemConfig>().config_path()} /
@@ -38,10 +38,11 @@ struct ICEMain::PImpl {
"onixs_ice" / this->options.ice_config_file())
.string()),
index(), recorder(
{utils::Dispatcher::current(),
[=, this](ExchangeKey k) {
return SecurityKey(index.get_key(k.value()));
}}),
nullptr,
[=, this](ExchangeKey k) {
return SecurityKey(index.get_key(k.value()));
},
&utils::Dispatcher::current()),
producer(recorder), handlers(), listeners() {}
IceChannelConfig channel_settings_for(const ICEChannel &channel) {
@@ -43,10 +43,11 @@ struct MDP3Main::PImpl {
: data_path(spark::SparkDataDirectory{}.root() / "cme_mdp3/"),
log_path(spark::SparkDataDirectory{}.log_dir() / "onixs/"), index(),
recorder(
{utils::Dispatcher::current(),
[=, this](ExchangeKey k) {
return SecurityKey(index.get_key(k.value()));
}}),
nullptr,
[=, this](ExchangeKey k) {
return SecurityKey(index.get_key(k.value()));
},
&utils::Dispatcher::current()),
producer(recorder), engines(), handlers(), listeners(), recovery() {}
};
@@ -293,9 +293,9 @@ void PositionServiceServer::update_manual_fill(
auto *this_resp = resp.add_response();
auto id = std::to_string(mf.record_id());
auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id);
auto found = p.first;
if (!found) {
auto [exists, rowid, val] =
me->manual_fills_store->retrieve_one(manual_fills_ns, id);
if (!exists) {
this_resp->set_ok(false);
this_resp->set_msg("Could not find record");
} else {
@@ -326,9 +326,9 @@ void PositionServiceServer::remove_manual_fill(
auto *this_resp = resp.add_response();
auto id = std::to_string(mf.record_id());
auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id);
auto found = p.first;
if (!found) {
auto [exists, rowid, val] =
me->manual_fills_store->retrieve_one(manual_fills_ns, id);
if (!exists) {
this_resp->set_ok(false);
this_resp->set_msg("Could not find record");
} else {
@@ -367,10 +367,9 @@ void PositionServiceServer::update_security_parameters(
req.op_params_size());
for (const auto &p : req.op_params()) {
std::string key_str = std::to_string(p.security_key());
auto db_p =
auto [exists, rowid, val] =
me->security_params_store->retrieve_one(security_params_ns, key_str);
auto found = db_p.first;
if (!found) {
if (!exists) {
me->security_params_store->insert(
security_params_ns, key_str, utils::pb::to_string(p));
} else {
@@ -20,7 +20,8 @@ AdminServiceDB::AdminServiceDB() : me(new PImpl()) {}
AdminServiceDB::~AdminServiceDB() {}
std::optional<api::admin::AdminData> AdminServiceDB::retrieve() {
auto [has_table, data] = me->store.retrieve_one(table_name, unique_key);
auto [has_table, rowid, data] =
me->store.retrieve_one(table_name, unique_key);
if (!has_table)
return {};
@@ -32,7 +33,8 @@ std::optional<api::admin::AdminData> AdminServiceDB::retrieve() {
}
bool AdminServiceDB::update(api::admin::AdminData const &state) {
auto [has_table, data] = me->store.retrieve_one(table_name, unique_key);
auto [has_table, rowid, data] =
me->store.retrieve_one(table_name, unique_key);
UNUSED(data);
std::string ser_state;
@@ -2,7 +2,7 @@
#include "spark/config/config.h"
#include "spark/config/protos/orders_fills_cleanup.pb.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/md/invalid_security_key.h"
#include "spark/md/common.h"
#include "spark/utils/datetime.h"
#include <boost/date_time/posix_time/posix_time.hpp>
+1 -78
View File
@@ -59,85 +59,8 @@ string dump_header(const ColdHeader &header) {
return JsonDumper::dump(header);
}
void write_date_string(
uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result);
string ShmDumper::dump_security(const FixedSecurity &security) {
ostringstream res;
res << "{";
res << "'key': " << security.key << ", ";
res << "'symbol': '" << security.symbol() << "', ";
res << "'type': '" << api::SecurityType_Name(security.type) << "', ";
res << "'exchange': '" << api::Venue_Name(security.exchange) << "', ";
res << "'alt_symbol': '" << security.alt_symbol() << "', ";
res << "'cfi_code': '" << security.cfi_code() << "', ";
res << "'security_desc': '" << security.security_desc() << "', ";
res << "'platform_id': '" << security.platform_id() << "', ";
res << "'display_factor': " << (int)security.display_factor << ", ";
res << "'numeric_id': " << security.numeric_id << ", ";
res << "'cboe_class_key': " << security.cboe_class_key << ", ";
res << "'main_frac': " << (int)security.cme_main_fraction << ", ";
res << "'sub_fract': " << (int)security.cme_sub_fraction << ", ";
res << "'price_format': " << (int)security.cme_price_display_format << ", ";
res << "'market_segment': " << (int)security.cme_market_segment_id << ", ";
res << "'price_break': " << security.price_break << ", ";
res << "'low_tick': " << security.low_tick << ", ";
res << "'high_tick': " << security.high_tick << ", ";
res << "'cpp': " << security.cash_per_point << ", ";
res << "'market_description ': " << security.ice_market_description()
<< ", ";
res << "'cleared_alias ': " << security.ice_cleared_alias() << ", ";
res << "'product_description ': " << security.ice_product_name() << ", ";
res << "'hub_alias ': " << security.ice_hub_alias() << ", ";
res << "'strip_name': " << security.ice_strip_name() << ", ";
res << "'mic_code': " << security.mic_code() << ", ";
switch (security.type) {
case SECURITYTYPE_SPOT:
break;
case SECURITYTYPE_FUTURE: {
auto *psec = static_cast<const FixedFutureSecurity *>(&security);
res << "'expiration': '";
write_date_string(
psec->expiration_year, psec->expiration_month, psec->expiration_day,
res);
res << "' ";
break;
}
case SECURITYTYPE_OPTION: {
auto *psec = static_cast<const FixedOptionSecurity *>(&security);
res << "'expiration': '";
write_date_string(
psec->expiration_year, psec->expiration_month, psec->expiration_day,
res);
res << "', ";
res << "'strike': " << psec->strike << ", ";
res << "'callput': '" << api::PutOrCall_Name(psec->call_put) << "' ";
break;
}
case SECURITYTYPE_SPREAD: {
auto *psec = static_cast<const FixedSpreadSecurityXL *>(&security);
res << "'legs': [";
for (size_t x = 0; x < psec->leg_count(); ++x) {
auto &leg = psec->legs[x];
if (x > 0)
res << ",";
res << "{ 'key': " << leg.key << ", 'exkey': " << leg.exchange_key
<< ", 'quantity': " << leg.leg_quantity
<< ", 'leg_price': " << leg.leg_price
<< ", 'delta': " << leg.leg_delta << ", 'side': '"
<< api::Side_Name(leg.leg_side) << "' }";
}
res << "]";
break;
}
default:
break;
}
res << "}";
return res.str();
return JsonDumper::dump(security);
}
string dump(datetime::Date date) {
-22
View File
@@ -63,26 +63,6 @@ datetime::Date sec_expiration_cut_off() {
return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day());
}
void apply_compatibilit_mutations(api::Security &security) {
// Migrate from old tick stuff
if (security.has_glbx()) {
if (security.has_glbx()) {
if (security.contract_spec().main_fraction() == 0) {
security.mutable_contract_spec()->set_main_fraction(
security.glbx().do_not_use_main_fraction());
}
if (security.contract_spec().sub_fraction() == 0) {
security.mutable_contract_spec()->set_sub_fraction(
security.glbx().do_not_use_sub_fraction());
}
if (security.contract_spec().price_display_format() == 0) {
security.mutable_contract_spec()->set_price_display_format(
security.glbx().do_not_use_price_display_format());
}
}
}
}
size_t load_securities(
std::function<bool(const UserSymbol &symbo)> const &sym_filter,
const std::function<void(const api::Security &)> &callback) {
@@ -106,7 +86,6 @@ size_t load_securities(
if (sym_filter(us)) {
auto blob = select_stmt.getColumn("security_blob");
if (security.ParseFromArray(blob, blob.getBytes())) {
apply_compatibilit_mutations(security);
bts_assert(security_key == security.id());
count++;
callback(security);
@@ -148,7 +127,6 @@ std::shared_ptr<const Security> find_security(SecurityKey key) {
auto result = std::make_shared<Security>();
auto blob = select.getColumn(0);
if (result->ParseFromArray(blob, blob.getBytes())) {
apply_compatibilit_mutations(*result);
return result;
}
}
+77 -124
View File
@@ -2,22 +2,18 @@
#include "SecurityIDRecorder.h"
#include "spark/domain_model/Security.h"
#include "spark/domain_model/Security.pb.h"
#include "spark/md/invalid_security_key.h"
#include "spark/spark_data_directory.h"
#include <boost/pool/object_pool.hpp>
#include "spark/config/config.h"
#include "spark/md/common.h"
#include "spark/persistence/SqliteStore.h"
#include "spark/utils/Sqlite_utils.h"
#include "system/logging/logging.h"
#include "system/math/checked_arithmetic.h"
#include <boost/filesystem.hpp>
#include <tbb/concurrent_queue.h>
#include "spark/config/protos/market_data.pb.h"
#include "system/protobufs/config/system_config.pb.h"
#include <boost/pool/object_pool.hpp>
#include <sqlite/sqlite3.h>
#include <stdexcept>
#include <string>
#include <tbb/concurrent_queue.h>
using namespace std;
using namespace bts::utils;
@@ -29,10 +25,32 @@ namespace md {
using bts::api::Security;
using bts::api::SecurityType;
using bts::api::Venue;
using bts::system::protobufs::SystemConfig;
using bts::persistence::IStore;
using bts::persistence::SqliteStore;
uint64_t exchange_id(const api::Security &security);
static std::unique_ptr<IStore> get_default_store() {
return std::make_unique<SqliteStore>("key_ids");
}
static persistence::StoreRow get_row(
Venue venue,
SecurityType type,
const std::string &name,
const std::string &prefix = "") {
return persistence::StoreRow{
fmt::format(
"{}{}.{}", std::string{prefix}, Venue_Name(venue).substr(6),
SecurityType_Name(type).substr(13)),
name, ""};
}
static persistence::StoreRow get_row(
const api::Security &s, const std::string &prefix = "") {
return get_row(s.exchange(), bts::securities::security_type(s), prefix);
}
struct SecurityIdRequest {
Venue venue;
SecurityType type;
@@ -42,14 +60,16 @@ struct SecurityIdRequest {
};
struct SecurityIDRecorderState {
SecurityIDRecorderState(SecurityIDRecorderParams &p)
: pending_handle(), thread_id(), db(nullptr, sqlite3_close), timer(),
lookup_key(p.lookup_key), unresolved_legs() {}
SecurityIDRecorderState(
std::unique_ptr<IStore> store, KeyLookupOption lookup_key)
: store(store == nullptr ? get_default_store() : std::move(store)),
pending_handle(), thread_id(), timer(), lookup_key(lookup_key),
unresolved_legs() {}
std::unique_ptr<IStore> store{nullptr};
tbb::concurrent_queue<std::shared_ptr<SecurityIdRequest>> id_request_queue;
tbb::concurrent_queue<MDProducerHandle> pending_handle;
std::thread::id thread_id;
sqlite3_ptr db;
unique_ptr<DispatcherTimer> timer;
KeyLookupOption lookup_key;
@@ -85,47 +105,28 @@ struct SecurityIDRecorderState {
void check_access() { bts_assert(thread_id == std::this_thread::get_id()); }
void add_unresolved_spread(MDProducerHandle &handle);
security_key search(const std::string &descriptor);
security_key insert(const std::string &descriptor, SecurityType);
security_key search(const persistence::StoreRow &row);
security_key insert(const persistence::StoreRow &row);
};
SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params)
: _self(new SecurityIDRecorderState(params)) {
if (params.db) {
_self->db = std::move(params.db);
}
SecurityIDRecorder::SecurityIDRecorder(
std::unique_ptr<persistence::IStore> store,
KeyLookupOption lookup_key,
utils::Dispatcher *dispatcher)
: _self(std::make_unique<SecurityIDRecorderState>(
store == nullptr
? std::make_unique<persistence::SqliteStore>("key_ids")
: std::move(store),
lookup_key)) {
init();
if (params.dispatcher) {
_self->run(*params.dispatcher);
if (dispatcher != nullptr) {
_self->run(*dispatcher);
}
}
SecurityIDRecorder::~SecurityIDRecorder() {}
void SecurityIDRecorder::init() {
LOG(md, Info, "SecurityIDRecorder::init");
spark::SparkDataDirectory data_dir;
const auto db_dir = data_dir.database_dir("md");
const auto db_path = data_dir.database_file("md", "security_keys");
bts_assert(db_path.parent_path() == db_dir);
if (!boost::filesystem::exists(db_dir)) {
boost::filesystem::create_directories(db_dir);
}
if (!_self->db) {
_self->db = sqlite::open_db(db_path.c_str());
bts_assert(_self->db);
}
// Set a long busy timeout
sqlite3_busy_timeout(_self->db.get(), 9999);
sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0");
sqlite::exec_nofail(
_self->db, "CREATE TABLE IF NOT EXISTS store (descriptor STRING "
"NOT NULL PRIMARY KEY, type int);");
}
void SecurityIDRecorder::init() {}
void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}",
@@ -139,7 +140,7 @@ void SecurityIDRecorder::assign_key(
assign_key(
security.exchange(), bts::securities::security_type(security),
bts::securities::descriptor(security),
[&callback, msec = security](
[callback, msec = security](
api::Venue, api::SecurityType, const std::string &,
uint32_t id) mutable {
msec.set_id(id);
@@ -274,27 +275,25 @@ void SecurityIDRecorderState::work_db() {
// SPARK-1446 Need to try to add existing securities to the db in case we
// deleted them by hand
vector<std::shared_ptr<SecurityIdRequest>> existing_securities;
vector<std::shared_ptr<SecurityIdRequest>> defer;
vector<string> new_descriptors;
vector<persistence::StoreRow> new_rows;
size_t count = 0;
string descriptor;
while (id_request_queue.try_pop(temp)) {
descriptor = temp->name;
// If we're unable to create a descriptor try again later
if (descriptor == "") {
defer.push_back(std::move(temp));
continue;
security_key key = invalid_security_key;
auto row = get_row(temp->venue, temp->type, temp->name);
try {
key = search(row);
} catch (const std::overflow_error &ex) {
LOG(persist, Error,
"SecurityIDRecorderState::work_db. Row search threw ex: {}",
std::string{ex.what()});
}
security_key key = search(descriptor);
if (key != invalid_security_key) {
temp->callback(temp->venue, temp->type, temp->name, key);
existing_securities.push_back(std::move(temp));
} else {
// always push in parallel
new_entries.push_back(std::make_pair(key, std::move(temp)));
new_descriptors.push_back(descriptor);
new_rows.push_back(std::move(row));
++count;
// Limit the amount of time we spend writing to sqlite during startup
// to avoid blocking the heartbeat
@@ -304,22 +303,14 @@ void SecurityIDRecorderState::work_db() {
}
}
for (auto &d : defer) {
LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}",
d->name);
id_request_queue.push(std::move(d));
}
if (!new_entries.empty()) {
sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION");
store->insert_many_resolve_ids(new_rows);
for (size_t x = 0; x < new_entries.size(); ++x) {
auto &entry = new_entries[x];
entry.first = insert(new_descriptors[x], entry.second->type);
auto &row = new_rows[x];
entry.first = row.id;
bts_assert_d(entry.first != invalid_security_key);
}
sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION");
}
if (!new_entries.empty()) {
@@ -332,65 +323,27 @@ void SecurityIDRecorderState::work_db() {
void SecurityIDRecorder::bulk_insert(
const std::vector<const api::Security *> &data, const char *prefix) {
LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size());
if (data.size() == 0)
return;
sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
std::string descriptor;
{
auto insert_stmt = sqlite::prepare_query(
_self->db, "INSERT INTO security_keys (rowid, descriptor, type) "
"values (?, ?, ?)");
for (const api::Security *sec : data) {
descriptor = prefix + bts::securities::unique_descriptor(*sec);
auto type = securitytype_of(*sec);
sqlite3_bind_int(insert_stmt.get(), 1, sec->id());
sqlite3_bind_text(
insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(),
SQLITE_STATIC);
sqlite3_bind_int(insert_stmt.get(), 3, (int)type);
sqlite3_step(insert_stmt.get());
sqlite3_reset(insert_stmt.get());
}
std::vector<persistence::StoreRow> rows;
rows.reserve(data.size());
for (const auto *s : data) {
rows.push_back(get_row(*s, prefix));
}
sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
_self->store->insert_many_resolve_ids(rows);
}
security_key SecurityIDRecorderState::search(const std::string &descriptor) {
LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor);
auto select_stmt = sqlite::prepare_query(
db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
sqlite3_bind_text(
select_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
SQLITE_STATIC);
int result = sqlite3_step(select_stmt.get());
if (result == SQLITE_DONE || result == SQLITE_ROW) {
return sqlite3_column_int(select_stmt.get(), 0);
security_key SecurityIDRecorderState::search(const persistence::StoreRow &row) {
auto [exists, rowid, val] = store->retrieve_one(row.ns, row.key);
if (exists) {
if (!val.empty()) {
bts_assert(stol(val) == rowid);
}
return system::math::checked_ltou(rowid);
}
return invalid_security_key;
}
security_key SecurityIDRecorderState::insert(
const std::string &descriptor, SecurityType type) {
LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor);
auto insert_stmt = sqlite::prepare_query(
db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
sqlite3_bind_text(
insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
SQLITE_STATIC);
sqlite3_bind_int(insert_stmt.get(), 2, (int)type);
int result = sqlite3_step(insert_stmt.get());
if (result == SQLITE_DONE || result == SQLITE_ROW) {
return sqlite3_last_insert_rowid(db.get());
}
// Just in case our insert failed due to duplicate row
auto key = search(descriptor);
if (key == invalid_security_key) {
LOG(md, Error, "Failed to assign security key. SQLite return code: {}",
result);
}
return key;
security_key SecurityIDRecorderState::insert(const persistence::StoreRow &row) {
return store->insert(row.ns, row.key, row.value);
}
} // namespace md
+6 -16
View File
@@ -1,4 +1,5 @@
#pragma once
#include "spark/persistence/IStore.h"
#ifndef _WIN32
#include "MDProducerHandle.h"
#include "common.h"
@@ -22,26 +23,15 @@ namespace md {
typedef const std::function<SecurityKey(ExchangeKey)> KeyLookup;
typedef std::optional<KeyLookup> KeyLookupOption;
struct SecurityIDRecorderParams {
SecurityIDRecorderParams()
: dispatcher(), db(sqlite::null_db()), lookup_key() {}
SecurityIDRecorderParams(utils::Dispatcher &d)
: dispatcher(&d), db(sqlite::null_db()), lookup_key() {}
SecurityIDRecorderParams(utils::Dispatcher &d, KeyLookup &get_key)
: dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {}
SecurityIDRecorderParams(sqlite3_ptr &db)
: dispatcher(), db(std::move(db)), lookup_key() {}
SecurityIDRecorderParams(sqlite3_ptr &db, KeyLookup &get_key)
: dispatcher(), db(std::move(db)), lookup_key(get_key) {}
utils::Dispatcher *dispatcher;
sqlite3_ptr db;
KeyLookupOption lookup_key;
};
struct SecurityIDRecorderState;
class SecurityIDRecorder {
public:
SecurityIDRecorder(SecurityIDRecorderParams);
SecurityIDRecorder(
std::unique_ptr<persistence::IStore> store = nullptr,
KeyLookupOption lookup_key = std::nullopt,
utils::Dispatcher *dispatcher = nullptr);
~SecurityIDRecorder();
typedef std::function<void(const api::Security &)> Callback;
+2 -24
View File
@@ -40,9 +40,6 @@ class SymbolLoader {
SymbolLoader(const SymbolLoader &);
SymbolLoader &operator=(const SymbolLoader &);
static constexpr bool venue_has_synths(api::Venue venue);
static constexpr bool venue_has_synths(domain::protobufs::Venue venue);
static void add_symbols(
const google::protobuf::RepeatedPtrField<std::string> &symbols,
Venue exchange,
@@ -64,22 +61,6 @@ class SymbolLoader {
// Definition
//////////////////////////////////////////////////
constexpr bool SymbolLoader::venue_has_synths(api::Venue venue) {
switch (venue) {
case api::VENUE_SIM:
case api::VENUE_SIMICE:
case api::VENUE_ICE:
case api::VENUE_GLBX:
return true;
default:
return false;
}
}
constexpr bool SymbolLoader::venue_has_synths(domain::protobufs::Venue venue) {
return venue_has_synths(domain::DomainSecurityUtils::convert(venue));
}
inline void SymbolLoader::add_symbols(
const google::protobuf::RepeatedPtrField<std::string> &symbols,
Venue exchange,
@@ -105,9 +86,7 @@ inline std::set<UserSymbol> SymbolLoader::load(
const bts::domain::SecurityCriteria &include,
const bts::domain::SecurityCriteria &exclude) {
std::set<UserSymbol> result;
load(result, include);
if (exclude.is_defined()) {
}
load(result, include, exclude);
return result;
}
@@ -142,8 +121,7 @@ inline std::size_t SymbolLoader::load(
(exclude_none || !exclude.includes(venue, SEC_TYPE(SPREAD)))) {
add_symbols(vcfg.spread(), venue, SEC_TYPE(SPREAD), result);
}
if ((include_all || (venue_has_synths(venue) &&
include.includes(venue, SEC_TYPE(SYNTHETIC)))) &&
if ((include_all || (include.includes(venue, SEC_TYPE(SYNTHETIC)))) &&
(exclude_none || !exclude.includes(venue, SEC_TYPE(SYNTHETIC)))) {
add_symbols(vcfg.spot(), VENUE(BTS), SEC_TYPE(SYNTHETIC), result);
add_symbols(vcfg.future(), VENUE(BTS), SEC_TYPE(SYNTHETIC), result);
@@ -35,8 +35,9 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher
struct SyntheticSecurityService::Pimpl {
Pimpl(const MetadataStatefulPublisher &fn)
: recorder(utils::Dispatcher::background()), synthetics(), on_security(),
meta(), on_metadata(), metadata_publisher(fn) {}
: recorder(nullptr, std::nullopt, &utils::Dispatcher::background()),
synthetics(), on_security(), meta(), on_metadata(),
metadata_publisher(fn) {}
SecurityIDRecorder recorder;
std::unordered_map<SecurityKey, Security> synthetics;
utils::Event<Security> on_security;
+5 -19
View File
@@ -219,26 +219,12 @@ void convert_security(const FixedSecurity &source, Security *presult) {
get_contract_spec(source, *dest.mutable_contract_spec());
}
SecurityType securitytype_of_contractcase(Security::ContractCase cc) {
switch (cc) {
case Security::ContractCase::kFuture:
return SECURITYTYPE_FUTURE;
case Security::ContractCase::kOption:
return SECURITYTYPE_OPTION;
case Security::ContractCase::kSpot:
return api::SECURITYTYPE_SPOT;
case Security::ContractCase::kSpread:
return SECURITYTYPE_SPREAD;
case Security::ContractCase::kSynthetic:
return SECURITYTYPE_SYNTHETIC;
case Security::ContractCase::CONTRACT_NOT_SET:
bts_assert(cc != Security::ContractCase::CONTRACT_NOT_SET);
}
bts_assert(false);
}
SecurityType securitytype_of(const Security &s) {
return securitytype_of_contractcase(s.Contract_case());
auto type = bts::securities::security_type(s.Contract_case());
bts_assert(type != api::SecurityType::SECURITYTYPE_UNSET);
return type;
}
size_t num_legs(const Security &sec) {
if (sec.Contract_case() == api::Security::ContractCase::kSpread) {
return sec.spread().legs_size();
@@ -286,7 +272,7 @@ uint64_t exchange_id(const Security &security) {
void convert_security(
const Security &src, FixedSecurity &dest, bool set_security_key) {
dest.key = set_security_key ? src.id() : invalid_security_key;
dest.type = securitytype_of_contractcase(src.Contract_case());
dest.type = bts::securities::security_type(src.Contract_case());
dest.symbol(src.symbol());
dest.currency(src.currency());
dest.exchange = src.exchange();
@@ -215,7 +215,6 @@ void convert_security(
FixedSecurity &dest,
bool set_security_key = false);
SecurityType securitytype_of_contractcase(api::Security::ContractCase cc);
SecurityType securitytype_of(const api::Security &);
size_t num_legs(const api::Security &);
uint64_t exchange_id(const api::Security &security);
+14 -15
View File
@@ -8,6 +8,7 @@
#include "spark/md/PriceSize.h"
#include "spark/md/relay/query_server.h"
#include "spark/metrics/metrics.h"
#include "spark/persistence/SqliteStore.h"
#include "spark/utils/Dispatcher.h"
#include "spark/utils/Shuffle.h"
#include "spark/utils/bts_chrono.h"
@@ -159,12 +160,10 @@ class Co2Tests {
#ifndef _WIN32
void test_security_id_recorder(const std::string &db_name) {
// Verify callbacks happen after work complete
auto db1 = sqlite::open_db(db_name.c_str());
REQUIRE(db1);
auto db2 = sqlite::open_db(db_name.c_str());
REQUIRE(db2);
SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}};
SecurityIDRecorder idsvc{
std::make_unique<persistence::SqliteStore>(db_name)};
SecurityIDRecorder idsvc2{
std::make_unique<persistence::SqliteStore>(db_name)};
security_key result = invalid_security_key;
Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850);
Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20);
@@ -214,18 +213,18 @@ class Co2Tests {
std::make_unique<Co2TestInstrument>(s1, cold_header, hot_header);
// Add spread to SecurityIDRecorder and work it
auto db = sqlite::open_db(":memory:");
auto db = std::make_unique<persistence::SqliteStore>(":memory:");
REQUIRE(db);
SecurityIDRecorder idsvc{
{db, [&](ExchangeKey k) -> SecurityKey {
for (const auto &fut :
{f1handle->producer(), f2handle->producer()}) {
if (k.value() == fut.security().numeric_id)
return SecurityKey(fut.key());
}
return SecurityKey(invalid_security_key);
}}};
std::move(db), [&](ExchangeKey k) -> SecurityKey {
for (const auto &fut :
{f1handle->producer(), f2handle->producer()}) {
if (k.value() == fut.security().numeric_id)
return SecurityKey(fut.key());
}
return SecurityKey(invalid_security_key);
}};
auto workwork = [&]() {
idsvc.do_work();
@@ -7,6 +7,7 @@
#include "spark/spark_data_directory.h"
#include "spark/utils/bts_assert.h"
#include "system/logging/logging.h"
#include "system/math/checked_arithmetic.h"
#include <cstdint>
#include <memory>
@@ -66,24 +67,22 @@ std::optional<uint64_t> IDGenerator::get_id(
bool add_if_new) {
std::string ns = fmt::format(
"{}.{}", Venue_Name(venue).substr(6),
SecurityType_Name(type).substr(12));
SecurityType_Name(type).substr(14));
return get_id(ns, name, add_if_new);
}
std::optional<uint64_t> IDGenerator::get_id(
const std::string &ns, const std::string &key, bool add_if_new) {
std::lock_guard<std::mutex> myLock(_instance_mx);
auto [exists, val] = me->kv_store->retrieve_one(ns, key);
auto [exists, rowid, val] = me->kv_store->retrieve_one(ns, key);
if (exists) {
uint64_t rv = str_to_id(val);
LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rv);
return std::optional<uint64_t>(rv);
LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rowid);
return std::optional<uint64_t>(rowid);
} else {
if (add_if_new) {
auto all = me->kv_store->retrieve_all();
uint64_t new_id = get_next_id(all);
me->kv_store->insert(ns, key, std::to_string(new_id));
auto new_id = me->kv_store->insert(ns, key, "");
LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key,
new_id);
return std::optional<uint64_t>(new_id);
@@ -94,13 +93,6 @@ std::optional<uint64_t> IDGenerator::get_id(
}
}
std::uint64_t IDGenerator::get_or_add(
const std::string &ns, const std::string &key) {
auto id = get_id(ns, key, true);
bts_assert(id.has_value());
return *id;
}
std::uint64_t IDGenerator::get_or_add(
domain::protobufs::Venue venue,
domain::protobufs::SecurityType type,
+35 -3
View File
@@ -1,8 +1,12 @@
#pragma once
#include "spark/persistence/store_row.h"
#include <cstdint>
#include <map>
#include <set>
#include <string>
#include <tuple>
#include <vector>
namespace bts {
namespace persistence {
@@ -11,17 +15,45 @@ class IStore {
public:
virtual std::map<std::string, std::map<std::string, std::string>>
retrieve_all() = 0;
virtual std::pair<bool, std::string>
retrieve_one(std::string const &ns, std::string const &k) = 0;
/**
* Finds the row with the specified namespace and key.
*
* @return std::tuple<bool, std::int64, std::string> The first item indicates
* whether or not a row was found. For successful queries, the second and third
* items are the rowid and value, respectively.
*/
virtual std::tuple<bool, std::int64_t, std::string> retrieve_one(
std::string const &ns, std::string const &k) = 0;
virtual void
remove(std::string const &ns, std::set<std::string> const &keys) = 0;
virtual void remove_all(std::string const &ns) = 0;
virtual void insert(
virtual std::int64_t insert(
std::string const &ns, std::string const &k, std::string const &v) = 0;
void insert_many(const std::vector<StoreRow> &rows);
void insert_many_resolve_ids(std::vector<StoreRow> &rows);
virtual void update(
std::string const &ns, std::string const &k, std::string const &v) = 0;
virtual ~IStore();
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline void IStore::insert_many(const std::vector<StoreRow> &rows) {
for (const auto &row : rows) {
insert(row.ns, row.key, row.value);
}
}
inline void IStore::insert_many_resolve_ids(std::vector<StoreRow> &rows) {
for (auto &row : rows) {
row.id = insert(row.ns, row.key, row.value);
}
}
} // namespace persistence
} // namespace bts
+227 -124
View File
@@ -1,16 +1,10 @@
#include "SqliteStore.h"
#include "spark/config/config.h"
#include "spark/persistence/persistence_utils.h"
#include "spark/spark_data_directory.h"
#include "spark/utils/Sqlite_utils.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/bts_chrono.h"
#include "spark/utils/bts_no_copy.h"
#include "spark/utils/defer.h"
#include "spark/utils/file_utils.h"
#include "system/logging/logging.h"
#include "system/os.h"
#include "system/protobufs/config/system_config.pb.h"
#include <boost/filesystem.hpp>
@@ -36,82 +30,95 @@ static boost::filesystem::path get_db_path(const std::string &str) {
}
struct SqliteStore::PImpl {
std::string const db;
boost::filesystem::path db_path;
sqlite3_ptr conn{nullptr, sqlite3_close};
sqlite3_stmt_ptr insert{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr insert_many{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr insert_with_id{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr remove{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr remove_all{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr update{nullptr, sqlite3_finalize};
PImpl(const boost::filesystem::path &db_path)
: db(db_path.stem().string()), db_path(db_path),
conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize),
remove(nullptr, sqlite3_finalize),
remove_all(nullptr, sqlite3_finalize),
update(nullptr, sqlite3_finalize) {
: db(db_path.stem().string()), db_path(db_path) {
bts_assert(db.size());
}
PImpl(std::string const &store_root, std::string const &db_name)
: PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {}
std::string const db;
boost::filesystem::path db_path;
sqlite3_ptr conn;
sqlite3_stmt_ptr insert;
sqlite3_stmt_ptr remove;
sqlite3_stmt_ptr remove_all;
sqlite3_stmt_ptr update;
BTS_NO_COPY(PImpl);
explicit PImpl(sqlite3_ptr conn) : db(), db_path(), conn(std::move(conn)) {}
PImpl(const PImpl &) = delete;
PImpl &operator=(const PImpl &) = delete;
};
SqliteStore::SqliteStore(std::string const &db)
: me(std::make_unique<PImpl>(get_db_path(db))) {
LOG(persist, Info, "SqliteStore ctr. db: {}; db_path: {}", db, me->db_path);
LOG(persist, Info, "SqliteStore ctr. db: {}; db_path: {}", db,
me->db_path.string());
init_db();
}
SqliteStore::SqliteStore(sqlite3_ptr db)
: me(std::make_unique<PImpl>(std::move(db))) {
LOG(persist, Info, "SqliteStore ctr. db: <imported ptr>");
init_db();
}
void SqliteStore::init_db() {
auto path = me->db_path;
auto parent = path.parent_path();
const auto path = me->db_path;
if (!boost::filesystem::is_directory(parent))
bts_assert(boost::filesystem::create_directories(parent));
if (!boost::filesystem::exists(path)) {
LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
me->db, path.string());
me->conn = sqlite::open_db(path.c_str());
bts_assert_msg(me->conn, path.c_str());
const char *create_schema_sql =
"create table store ( ns text, key text, value text)";
sqlite::exec_nofail(me->conn, create_schema_sql);
const char *key_index_sql =
"create unique index key_index on store(ns,key)";
sqlite::exec_nofail(me->conn, key_index_sql);
const char *ns_index_sql = "create index ns_index on store(ns)";
sqlite::exec_nofail(me->conn, ns_index_sql);
} else {
me->conn = sqlite::open_db(path.c_str());
if (!path.empty()) {
if (!boost::filesystem::is_directory(path.parent_path())) {
bts_assert(boost::filesystem::create_directories(path.parent_path()));
}
if (!boost::filesystem::exists(path)) {
LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
me->db, path.string());
}
}
auto &conn = me->conn;
if (conn == nullptr) {
conn = sqlite::open_db(path.c_str());
bts_assert(conn != nullptr);
}
sqlite::exec_nofail(
conn,
"CREATE TABLE IF NOT EXISTS store (ns TEXT, key TEXT, value TEXT)");
sqlite::exec_nofail(
conn, "CREATE UNIQUE INDEX IF NOT EXISTS key_index on store(ns, key)");
sqlite::exec_nofail(
conn, "CREATE INDEX IF NOT EXISTS ns_index on store(ns)");
conn = sqlite::open_db(path.c_str());
// Asynchronous for speed
sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;");
sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL");
sqlite::exec_nofail(conn, "PRAGMA synchronous = 0;");
sqlite::exec_nofail(conn, "PRAGMA journal_mode=WAL");
// prepare insert statement
const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)";
me->insert = sqlite::prepare_query(me->conn, insert_sql);
me->insert = sqlite::prepare_query(conn, insert_sql);
// prepare insert_with_id statement
const char *insert_with_id_sql =
"insert into store (rowid, ns, key, value) values (?,?,?,?)";
me->insert_with_id = sqlite::prepare_query(conn, insert_with_id_sql);
// prepare update statement
const char *update_sql =
"update store set value = ? where ns = ? and key = ?";
me->update = sqlite::prepare_query(me->conn, update_sql);
me->update = sqlite::prepare_query(conn, update_sql);
// prepare remove statement
const char *remove_all_sql = "delete from store where ns = ?";
me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql);
me->remove_all = sqlite::prepare_query(conn, remove_all_sql);
// prepare remove statement
const char *remove_sql = "delete from store where ns = ? and key = ?";
me->remove = sqlite::prepare_query(me->conn, remove_sql);
me->remove = sqlite::prepare_query(conn, remove_sql);
}
SqliteStore::~SqliteStore() {}
@@ -140,25 +147,27 @@ SqliteStore::Data SqliteStore::retrieve_all() {
return result;
}
std::pair<bool, std::string> SqliteStore::retrieve_one(
std::tuple<bool, std::int64_t, std::string> SqliteStore::retrieve_one(
std::string const &ns, std::string const &k) {
auto query_str = fmt::format(
"select ns,key,value from store where ns='{}' and key='{}'", ns, k);
"select rowid,ns,key,value from store where ns='{}' and key='{}'", ns,
k);
sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str());
std::pair<bool, std::string> result{false, ""};
auto result = std::make_tuple<bool, std::int64_t, std::string>(false, 0, "");
auto pq = query.get();
auto sqlite_result = sqlite3_step(pq);
if (sqlite_result == SQLITE_ROW) {
std::int64_t rowid = sqlite3_column_int64(pq, 0);
std::string ns(
(char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0));
std::string key(
(char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1));
std::string key(
(char *)sqlite3_column_text(pq, 2), sqlite3_column_bytes(pq, 2));
std::string val(
(char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2));
result.first = true;
result.second = std::move(val);
(char *)sqlite3_column_blob(pq, 3), sqlite3_column_bytes(pq, 3));
std::get<0>(result) = true;
std::get<1>(result) = rowid;
std::get<2>(result) = std::move(val);
}
return result;
}
@@ -167,91 +176,185 @@ void SqliteStore::remove(
std::string const &ns, std::set<std::string> const &keys) {
bts_assert(ns.size());
for (auto &k : keys) {
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not delete row in store. Will retry once. error_code: {} "
"ns:{}, k: {}",
r, ns, k);
sleep_for(2s);
r = sqlite3_step(me->remove.get());
try {
for (auto &k : keys) {
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete row. Bailing out. error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
"Could not delete row in store. Will retry once. error_code: "
"{} "
"ns:{}, k: {}",
r, ns, k);
sleep_for(2s);
r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete row. Bailing out. "
"error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn);
}
SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn);
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::remove threw. what: {}",
std::string{ex.what()});
}
}
void SqliteStore::remove_all(std::string const &ns) {
bts_assert(ns.size());
SQLITE_CHK(
sqlite3_bind_text(
me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not delete all rows in store. Will retry once. error_code: {} "
"ns:{}",
r, ns);
sleep_for(2s);
r = sqlite3_step(me->remove_all.get());
try {
SQLITE_CHK(
sqlite3_bind_text(
me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete all rows. Bailing out. "
"error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
"Could not delete all rows in store. Will retry once. error_code: "
"{} "
"ns:{}",
r, ns);
sleep_for(2s);
r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete all rows. Bailing out. "
"error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn);
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::remove_all threw. what: {}",
std::string{ex.what()});
}
SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn);
}
void SqliteStore::insert(
std::int64_t SqliteStore::insert(
std::string const &ns, std::string const &k, std::string const &v) {
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not insert row in store. Will retry once. error_code: {} ns: "
"{}, k: {}, v: {}",
r, ns, k, v);
sleep_for(2s);
r = sqlite3_step(me->insert.get());
try {
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to insert row. Bailing out. error_code: {}",
r);
bts_assert(r == SQLITE_DONE);
"Could not insert row in store. Will retry once. error_code: {} "
"ns: "
"{}, k: {}, v: {}",
r, ns, k, v);
sleep_for(2s);
r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to insert row. Bailing out. error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
const auto rowid = sqlite3_last_insert_rowid(me->conn.get());
SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn);
return rowid;
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::insert threw. what: {}",
std::string{ex.what()});
}
SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn);
return 0;
}
void SqliteStore::insert_many(const std::vector<StoreRow> &rows) {
LOG(md, Info, "SqliteStore::insert_many size: {}", rows.size());
if (rows.empty()) {
return;
}
try {
sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION");
sqlite3 *conn_rawp = me->conn.get();
sqlite3_stmt *insert_rawp = me->insert.get();
sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get();
for (const auto &row : rows) {
int col_idx = 1;
sqlite3_stmt *stmt;
if (row.id > 0) {
stmt = insert_with_id_rawp;
sqlite3_bind_int(stmt, col_idx++, row.id);
} else {
stmt = insert_rawp;
}
sqlite3_bind_text(
stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC);
sqlite3_bind_text(
stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC);
int result = sqlite3_step(stmt);
if (result == SQLITE_DONE || result == SQLITE_ROW) {
sqlite3_last_insert_rowid(conn_rawp);
}
sqlite3_reset(stmt);
}
sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION");
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::insert_many threw. what: {}",
std::string{ex.what()});
}
}
void SqliteStore::insert_many_resolve_ids(std::vector<StoreRow> &rows) {
LOG(md, Info, "SqliteStore::insert_many_resolve_ids size: {}", rows.size());
if (rows.empty()) {
return;
}
sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION");
sqlite3 *conn_rawp = me->conn.get();
sqlite3_stmt *insert_rawp = me->insert.get();
sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get();
for (auto &row : rows) {
int col_idx = 1;
sqlite3_stmt *stmt;
if (row.id > 0) {
stmt = insert_with_id_rawp;
sqlite3_bind_int(stmt, col_idx++, row.id);
} else {
stmt = insert_rawp;
}
sqlite3_bind_text(
stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC);
sqlite3_bind_text(
stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC);
int result = sqlite3_step(stmt);
if (result == SQLITE_DONE || result == SQLITE_ROW) {
if (row.id > 0) {
bts_assert(sqlite3_last_insert_rowid(conn_rawp) == row.id);
} else {
row.id = sqlite3_last_insert_rowid(conn_rawp);
}
}
sqlite3_reset(stmt);
}
sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION");
}
void SqliteStore::update(
+11 -2
View File
@@ -1,7 +1,12 @@
#pragma once
#include "IStore.h"
#include "spark/persistence/store_row.h"
#include "spark/utils/Sqlite_utils.h"
#include <cstdint>
#include <memory>
#include <vector>
namespace bts {
namespace persistence {
@@ -9,22 +14,26 @@ namespace persistence {
class SqliteStore : public IStore {
public:
SqliteStore(std::string const &db);
explicit SqliteStore(sqlite3_ptr db);
~SqliteStore();
static SqliteStore from_path(const std::string &full_path);
typedef std::map<std::string, std::map<std::string, std::string>> Data;
Data retrieve_all() override;
std::pair<bool, std::string> retrieve_one(
std::tuple<bool, std::int64_t, std::string> retrieve_one(
std::string const &ns, std::string const &k) override;
void remove(
std::string const &ns, std::set<std::string> const &keys) override;
void remove_all(std::string const &ns) override;
void insert(
std::int64_t insert(
std::string const &ns,
std::string const &k,
std::string const &v) override;
void insert_many(const std::vector<StoreRow> &rows);
void insert_many_resolve_ids(std::vector<StoreRow> &rows);
void update(
std::string const &ns,
std::string const &k,
@@ -0,0 +1,20 @@
#pragma once
#include <cstdint>
#include <string>
namespace bts::persistence {
struct StoreRow {
std::string ns;
std::string key;
std::string value;
/**
* Represents the unique 'rowid' in the sqlite db. A value of 0 indicates that
* the rowid isn't known, and should either be generated by the db (for inserts),
* or resolved by finding the row with matching ns, key, value (for retrieves).
*/
std::int64_t id{0};
};
} // namespace bts::persistence
@@ -0,0 +1,37 @@
#pragma once
#include <cstdint>
#include <limits>
#include <stdexcept>
namespace bts::system::math {
static std::uint32_t checked_ltou(std::int64_t l);
static std::uint64_t checked_ltoul(std::int64_t l);
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline std::uint32_t checked_ltou(std::int64_t l) {
if (l < 0) {
throw std::overflow_error{
"Failed to convert int64 to uint32 (value < 0)"};
}
if (l >
static_cast<std::int64_t>(std::numeric_limits<std::uint32_t>::max())) {
throw std::overflow_error{
"Failed to convert int64 to uint32. value > max)"};
}
return static_cast<std::uint32_t>(l);
}
inline std::uint64_t checked_ltoul(std::int64_t l) {
if (l < 0) {
throw std::overflow_error{
"Failed to convert int64 to uint64 (value < 0)"};
}
return static_cast<std::uint64_t>(l);
}
} // namespace bts::system::math
+10
View File
@@ -50,7 +50,9 @@ class TimeSpan {
constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept;
std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept;
@@ -116,10 +118,18 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept {
return lhs.nanos() < rhs.nanos();
}
constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept {
return lhs.nanos() <= rhs.nanos();
}
constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept {
return lhs.nanos() > rhs.nanos();
}
constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept {
return lhs.nanos() >= rhs.nanos();
}
} // namespace bts::core
FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string());