267 lines
9.0 KiB
C++
267 lines
9.0 KiB
C++
#include "SqliteStore.h"
|
|
|
|
#include "lib/logging/logging.h"
|
|
#include "lib/os/os.h"
|
|
#include "lib/persistence/persistence_utils.h"
|
|
#include "lib/utils/Sqlite_utils.h"
|
|
#include "lib/utils/bts_assert.h"
|
|
#include "lib/utils/bts_chrono.h"
|
|
#include "lib/utils/bts_no_copy.h"
|
|
#include "lib/utils/defer.h"
|
|
#include "lib/utils/file_utils.h"
|
|
|
|
#include <boost/filesystem.hpp>
|
|
#include <sqlite/sqlite3.h>
|
|
|
|
namespace bts {
|
|
namespace persistence {
|
|
|
|
struct SqliteStore::PImpl {
|
|
PImpl(std::string const &store_root, std::string const &db_name)
|
|
: db(db_name), db_path(persistence::storage_location(db, store_root)),
|
|
conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize),
|
|
remove(nullptr, sqlite3_finalize),
|
|
remove_all(nullptr, sqlite3_finalize),
|
|
update(nullptr, sqlite3_finalize) {
|
|
bts_assert(db.size());
|
|
}
|
|
|
|
std::string const db;
|
|
boost::filesystem::path db_path;
|
|
sqlite3_ptr conn;
|
|
sqlite3_stmt_ptr insert;
|
|
sqlite3_stmt_ptr remove;
|
|
sqlite3_stmt_ptr remove_all;
|
|
sqlite3_stmt_ptr update;
|
|
BTS_NO_COPY(PImpl);
|
|
};
|
|
|
|
SqliteStore::SqliteStore(std::string const &db)
|
|
: me(new PImpl(os::bts_data_path(), db)) {
|
|
init_db();
|
|
}
|
|
|
|
SqliteStore::SqliteStore(std::string const &root_path, std::string const &db)
|
|
: me(new PImpl(root_path, db)) {
|
|
init_db();
|
|
}
|
|
|
|
void SqliteStore::init_db() {
|
|
auto path = me->db_path;
|
|
auto parent = path.parent_path();
|
|
|
|
if (!boost::filesystem::is_directory(parent))
|
|
bts_assert(boost::filesystem::create_directories(parent));
|
|
|
|
if (!boost::filesystem::exists(path)) {
|
|
LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
|
|
me->db, path.string());
|
|
|
|
me->conn = sqlite::open_db(path.c_str());
|
|
bts_assert_msg(me->conn, path.c_str());
|
|
|
|
const char *create_schema_sql =
|
|
"create table store ( ns text, key text, value text)";
|
|
sqlite::exec_nofail(me->conn, create_schema_sql);
|
|
|
|
const char *key_index_sql =
|
|
"create unique index key_index on store(ns,key)";
|
|
sqlite::exec_nofail(me->conn, key_index_sql);
|
|
|
|
const char *ns_index_sql = "create index ns_index on store(ns)";
|
|
sqlite::exec_nofail(me->conn, ns_index_sql);
|
|
} else {
|
|
me->conn = sqlite::open_db(path.c_str());
|
|
}
|
|
|
|
// Asynchronous for speed
|
|
sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;");
|
|
sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL");
|
|
|
|
// prepare insert statement
|
|
const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)";
|
|
me->insert = sqlite::prepare_query(me->conn, insert_sql);
|
|
|
|
// prepare update statement
|
|
const char *update_sql =
|
|
"update store set value = ? where ns = ? and key = ?";
|
|
me->update = sqlite::prepare_query(me->conn, update_sql);
|
|
|
|
// prepare remove statement
|
|
const char *remove_all_sql = "delete from store where ns = ?";
|
|
me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql);
|
|
|
|
// prepare remove statement
|
|
const char *remove_sql = "delete from store where ns = ? and key = ?";
|
|
me->remove = sqlite::prepare_query(me->conn, remove_sql);
|
|
}
|
|
|
|
SqliteStore::~SqliteStore() {}
|
|
|
|
SqliteStore::Data SqliteStore::retrieve_all() {
|
|
SqliteStore::Data result;
|
|
sqlite3_stmt_ptr query =
|
|
sqlite::prepare_query(me->conn, "select ns,key,value from store");
|
|
|
|
int sqlite_result;
|
|
do {
|
|
auto pq = query.get();
|
|
sqlite_result = sqlite3_step(pq);
|
|
if (sqlite_result == SQLITE_ROW) {
|
|
std::string ns((char *)sqlite3_column_text(pq, 0),
|
|
sqlite3_column_bytes(pq, 0));
|
|
std::string key((char *)sqlite3_column_text(pq, 1),
|
|
sqlite3_column_bytes(pq, 1));
|
|
std::string val((char *)sqlite3_column_blob(pq, 2),
|
|
sqlite3_column_bytes(pq, 2));
|
|
auto &r = result[ns][key];
|
|
r = std::move(val);
|
|
}
|
|
} while (sqlite_result == SQLITE_ROW);
|
|
|
|
return result;
|
|
}
|
|
|
|
std::pair<bool, std::string> SqliteStore::retrieve_one(std::string const &ns,
|
|
std::string const &k) {
|
|
auto query_str = fmt::format(
|
|
"select ns,key,value from store where ns='{}' and key='{}'", ns, k);
|
|
sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str());
|
|
|
|
std::pair<bool, std::string> result{false, ""};
|
|
|
|
auto pq = query.get();
|
|
auto sqlite_result = sqlite3_step(pq);
|
|
if (sqlite_result == SQLITE_ROW) {
|
|
std::string ns((char *)sqlite3_column_text(pq, 0),
|
|
sqlite3_column_bytes(pq, 0));
|
|
std::string key((char *)sqlite3_column_text(pq, 1),
|
|
sqlite3_column_bytes(pq, 1));
|
|
std::string val((char *)sqlite3_column_blob(pq, 2),
|
|
sqlite3_column_bytes(pq, 2));
|
|
result.first = true;
|
|
result.second = std::move(val);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
void SqliteStore::remove(std::string const &ns,
|
|
std::set<std::string> const &keys) {
|
|
bts_assert(ns.size());
|
|
|
|
for (auto &k : keys) {
|
|
SQLITE_CHK(sqlite3_bind_text(me->remove.get(), 1, ns.c_str(), ns.size(),
|
|
SQLITE_TRANSIENT),
|
|
me->conn);
|
|
SQLITE_CHK(sqlite3_bind_text(me->remove.get(), 2, k.c_str(), k.size(),
|
|
SQLITE_TRANSIENT),
|
|
me->conn);
|
|
auto r = sqlite3_step(me->remove.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Could not delete row in store. Will retry once. error_code: {} "
|
|
"ns:{}, k: {}",
|
|
r, ns, k);
|
|
sleep_for(2s);
|
|
r = sqlite3_step(me->remove.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Failed a second time to delete row. Bailing out. error_code: "
|
|
"{}",
|
|
r);
|
|
bts_assert(r == SQLITE_DONE);
|
|
}
|
|
}
|
|
SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn);
|
|
}
|
|
}
|
|
|
|
void SqliteStore::remove_all(std::string const &ns) {
|
|
bts_assert(ns.size());
|
|
SQLITE_CHK(sqlite3_bind_text(me->remove_all.get(), 1, ns.c_str(), ns.size(),
|
|
SQLITE_TRANSIENT),
|
|
me->conn);
|
|
auto r = sqlite3_step(me->remove_all.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Could not delete all rows in store. Will retry once. error_code: {} "
|
|
"ns:{}",
|
|
r, ns);
|
|
sleep_for(2s);
|
|
r = sqlite3_step(me->remove_all.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Failed a second time to delete all rows. Bailing out. error_code: "
|
|
"{}",
|
|
r);
|
|
bts_assert(r == SQLITE_DONE);
|
|
}
|
|
}
|
|
SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn);
|
|
}
|
|
|
|
void SqliteStore::insert(std::string const &ns, std::string const &k,
|
|
std::string const &v) {
|
|
SQLITE_CHK(sqlite3_bind_text(me->insert.get(), 1, ns.c_str(), ns.size(),
|
|
SQLITE_TRANSIENT),
|
|
me->conn);
|
|
SQLITE_CHK(sqlite3_bind_text(me->insert.get(), 2, k.c_str(), k.size(),
|
|
SQLITE_TRANSIENT),
|
|
me->conn);
|
|
SQLITE_CHK(sqlite3_bind_text(me->insert.get(), 3, v.c_str(), v.size(),
|
|
SQLITE_TRANSIENT),
|
|
me->conn);
|
|
auto r = sqlite3_step(me->insert.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Could not insert row in store. Will retry once. error_code: {} ns: "
|
|
"{}, k: {}, v: {}",
|
|
r, ns, k, v);
|
|
sleep_for(2s);
|
|
r = sqlite3_step(me->insert.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Failed a second time to insert row. Bailing out. error_code: {}",
|
|
r);
|
|
bts_assert(r == SQLITE_DONE);
|
|
}
|
|
}
|
|
SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn);
|
|
}
|
|
|
|
void SqliteStore::update(std::string const &ns, std::string const &k,
|
|
std::string const &v) {
|
|
SQLITE_CHK(sqlite3_bind_text(me->update.get(), 1, v.c_str(), v.size(),
|
|
SQLITE_TRANSIENT),
|
|
me->conn);
|
|
SQLITE_CHK(sqlite3_bind_text(me->update.get(), 2, ns.c_str(), ns.size(),
|
|
SQLITE_TRANSIENT),
|
|
me->conn);
|
|
SQLITE_CHK(sqlite3_bind_text(me->update.get(), 3, k.c_str(), k.size(),
|
|
SQLITE_TRANSIENT),
|
|
me->conn);
|
|
auto r = sqlite3_step(me->update.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Could not update row in store. Will retry once. error_code: {} ns: "
|
|
"{}, k: {}, v: {}",
|
|
r, ns, k, v);
|
|
sleep_for(2s);
|
|
r = sqlite3_step(me->update.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Failed a second time to update row. Bailing out. error_code: {}",
|
|
r);
|
|
bts_assert(r == SQLITE_DONE);
|
|
}
|
|
}
|
|
SQLITE_CHK(sqlite3_reset(me->update.get()), me->conn);
|
|
}
|
|
|
|
std::string SqliteStore::storage_location(std::string const &db) {
|
|
return persistence::storage_location(db);
|
|
}
|
|
|
|
} // namespace persistence
|
|
} // namespace bts
|