Files
spark/src/lib/persistence/SqliteStore.cpp
T

267 lines
9.0 KiB
C++

#include "SqliteStore.h"
#include "lib/logging/logging.h"
#include "lib/os/os.h"
#include "lib/persistence/persistence_utils.h"
#include "lib/utils/Sqlite_utils.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/bts_no_copy.h"
#include "lib/utils/defer.h"
#include "lib/utils/file_utils.h"
#include <boost/filesystem.hpp>
#include <sqlite/sqlite3.h>
namespace bts {
namespace persistence {
struct SqliteStore::PImpl {
PImpl(std::string const &store_root, std::string const &db_name)
: db(db_name), db_path(persistence::storage_location(db, store_root)),
conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize),
remove(nullptr, sqlite3_finalize),
remove_all(nullptr, sqlite3_finalize),
update(nullptr, sqlite3_finalize) {
bts_assert(db.size());
}
std::string const db;
boost::filesystem::path db_path;
sqlite3_ptr conn;
sqlite3_stmt_ptr insert;
sqlite3_stmt_ptr remove;
sqlite3_stmt_ptr remove_all;
sqlite3_stmt_ptr update;
BTS_NO_COPY(PImpl);
};
SqliteStore::SqliteStore(std::string const &db)
: me(new PImpl(os::bts_data_path(), db)) {
init_db();
}
SqliteStore::SqliteStore(std::string const &root_path, std::string const &db)
: me(new PImpl(root_path, db)) {
init_db();
}
void SqliteStore::init_db() {
auto path = me->db_path;
auto parent = path.parent_path();
if (!boost::filesystem::is_directory(parent))
bts_assert(boost::filesystem::create_directories(parent));
if (!boost::filesystem::exists(path)) {
LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
me->db, path.string());
me->conn = sqlite::open_db(path.c_str());
bts_assert_msg(me->conn, path.c_str());
const char *create_schema_sql =
"create table store ( ns text, key text, value text)";
sqlite::exec_nofail(me->conn, create_schema_sql);
const char *key_index_sql =
"create unique index key_index on store(ns,key)";
sqlite::exec_nofail(me->conn, key_index_sql);
const char *ns_index_sql = "create index ns_index on store(ns)";
sqlite::exec_nofail(me->conn, ns_index_sql);
} else {
me->conn = sqlite::open_db(path.c_str());
}
// Asynchronous for speed
sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;");
sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL");
// prepare insert statement
const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)";
me->insert = sqlite::prepare_query(me->conn, insert_sql);
// prepare update statement
const char *update_sql =
"update store set value = ? where ns = ? and key = ?";
me->update = sqlite::prepare_query(me->conn, update_sql);
// prepare remove statement
const char *remove_all_sql = "delete from store where ns = ?";
me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql);
// prepare remove statement
const char *remove_sql = "delete from store where ns = ? and key = ?";
me->remove = sqlite::prepare_query(me->conn, remove_sql);
}
SqliteStore::~SqliteStore() {}
SqliteStore::Data SqliteStore::retrieve_all() {
SqliteStore::Data result;
sqlite3_stmt_ptr query =
sqlite::prepare_query(me->conn, "select ns,key,value from store");
int sqlite_result;
do {
auto pq = query.get();
sqlite_result = sqlite3_step(pq);
if (sqlite_result == SQLITE_ROW) {
std::string ns((char *)sqlite3_column_text(pq, 0),
sqlite3_column_bytes(pq, 0));
std::string key((char *)sqlite3_column_text(pq, 1),
sqlite3_column_bytes(pq, 1));
std::string val((char *)sqlite3_column_blob(pq, 2),
sqlite3_column_bytes(pq, 2));
auto &r = result[ns][key];
r = std::move(val);
}
} while (sqlite_result == SQLITE_ROW);
return result;
}
std::pair<bool, std::string> SqliteStore::retrieve_one(std::string const &ns,
std::string const &k) {
auto query_str = fmt::format(
"select ns,key,value from store where ns='{}' and key='{}'", ns, k);
sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str());
std::pair<bool, std::string> result{false, ""};
auto pq = query.get();
auto sqlite_result = sqlite3_step(pq);
if (sqlite_result == SQLITE_ROW) {
std::string ns((char *)sqlite3_column_text(pq, 0),
sqlite3_column_bytes(pq, 0));
std::string key((char *)sqlite3_column_text(pq, 1),
sqlite3_column_bytes(pq, 1));
std::string val((char *)sqlite3_column_blob(pq, 2),
sqlite3_column_bytes(pq, 2));
result.first = true;
result.second = std::move(val);
}
return result;
}
void SqliteStore::remove(std::string const &ns,
std::set<std::string> const &keys) {
bts_assert(ns.size());
for (auto &k : keys) {
SQLITE_CHK(sqlite3_bind_text(me->remove.get(), 1, ns.c_str(), ns.size(),
SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(sqlite3_bind_text(me->remove.get(), 2, k.c_str(), k.size(),
SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not delete row in store. Will retry once. error_code: {} "
"ns:{}, k: {}",
r, ns, k);
sleep_for(2s);
r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete row. Bailing out. error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn);
}
}
void SqliteStore::remove_all(std::string const &ns) {
bts_assert(ns.size());
SQLITE_CHK(sqlite3_bind_text(me->remove_all.get(), 1, ns.c_str(), ns.size(),
SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not delete all rows in store. Will retry once. error_code: {} "
"ns:{}",
r, ns);
sleep_for(2s);
r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete all rows. Bailing out. error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn);
}
void SqliteStore::insert(std::string const &ns, std::string const &k,
std::string const &v) {
SQLITE_CHK(sqlite3_bind_text(me->insert.get(), 1, ns.c_str(), ns.size(),
SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(sqlite3_bind_text(me->insert.get(), 2, k.c_str(), k.size(),
SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(sqlite3_bind_text(me->insert.get(), 3, v.c_str(), v.size(),
SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not insert row in store. Will retry once. error_code: {} ns: "
"{}, k: {}, v: {}",
r, ns, k, v);
sleep_for(2s);
r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to insert row. Bailing out. error_code: {}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn);
}
void SqliteStore::update(std::string const &ns, std::string const &k,
std::string const &v) {
SQLITE_CHK(sqlite3_bind_text(me->update.get(), 1, v.c_str(), v.size(),
SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(sqlite3_bind_text(me->update.get(), 2, ns.c_str(), ns.size(),
SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(sqlite3_bind_text(me->update.get(), 3, k.c_str(), k.size(),
SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->update.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not update row in store. Will retry once. error_code: {} ns: "
"{}, k: {}, v: {}",
r, ns, k, v);
sleep_for(2s);
r = sqlite3_step(me->update.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to update row. Bailing out. error_code: {}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->update.get()), me->conn);
}
std::string SqliteStore::storage_location(std::string const &db) {
return persistence::storage_location(db);
}
} // namespace persistence
} // namespace bts