#include "SqliteStore.h" #include "lib/logging/logging.h" #include "lib/os/os.h" #include "lib/persistence/persistence_utils.h" #include "lib/utils/Sqlite_utils.h" #include "lib/utils/bts_assert.h" #include "lib/utils/bts_chrono.h" #include "lib/utils/bts_no_copy.h" #include "lib/utils/defer.h" #include "lib/utils/file_utils.h" #include #include namespace bts { namespace persistence { struct SqliteStore::PImpl { PImpl(std::string const &store_root, std::string const &db_name) : db(db_name), db_path(persistence::storage_location(db, store_root)), conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize), remove(nullptr, sqlite3_finalize), remove_all(nullptr, sqlite3_finalize), update(nullptr, sqlite3_finalize) { bts_assert(db.size()); } std::string const db; boost::filesystem::path db_path; sqlite3_ptr conn; sqlite3_stmt_ptr insert; sqlite3_stmt_ptr remove; sqlite3_stmt_ptr remove_all; sqlite3_stmt_ptr update; BTS_NO_COPY(PImpl); }; SqliteStore::SqliteStore(std::string const &db) : me(new PImpl(os::bts_data_path(), db)) { init_db(); } SqliteStore::SqliteStore(std::string const &root_path, std::string const &db) : me(new PImpl(root_path, db)) { init_db(); } void SqliteStore::init_db() { auto path = me->db_path; auto parent = path.parent_path(); if (!boost::filesystem::is_directory(parent)) bts_assert(boost::filesystem::create_directories(parent)); if (!boost::filesystem::exists(path)) { LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", me->db, path.string()); me->conn = sqlite::open_db(path.c_str()); bts_assert_msg(me->conn, path.c_str()); const char *create_schema_sql = "create table store ( ns text, key text, value text)"; sqlite::exec_nofail(me->conn, create_schema_sql); const char *key_index_sql = "create unique index key_index on store(ns,key)"; sqlite::exec_nofail(me->conn, key_index_sql); const char *ns_index_sql = "create index ns_index on store(ns)"; sqlite::exec_nofail(me->conn, ns_index_sql); } else { me->conn = sqlite::open_db(path.c_str()); } // Asynchronous for speed sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;"); sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL"); // prepare insert statement const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)"; me->insert = sqlite::prepare_query(me->conn, insert_sql); // prepare update statement const char *update_sql = "update store set value = ? where ns = ? and key = ?"; me->update = sqlite::prepare_query(me->conn, update_sql); // prepare remove statement const char *remove_all_sql = "delete from store where ns = ?"; me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql); // prepare remove statement const char *remove_sql = "delete from store where ns = ? and key = ?"; me->remove = sqlite::prepare_query(me->conn, remove_sql); } SqliteStore::~SqliteStore() {} SqliteStore::Data SqliteStore::retrieve_all() { SqliteStore::Data result; sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, "select ns,key,value from store"); int sqlite_result; do { auto pq = query.get(); sqlite_result = sqlite3_step(pq); if (sqlite_result == SQLITE_ROW) { std::string ns((char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0)); std::string key((char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1)); std::string val((char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2)); auto &r = result[ns][key]; r = std::move(val); } } while (sqlite_result == SQLITE_ROW); return result; } std::pair SqliteStore::retrieve_one(std::string const &ns, std::string const &k) { auto query_str = fmt::format( "select ns,key,value from store where ns='{}' and key='{}'", ns, k); sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str()); std::pair result{false, ""}; auto pq = query.get(); auto sqlite_result = sqlite3_step(pq); if (sqlite_result == SQLITE_ROW) { std::string ns((char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0)); std::string key((char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1)); std::string val((char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2)); result.first = true; result.second = std::move(val); } return result; } void SqliteStore::remove(std::string const &ns, std::set const &keys) { bts_assert(ns.size()); for (auto &k : keys) { SQLITE_CHK(sqlite3_bind_text(me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), me->conn); SQLITE_CHK(sqlite3_bind_text(me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), me->conn); auto r = sqlite3_step(me->remove.get()); if (r != SQLITE_DONE) { LOG(persist, Error, "Could not delete row in store. Will retry once. error_code: {} " "ns:{}, k: {}", r, ns, k); sleep_for(2s); r = sqlite3_step(me->remove.get()); if (r != SQLITE_DONE) { LOG(persist, Error, "Failed a second time to delete row. Bailing out. error_code: " "{}", r); bts_assert(r == SQLITE_DONE); } } SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); } } void SqliteStore::remove_all(std::string const &ns) { bts_assert(ns.size()); SQLITE_CHK(sqlite3_bind_text(me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), me->conn); auto r = sqlite3_step(me->remove_all.get()); if (r != SQLITE_DONE) { LOG(persist, Error, "Could not delete all rows in store. Will retry once. error_code: {} " "ns:{}", r, ns); sleep_for(2s); r = sqlite3_step(me->remove_all.get()); if (r != SQLITE_DONE) { LOG(persist, Error, "Failed a second time to delete all rows. Bailing out. error_code: " "{}", r); bts_assert(r == SQLITE_DONE); } } SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); } void SqliteStore::insert(std::string const &ns, std::string const &k, std::string const &v) { SQLITE_CHK(sqlite3_bind_text(me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT), me->conn); SQLITE_CHK(sqlite3_bind_text(me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT), me->conn); SQLITE_CHK(sqlite3_bind_text(me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT), me->conn); auto r = sqlite3_step(me->insert.get()); if (r != SQLITE_DONE) { LOG(persist, Error, "Could not insert row in store. Will retry once. error_code: {} ns: " "{}, k: {}, v: {}", r, ns, k, v); sleep_for(2s); r = sqlite3_step(me->insert.get()); if (r != SQLITE_DONE) { LOG(persist, Error, "Failed a second time to insert row. Bailing out. error_code: {}", r); bts_assert(r == SQLITE_DONE); } } SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); } void SqliteStore::update(std::string const &ns, std::string const &k, std::string const &v) { SQLITE_CHK(sqlite3_bind_text(me->update.get(), 1, v.c_str(), v.size(), SQLITE_TRANSIENT), me->conn); SQLITE_CHK(sqlite3_bind_text(me->update.get(), 2, ns.c_str(), ns.size(), SQLITE_TRANSIENT), me->conn); SQLITE_CHK(sqlite3_bind_text(me->update.get(), 3, k.c_str(), k.size(), SQLITE_TRANSIENT), me->conn); auto r = sqlite3_step(me->update.get()); if (r != SQLITE_DONE) { LOG(persist, Error, "Could not update row in store. Will retry once. error_code: {} ns: " "{}, k: {}, v: {}", r, ns, k, v); sleep_for(2s); r = sqlite3_step(me->update.get()); if (r != SQLITE_DONE) { LOG(persist, Error, "Failed a second time to update row. Bailing out. error_code: {}", r); bts_assert(r == SQLITE_DONE); } } SQLITE_CHK(sqlite3_reset(me->update.get()), me->conn); } std::string SqliteStore::storage_location(std::string const &db) { return persistence::storage_location(db); } } // namespace persistence } // namespace bts