diff --git a/src/server/sparkd/CMakeLists.txt b/src/server/sparkd/CMakeLists.txt index 8e1805235..d0f3224b0 100644 --- a/src/server/sparkd/CMakeLists.txt +++ b/src/server/sparkd/CMakeLists.txt @@ -67,6 +67,7 @@ set (Sparkd_Sources persistence/PersistenceServiceServer.cpp persistence/tests/PersistenceServiceTests.cpp persistence/IStore.cpp + persistence/SqliteStore.cpp # testing utilities testing/catch_test_runner.cpp diff --git a/src/server/sparkd/persistence/IStore.cpp b/src/server/sparkd/persistence/IStore.cpp new file mode 100644 index 000000000..34d652c26 --- /dev/null +++ b/src/server/sparkd/persistence/IStore.cpp @@ -0,0 +1,8 @@ +#include "IStore.h" + +namespace bts { +namespace persistence { + +IStore::~IStore () {} + +}} diff --git a/src/server/sparkd/persistence/IStore.h b/src/server/sparkd/persistence/IStore.h new file mode 100644 index 000000000..6a2968eac --- /dev/null +++ b/src/server/sparkd/persistence/IStore.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include +#include + +namespace bts { +namespace persistence { + +class IStore { +public: + virtual std::map> retrieve_all() = 0; + virtual void remove (std::string const &ns, std::set const &keys) = 0; + virtual void remove_all (std::string const &ns) = 0; + virtual void insert (std::string const &ns, std::string const&k, std::string const&v) = 0; + virtual void update (std::string const &ns, std::string const&k, std::string const&v) = 0; + virtual ~IStore(); +}; + +}} diff --git a/src/server/sparkd/persistence/PersistenceServiceServer.cpp b/src/server/sparkd/persistence/PersistenceServiceServer.cpp index b7c3cc6db..9fcb166ec 100644 --- a/src/server/sparkd/persistence/PersistenceServiceServer.cpp +++ b/src/server/sparkd/persistence/PersistenceServiceServer.cpp @@ -5,25 +5,12 @@ #include "lib/utils/bts_no_copy.h" #include "lib/os/os.h" #include "lib/logging/logging.h" -#include "lib/utils/Sqlite_utils.h" - -#include -#include +#include "SqliteStore.h" namespace bts { -namespace api { namespace persistence { -static char *sqlite_error = 0; - -boost::filesystem::path db_path (std::string const &db) -{ - std::string base_path = os::bts_data_path (); - boost::filesystem::path result = utils::normalize_data_path (base_path); - result /= "PersistenceService"; - result /= db + ".sqlite"; - return result; -} +using namespace api::persistence; NamespaceSubArgs to_sub_arg(std::string const &ns) { @@ -33,6 +20,8 @@ NamespaceSubArgs to_sub_arg(std::string const &ns) } struct PersistenceServiceServer::NSRecord { + NSRecord() : publish(), data() {} + Publisher publish; std::map data; @@ -58,14 +47,10 @@ struct PersistenceServiceServer::NSRecord { struct PersistenceServiceServer::PImpl { - PImpl(std::string const &db) : db(db), conn(), insert(), remove(), - remove_all(), update(),data() {} + PImpl(std::string const &db) : db(db),store (db) {} + std::string const db; - sqlite3 *conn; - sqlite3_stmt* insert; - sqlite3_stmt* remove; - sqlite3_stmt* remove_all; - sqlite3_stmt* update; + SqliteStore store; std::map data; BTS_NO_COPY (PImpl); }; @@ -75,22 +60,23 @@ PersistenceServiceServer::PersistenceServiceServer(bts::api::ServiceContext & co me (new PImpl(db)) { bts_assert(db.size()); - init_db(); - init_streams(); + + // load data + auto data = me->store.retrieve_all(); + for (auto const &kvp : data) { + auto const &ns = kvp.first; + auto &ns_rec = me->data[ns]; + ns_rec.publish = make_ns_publisher(to_sub_arg(ns)); + for (auto const &kvp : kvp.second) { + auto const &k = kvp.first; + auto const &v = kvp.second; + ns_rec.data[k] = std::move(v); + } + ns_rec.publish_all(); + } } -PersistenceServiceServer::~PersistenceServiceServer() -{ - if (me->insert) - SQLITE_CHK (sqlite3_finalize (me->insert)); - if (me->update) - SQLITE_CHK (sqlite3_finalize (me->update)); - if (me->remove) - SQLITE_CHK (sqlite3_finalize (me->remove)); - - if (me->conn != 0) - SQLITE_CHK (sqlite3_close (me->conn)); -} +PersistenceServiceServer::~PersistenceServiceServer() {} void PersistenceServiceServer::retrieve(const RetrieveRequest& request, std::function reply) @@ -107,7 +93,8 @@ void PersistenceServiceServer::retrieve(const RetrieveRequest& request, auto it = me->data.find(request.ns()); if (it == me->data.end()) { - LOG(Persistence,Error, "Client tried to retrieve specific rows from unknown namespace {}/{}", me->db,request.ns()); + LOG(Persistence,Error, "Client tried to retrieve specific rows from unknown namespace {}/{}", + me->db,request.ns()); response.set_result ( RetrieveResponse::NO_SUCH_NS); reply(response); return; @@ -219,13 +206,7 @@ void PersistenceServiceServer::remove(const RemoveRequest& request, std::functio return; } } - - for (auto &k : request.key()) { - SQLITE_CHK (sqlite3_bind_text(me->remove, 1,request.ns().c_str(),-1,SQLITE_TRANSIENT)); - SQLITE_CHK (sqlite3_bind_text(me->remove, 2,k.c_str(),-1,SQLITE_TRANSIENT)); - bts_assert (sqlite3_step(me->remove) == SQLITE_DONE); - SQLITE_CHK (sqlite3_reset (me->remove)); - } + me->store.remove(request.ns(),keys); ns_rec.publish(nullptr,nullptr,&keys); response.set_result ( RemoveResponse::OK); reply(response); @@ -260,11 +241,8 @@ void PersistenceServiceServer::remove_all(const RemoveAllRequest& request, std:: std::set keys; for (auto &kvp : ns_rec.data) keys.insert(kvp.first); - - SQLITE_CHK (sqlite3_bind_text(me->remove_all, 1,request.ns().c_str(),-1,SQLITE_TRANSIENT)); - bts_assert (sqlite3_step(me->remove_all) == SQLITE_DONE); - SQLITE_CHK (sqlite3_reset (me->remove_all)); + me->store.remove_all(request.ns()); ns_rec.publish(nullptr,nullptr,&keys); response.set_result ( RemoveResponse::OK); reply(response); @@ -315,112 +293,23 @@ void PersistenceServiceServer::store(const StoreRequest& request, std::function< std::vector insert; std::vector update; for (auto &record : request.record()) { - auto it = rec->data.find(record.key()); if (it != rec->data.end()) { - // update - SQLITE_CHK (sqlite3_bind_text(me->update, 1,request.ns().c_str(),-1,SQLITE_TRANSIENT)); - SQLITE_CHK (sqlite3_bind_text(me->update, 2,record.key().c_str(),-1,SQLITE_TRANSIENT)); - SQLITE_CHK (sqlite3_bind_text(me->update, 3,record.value().c_str(),-1,SQLITE_TRANSIENT)); - bts_assert (sqlite3_step(me->update) == SQLITE_DONE); - SQLITE_CHK (sqlite3_reset (me->update)); - update.push_back(record); + // update + update.push_back(record); + me->store.update(request.ns(),record.key(),record.value()); } - else { + else { // insert - SQLITE_CHK (sqlite3_bind_text(me->insert, 1,request.ns().c_str(),-1,SQLITE_TRANSIENT)); - SQLITE_CHK (sqlite3_bind_text(me->insert, 2,record.key().c_str(),-1,SQLITE_TRANSIENT)); - SQLITE_CHK (sqlite3_bind_text(me->insert, 3,record.value().c_str(),-1,SQLITE_TRANSIENT)); - bts_assert (sqlite3_step(me->insert) == SQLITE_DONE); - SQLITE_CHK (sqlite3_reset (me->insert)); - insert.push_back(record); + insert.push_back(record); + me->store.insert(request.ns(),record.key(),record.value()); } } rec->publish(&insert,&update,nullptr); response.set_result ( StoreResponse::OK); reply(response); - LOG(Persistence,Diag, "Successfully stored {} rows in {}/{}", request.record().size(),me->db,request.ns()); } - - -int PersistenceServiceServer::load_record (void *ptr, int n_cols, char **values, char**) -{ - PersistenceServiceServer* persister = (PersistenceServiceServer*) ptr; - bts_assert (n_cols == 3); - std::string ns = values[0]; - std::string key = values[1]; - std::string val = values[2]; - auto &ns_rec = persister->me->data[ns]; - ns_rec.publish = persister->make_ns_publisher(to_sub_arg(ns)); - ns_rec.data[key] = std::move(val); - return 0; -} - -void PersistenceServiceServer::init_db () -{ - auto path = db_path (me->db); - auto parent = path.parent_path(); - - if (!boost::filesystem::is_directory(parent)) { - LOG (Persistence,Warn, "{} database does not exist yet. Creating it at {}", path.string()); - bts_assert(boost::filesystem::create_directories(parent)); - - SQLITE_CHK (sqlite3_open (path.c_str (), &(me->conn))); - - const char* create_schema_sql= - "create table store ( ns text, key text, value text)"; - SQLITE_CHK (sqlite3_exec (me->conn,create_schema_sql,0,0,&sqlite_error)); - - const char* key_index_sql = - "create unique index key_index on store(ns,key)"; - SQLITE_CHK (sqlite3_exec (me->conn,key_index_sql,0,0,&sqlite_error)); - - const char* ns_index_sql = - "create index ns_index on store(ns)"; - SQLITE_CHK (sqlite3_exec (me->conn,ns_index_sql,0,0,&sqlite_error)); - } - else - SQLITE_CHK (sqlite3_open (path.c_str (), &(me->conn))); - - // Asynchronous for speed - const char* set_async_sql = "PRAGMA synchronous = 0;"; - SQLITE_CHK (sqlite3_exec (me->conn,set_async_sql,0,0,&sqlite_error)); - - // prepare insert statement - const char *insert_sql = - "insert into store (ns, key, text) values(?,?,?)"; - SQLITE_CHK (sqlite3_prepare_v2 (me->conn, insert_sql,-1,&(me->insert),0)); - - // prepare update statement - const char *update_sql = - "update store set value = ? where ns = ? and key = ?"; - SQLITE_CHK (sqlite3_prepare_v2 (me->conn, update_sql,-1,&(me->update),0)); - - // prepare remove statement - const char *remove_all_sql = - "delete store where ns = ?"; - SQLITE_CHK (sqlite3_prepare_v2 (me->conn, remove_all_sql,-1,&(me->remove),0)); - - // prepare remove statement - const char *remove_sql = - "delete store where ns = ? and key = ?"; - SQLITE_CHK (sqlite3_prepare_v2 (me->conn, remove_sql,-1,&(me->remove),0)); - - // load db in memory - SQLITE_CHK (sqlite3_exec (me->conn, - "select ns,key,text from store", - load_record, - this, - &sqlite_error)); -} - -void PersistenceServiceServer::init_streams() -{ - for (auto &kvp : me->data) - kvp.second.publish_all(); -} - -}}} +}} diff --git a/src/server/sparkd/persistence/PersistenceServiceServer.h b/src/server/sparkd/persistence/PersistenceServiceServer.h index 436dad972..e804ec3a4 100644 --- a/src/server/sparkd/persistence/PersistenceServiceServer.h +++ b/src/server/sparkd/persistence/PersistenceServiceServer.h @@ -4,30 +4,31 @@ #include "lib/api/services/PersistenceService.server.pb.h" namespace bts { -namespace api { namespace persistence { -class PersistenceServiceServer : public PersistenceServiceBase +class PersistenceServiceServer : public api::persistence::PersistenceServiceBase { public: PersistenceServiceServer (bts::api::ServiceContext& context,std::string const &db); ~PersistenceServiceServer(); - void retrieve(const RetrieveRequest& request, std::function reply)override; - void retrieve_all(const RetrieveAllRequest& request, std::function reply)override; + void retrieve(const api::persistence::RetrieveRequest& request, + std::function reply)override; + void retrieve_all(const api::persistence::RetrieveAllRequest& request, + std::function reply)override; - void remove(const RemoveRequest& request, std::function reply)override; - void remove_all(const RemoveAllRequest& request, std::function reply)override; + void remove(const api::persistence::RemoveRequest& request, + std::function reply)override; + void remove_all(const api::persistence::RemoveAllRequest& request, + std::function reply)override; - void store(const StoreRequest& request, std::function reply)override; + void store(const api::persistence::StoreRequest& request, + std::function reply)override; private: typedef PersistenceServiceServer::NsPublisher Publisher; struct PImpl; struct NSRecord; std::unique_ptr me; - void init_db(); - void init_streams(); - static int load_record (void *ptr, int n_cols, char **values, char**columns); }; -}}} +}} diff --git a/src/server/sparkd/persistence/SqliteStore.cpp b/src/server/sparkd/persistence/SqliteStore.cpp new file mode 100644 index 000000000..4551f45d6 --- /dev/null +++ b/src/server/sparkd/persistence/SqliteStore.cpp @@ -0,0 +1,170 @@ +#include "SqliteStore.h" + +#include "lib/utils/bts_assert.h" +#include "lib/utils/file_utils.h" +#include "lib/utils/bts_no_copy.h" +#include "lib/os/os.h" +#include "lib/logging/logging.h" +#include "lib/utils/Sqlite_utils.h" + +#include +#include + +namespace bts { +namespace persistence { + +static char *sqlite_error = 0; + +boost::filesystem::path db_location (std::string const &db) +{ + std::string base_path = os::bts_data_path (); + boost::filesystem::path result = utils::normalize_data_path (base_path); + result /= "PersistenceService"; + result /= db + ".sqlite"; + return result; +} + +struct SqliteStore::PImpl +{ + PImpl(std::string const &db) : db(db), conn(), insert(), remove(), + remove_all(), update() + {} + + std::string const db; + sqlite3 *conn; + sqlite3_stmt* insert; + sqlite3_stmt* remove; + sqlite3_stmt* remove_all; + sqlite3_stmt* update; + BTS_NO_COPY (PImpl); +}; + +SqliteStore::SqliteStore (std::string const &db) : me (new PImpl(db)) { + bts_assert(db.size()); + + auto path = db_location (me->db); + auto parent = path.parent_path(); + + if (!boost::filesystem::is_directory(parent)) { + LOG (Persistence,Warn, "{} database does not exist yet. Creating it at {}", path.string()); + bts_assert(boost::filesystem::create_directories(parent)); + + SQLITE_CHK (sqlite3_open (path.c_str (), &(me->conn))); + + const char* create_schema_sql= + "create table store ( ns text, key text, value text)"; + SQLITE_CHK (sqlite3_exec (me->conn,create_schema_sql,0,0,&sqlite_error)); + + const char* key_index_sql = + "create unique index key_index on store(ns,key)"; + SQLITE_CHK (sqlite3_exec (me->conn,key_index_sql,0,0,&sqlite_error)); + + const char* ns_index_sql = + "create index ns_index on store(ns)"; + SQLITE_CHK (sqlite3_exec (me->conn,ns_index_sql,0,0,&sqlite_error)); + } + else + SQLITE_CHK (sqlite3_open (path.c_str (), &(me->conn))); + + // Asynchronous for speed + const char* set_async_sql = "PRAGMA synchronous = 0;"; + SQLITE_CHK (sqlite3_exec (me->conn,set_async_sql,0,0,&sqlite_error)); + + // prepare insert statement + const char *insert_sql = + "insert into store (ns, key, text) values(?,?,?)"; + SQLITE_CHK (sqlite3_prepare_v2 (me->conn, insert_sql,-1,&(me->insert),0)); + + // prepare update statement + const char *update_sql = + "update store set value = ? where ns = ? and key = ?"; + SQLITE_CHK (sqlite3_prepare_v2 (me->conn, update_sql,-1,&(me->update),0)); + + // prepare remove statement + const char *remove_all_sql = + "delete store where ns = ?"; + SQLITE_CHK (sqlite3_prepare_v2 (me->conn, remove_all_sql,-1,&(me->remove),0)); + + // prepare remove statement + const char *remove_sql = + "delete store where ns = ? and key = ?"; + SQLITE_CHK (sqlite3_prepare_v2 (me->conn, remove_sql,-1,&(me->remove),0)); +} + + +SqliteStore::~SqliteStore () +{ + if (me->insert) + SQLITE_CHK (sqlite3_finalize (me->insert)); + if (me->update) + SQLITE_CHK (sqlite3_finalize (me->update)); + if (me->remove) + SQLITE_CHK (sqlite3_finalize (me->remove)); + if (me->remove_all) + SQLITE_CHK (sqlite3_finalize (me->remove_all)); + if (me->conn != 0) + SQLITE_CHK (sqlite3_close (me->conn)); +} + +SqliteStore::Data SqliteStore::retrieve_all() +{ + SqliteStore::Data result; + + SQLITE_CHK (sqlite3_exec (me->conn, + "select ns,key,text from store", + load_record, + &result, + &sqlite_error)); + return result; +} + +int SqliteStore::load_record (void *ptr, int n_cols, char **values, char**) +{ + SqliteStore::Data& data = *(SqliteStore::Data*) ptr; + bts_assert (n_cols == 3); + std::string ns = values[0]; + std::string key = values[1]; + std::string val = values[2]; + auto &r = data[ns][key]; + r = std::move(val); + return 0; +} + +void SqliteStore::remove (std::string const &ns, std::set const &keys) +{ + bts_assert(ns.size()); + + for (auto &k : keys) { + SQLITE_CHK (sqlite3_bind_text(me->remove, 1,ns.c_str(),-1,SQLITE_TRANSIENT)); + SQLITE_CHK (sqlite3_bind_text(me->remove, 2,k.c_str(),-1,SQLITE_TRANSIENT)); + bts_assert (sqlite3_step(me->remove) == SQLITE_DONE); + SQLITE_CHK (sqlite3_reset (me->remove)); + } +} + +void SqliteStore::remove_all (std::string const &ns) { + bts_assert(ns.size()); + SQLITE_CHK (sqlite3_bind_text(me->remove_all, 1,ns.c_str(),-1,SQLITE_TRANSIENT)); + bts_assert (sqlite3_step(me->remove_all) == SQLITE_DONE); + SQLITE_CHK (sqlite3_reset (me->remove_all)); +} + +void SqliteStore::insert (std::string const &ns, std::string const&k,std::string const&v) +{ + SQLITE_CHK (sqlite3_bind_text(me->insert, 1,ns.c_str(),-1,SQLITE_TRANSIENT)); + SQLITE_CHK (sqlite3_bind_text(me->insert, 2,k.c_str(),-1,SQLITE_TRANSIENT)); + SQLITE_CHK (sqlite3_bind_text(me->insert, 3,v.c_str(),-1,SQLITE_TRANSIENT)); + bts_assert (sqlite3_step(me->insert) == SQLITE_DONE); + SQLITE_CHK (sqlite3_reset (me->insert)); +} + +void SqliteStore::update (std::string const &ns, std::string const&k,std::string const&v) +{ + SQLITE_CHK (sqlite3_bind_text(me->update, 1,ns.c_str(),-1,SQLITE_TRANSIENT)); + SQLITE_CHK (sqlite3_bind_text(me->update, 2,k.c_str(),-1,SQLITE_TRANSIENT)); + SQLITE_CHK (sqlite3_bind_text(me->update, 3,v.c_str(),-1,SQLITE_TRANSIENT)); + bts_assert (sqlite3_step(me->update) == SQLITE_DONE); + SQLITE_CHK (sqlite3_reset (me->update)); +} + +}} diff --git a/src/server/sparkd/persistence/SqliteStore.h b/src/server/sparkd/persistence/SqliteStore.h new file mode 100644 index 000000000..1f3dbbff6 --- /dev/null +++ b/src/server/sparkd/persistence/SqliteStore.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include "IStore.h" + +namespace bts { +namespace persistence { + +class SqliteStore : public IStore +{ +public: + SqliteStore(std::string const&db); + ~SqliteStore(); + + typedef std::map> Data ; + Data retrieve_all() override ; + void remove (std::string const &ns, std::set const &keys) override ; + void remove_all (std::string const &ns) override ; + void insert (std::string const &ns, std::string const&k, std::string const&v) override ; + void update (std::string const &ns, std::string const&k, std::string const&v) override ; +private: + struct PImpl; + std::unique_ptr me; + static int load_record (void *ptr, int n_cols, char **values, char**); +}; + +}} diff --git a/src/server/sparkd/persistence/tests/PersistenceServiceTests.cpp b/src/server/sparkd/persistence/tests/PersistenceServiceTests.cpp new file mode 100644 index 000000000..8ff5cdf5e --- /dev/null +++ b/src/server/sparkd/persistence/tests/PersistenceServiceTests.cpp @@ -0,0 +1,33 @@ +#include "catch/catch.hpp" + +#include "server/sparkd/persistence/PersistenceServiceServer.h" +#include "lib/api/public/ServiceContext.h" +#include "lib/utils/Uuid.h" + +#include "lib/api/services/PersistenceService.client.pb.h" + +namespace bts { +namespace persistence { + +using namespace api::persistence; + +TEST_CASE ("PersistenceService Basic Tests", "[persistence]") +{ + api::ServiceContext ctx; + utils::Uuid id; + PersistenceServiceServer server(ctx,std::string("test_db") + id.to_text_string()); + + SECTION("smoke_test") + { + auto cli = ctx.locate("test_db"); + + RetrieveAllRequest req; + cli.retrieve_all(req,[](api::Result r) { + REQUIRE(r.result()->result() == RetrieveResponse::EMPTY_NS); + }); + } +} + +}} + +