Refactored persistence.

This commit is contained in:
Pedro Pinto committed 2017-08-14 19:44:35 +00:00
1 parent c5aeb46c67
commit b2d0bd43eb
8 files changed
+304 -155

No files matched your search

+1
View File
@@ -67,6 +67,7 @@ set (Sparkd_Sources
persistence/PersistenceServiceServer.cpp
persistence/tests/PersistenceServiceTests.cpp
persistence/IStore.cpp
persistence/SqliteStore.cpp
# testing utilities
testing/catch_test_runner.cpp
+8
View File
@@ -0,0 +1,8 @@
#include "IStore.h"
namespace bts {
namespace persistence {
IStore::~IStore () {}
}}
+20
View File
@@ -0,0 +1,20 @@
#pragma once
#include <map>
#include <set>
#include <string>
namespace bts {
namespace persistence {
class IStore {
public:
virtual std::map<std::string,std::map<std::string,std::string>> retrieve_all() = 0;
virtual void remove (std::string const &ns, std::set<std::string> const &keys) = 0;
virtual void remove_all (std::string const &ns) = 0;
virtual void insert (std::string const &ns, std::string const&k, std::string const&v) = 0;
virtual void update (std::string const &ns, std::string const&k, std::string const&v) = 0;
virtual ~IStore();
};
}}
@@ -5,25 +5,12 @@
#include "lib/utils/bts_no_copy.h"
#include "lib/os/os.h"
#include "lib/logging/logging.h"
#include "lib/utils/Sqlite_utils.h"
#include <boost/filesystem.hpp>
#include <sqlite/sqlite3.h>
#include "SqliteStore.h"
namespace bts {
namespace api {
namespace persistence {
static char *sqlite_error = 0;
boost::filesystem::path db_path (std::string const &db)
{
std::string base_path = os::bts_data_path ();
boost::filesystem::path result = utils::normalize_data_path (base_path);
result /= "PersistenceService";
result /= db + ".sqlite";
return result;
}
using namespace api::persistence;
NamespaceSubArgs to_sub_arg(std::string const &ns)
{
@@ -33,6 +20,8 @@ NamespaceSubArgs to_sub_arg(std::string const &ns)
}
struct PersistenceServiceServer::NSRecord {
NSRecord() : publish(), data() {}
Publisher publish;
std::map<std::string,std::string> data;
@@ -58,14 +47,10 @@ struct PersistenceServiceServer::NSRecord {
struct PersistenceServiceServer::PImpl
{
PImpl(std::string const &db) : db(db), conn(), insert(), remove(),
remove_all(), update(),data() {}
PImpl(std::string const &db) : db(db),store (db) {}
std::string const db;
sqlite3 *conn;
sqlite3_stmt* insert;
sqlite3_stmt* remove;
sqlite3_stmt* remove_all;
sqlite3_stmt* update;
SqliteStore store;
std::map<std::string,NSRecord> data;
BTS_NO_COPY (PImpl);
};
@@ -75,22 +60,23 @@ PersistenceServiceServer::PersistenceServiceServer(bts::api::ServiceContext & co
me (new PImpl(db))
{
bts_assert(db.size());
init_db();
init_streams();
// load data
auto data = me->store.retrieve_all();
for (auto const &kvp : data) {
auto const &ns = kvp.first;
auto &ns_rec = me->data[ns];
ns_rec.publish = make_ns_publisher(to_sub_arg(ns));
for (auto const &kvp : kvp.second) {
auto const &k = kvp.first;
auto const &v = kvp.second;
ns_rec.data[k] = std::move(v);
}
ns_rec.publish_all();
}
}
PersistenceServiceServer::~PersistenceServiceServer()
{
if (me->insert)
SQLITE_CHK (sqlite3_finalize (me->insert));
if (me->update)
SQLITE_CHK (sqlite3_finalize (me->update));
if (me->remove)
SQLITE_CHK (sqlite3_finalize (me->remove));
if (me->conn != 0)
SQLITE_CHK (sqlite3_close (me->conn));
}
PersistenceServiceServer::~PersistenceServiceServer() {}
void PersistenceServiceServer::retrieve(const RetrieveRequest& request,
std::function<void(const RetrieveResponse&)> reply)
@@ -107,7 +93,8 @@ void PersistenceServiceServer::retrieve(const RetrieveRequest& request,
auto it = me->data.find(request.ns());
if (it == me->data.end()) {
LOG(Persistence,Error, "Client tried to retrieve specific rows from unknown namespace {}/{}", me->db,request.ns());
LOG(Persistence,Error, "Client tried to retrieve specific rows from unknown namespace {}/{}",
me->db,request.ns());
response.set_result ( RetrieveResponse::NO_SUCH_NS);
reply(response);
return;
@@ -219,13 +206,7 @@ void PersistenceServiceServer::remove(const RemoveRequest& request, std::functio
return;
}
}
for (auto &k : request.key()) {
SQLITE_CHK (sqlite3_bind_text(me->remove, 1,request.ns().c_str(),-1,SQLITE_TRANSIENT));
SQLITE_CHK (sqlite3_bind_text(me->remove, 2,k.c_str(),-1,SQLITE_TRANSIENT));
bts_assert (sqlite3_step(me->remove) == SQLITE_DONE);
SQLITE_CHK (sqlite3_reset (me->remove));
}
me->store.remove(request.ns(),keys);
ns_rec.publish(nullptr,nullptr,&keys);
response.set_result ( RemoveResponse::OK);
reply(response);
@@ -260,11 +241,8 @@ void PersistenceServiceServer::remove_all(const RemoveAllRequest& request, std::
std::set<std::string> keys;
for (auto &kvp : ns_rec.data)
keys.insert(kvp.first);
SQLITE_CHK (sqlite3_bind_text(me->remove_all, 1,request.ns().c_str(),-1,SQLITE_TRANSIENT));
bts_assert (sqlite3_step(me->remove_all) == SQLITE_DONE);
SQLITE_CHK (sqlite3_reset (me->remove_all));
me->store.remove_all(request.ns());
ns_rec.publish(nullptr,nullptr,&keys);
response.set_result ( RemoveResponse::OK);
reply(response);
@@ -315,112 +293,23 @@ void PersistenceServiceServer::store(const StoreRequest& request, std::function<
std::vector<Record> insert;
std::vector<Record> update;
for (auto &record : request.record()) {
auto it = rec->data.find(record.key());
if (it != rec->data.end()) {
// update
SQLITE_CHK (sqlite3_bind_text(me->update, 1,request.ns().c_str(),-1,SQLITE_TRANSIENT));
SQLITE_CHK (sqlite3_bind_text(me->update, 2,record.key().c_str(),-1,SQLITE_TRANSIENT));
SQLITE_CHK (sqlite3_bind_text(me->update, 3,record.value().c_str(),-1,SQLITE_TRANSIENT));
bts_assert (sqlite3_step(me->update) == SQLITE_DONE);
SQLITE_CHK (sqlite3_reset (me->update));
update.push_back(record);
// update
update.push_back(record);
me->store.update(request.ns(),record.key(),record.value());
}
else {
else {
// insert
SQLITE_CHK (sqlite3_bind_text(me->insert, 1,request.ns().c_str(),-1,SQLITE_TRANSIENT));
SQLITE_CHK (sqlite3_bind_text(me->insert, 2,record.key().c_str(),-1,SQLITE_TRANSIENT));
SQLITE_CHK (sqlite3_bind_text(me->insert, 3,record.value().c_str(),-1,SQLITE_TRANSIENT));
bts_assert (sqlite3_step(me->insert) == SQLITE_DONE);
SQLITE_CHK (sqlite3_reset (me->insert));
insert.push_back(record);
insert.push_back(record);
me->store.insert(request.ns(),record.key(),record.value());
}
}
rec->publish(&insert,&update,nullptr);
response.set_result ( StoreResponse::OK);
reply(response);
LOG(Persistence,Diag, "Successfully stored {} rows in {}/{}", request.record().size(),me->db,request.ns());
}
int PersistenceServiceServer::load_record (void *ptr, int n_cols, char **values, char**)
{
PersistenceServiceServer* persister = (PersistenceServiceServer*) ptr;
bts_assert (n_cols == 3);
std::string ns = values[0];
std::string key = values[1];
std::string val = values[2];
auto &ns_rec = persister->me->data[ns];
ns_rec.publish = persister->make_ns_publisher(to_sub_arg(ns));
ns_rec.data[key] = std::move(val);
return 0;
}
void PersistenceServiceServer::init_db ()
{
auto path = db_path (me->db);
auto parent = path.parent_path();
if (!boost::filesystem::is_directory(parent)) {
LOG (Persistence,Warn, "{} database does not exist yet. Creating it at {}", path.string());
bts_assert(boost::filesystem::create_directories(parent));
SQLITE_CHK (sqlite3_open (path.c_str (), &(me->conn)));
const char* create_schema_sql=
"create table store ( ns text, key text, value text)";
SQLITE_CHK (sqlite3_exec (me->conn,create_schema_sql,0,0,&sqlite_error));
const char* key_index_sql =
"create unique index key_index on store(ns,key)";
SQLITE_CHK (sqlite3_exec (me->conn,key_index_sql,0,0,&sqlite_error));
const char* ns_index_sql =
"create index ns_index on store(ns)";
SQLITE_CHK (sqlite3_exec (me->conn,ns_index_sql,0,0,&sqlite_error));
}
else
SQLITE_CHK (sqlite3_open (path.c_str (), &(me->conn)));
// Asynchronous for speed
const char* set_async_sql = "PRAGMA synchronous = 0;";
SQLITE_CHK (sqlite3_exec (me->conn,set_async_sql,0,0,&sqlite_error));
// prepare insert statement
const char *insert_sql =
"insert into store (ns, key, text) values(?,?,?)";
SQLITE_CHK (sqlite3_prepare_v2 (me->conn, insert_sql,-1,&(me->insert),0));
// prepare update statement
const char *update_sql =
"update store set value = ? where ns = ? and key = ?";
SQLITE_CHK (sqlite3_prepare_v2 (me->conn, update_sql,-1,&(me->update),0));
// prepare remove statement
const char *remove_all_sql =
"delete store where ns = ?";
SQLITE_CHK (sqlite3_prepare_v2 (me->conn, remove_all_sql,-1,&(me->remove),0));
// prepare remove statement
const char *remove_sql =
"delete store where ns = ? and key = ?";
SQLITE_CHK (sqlite3_prepare_v2 (me->conn, remove_sql,-1,&(me->remove),0));
// load db in memory
SQLITE_CHK (sqlite3_exec (me->conn,
"select ns,key,text from store",
load_record,
this,
&sqlite_error));
}
void PersistenceServiceServer::init_streams()
{
for (auto &kvp : me->data)
kvp.second.publish_all();
}
}}}
}}
@@ -4,30 +4,31 @@
#include "lib/api/services/PersistenceService.server.pb.h"
namespace bts {
namespace api {
namespace persistence {
class PersistenceServiceServer : public PersistenceServiceBase
class PersistenceServiceServer : public api::persistence::PersistenceServiceBase
{
public:
PersistenceServiceServer (bts::api::ServiceContext& context,std::string const &db);
~PersistenceServiceServer();
void retrieve(const RetrieveRequest& request, std::function<void(const RetrieveResponse&)> reply)override;
void retrieve_all(const RetrieveAllRequest& request, std::function<void(const RetrieveResponse&)> reply)override;
void retrieve(const api::persistence::RetrieveRequest& request,
std::function<void(const api::persistence::RetrieveResponse&)> reply)override;
void retrieve_all(const api::persistence::RetrieveAllRequest& request,
std::function<void(const api::persistence::RetrieveResponse&)> reply)override;
void remove(const RemoveRequest& request, std::function<void(const RemoveResponse&)> reply)override;
void remove_all(const RemoveAllRequest& request, std::function<void(const RemoveResponse&)> reply)override;
void remove(const api::persistence::RemoveRequest& request,
std::function<void(const api::persistence::RemoveResponse&)> reply)override;
void remove_all(const api::persistence::RemoveAllRequest& request,
std::function<void(const api::persistence::RemoveResponse&)> reply)override;
void store(const StoreRequest& request, std::function<void(const StoreResponse&)> reply)override;
void store(const api::persistence::StoreRequest& request,
std::function<void(const api::persistence::StoreResponse&)> reply)override;
private:
typedef PersistenceServiceServer::NsPublisher Publisher;
struct PImpl;
struct NSRecord;
std::unique_ptr<PImpl> me;
void init_db();
void init_streams();
static int load_record (void *ptr, int n_cols, char **values, char**columns);
};
}}}
}}
@@ -0,0 +1,170 @@
#include "SqliteStore.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/file_utils.h"
#include "lib/utils/bts_no_copy.h"
#include "lib/os/os.h"
#include "lib/logging/logging.h"
#include "lib/utils/Sqlite_utils.h"
#include <boost/filesystem.hpp>
#include <sqlite/sqlite3.h>
namespace bts {
namespace persistence {
static char *sqlite_error = 0;
boost::filesystem::path db_location (std::string const &db)
{
std::string base_path = os::bts_data_path ();
boost::filesystem::path result = utils::normalize_data_path (base_path);
result /= "PersistenceService";
result /= db + ".sqlite";
return result;
}
struct SqliteStore::PImpl
{
PImpl(std::string const &db) : db(db), conn(), insert(), remove(),
remove_all(), update()
{}
std::string const db;
sqlite3 *conn;
sqlite3_stmt* insert;
sqlite3_stmt* remove;
sqlite3_stmt* remove_all;
sqlite3_stmt* update;
BTS_NO_COPY (PImpl);
};
SqliteStore::SqliteStore (std::string const &db) : me (new PImpl(db)) {
bts_assert(db.size());
auto path = db_location (me->db);
auto parent = path.parent_path();
if (!boost::filesystem::is_directory(parent)) {
LOG (Persistence,Warn, "{} database does not exist yet. Creating it at {}", path.string());
bts_assert(boost::filesystem::create_directories(parent));
SQLITE_CHK (sqlite3_open (path.c_str (), &(me->conn)));
const char* create_schema_sql=
"create table store ( ns text, key text, value text)";
SQLITE_CHK (sqlite3_exec (me->conn,create_schema_sql,0,0,&sqlite_error));
const char* key_index_sql =
"create unique index key_index on store(ns,key)";
SQLITE_CHK (sqlite3_exec (me->conn,key_index_sql,0,0,&sqlite_error));
const char* ns_index_sql =
"create index ns_index on store(ns)";
SQLITE_CHK (sqlite3_exec (me->conn,ns_index_sql,0,0,&sqlite_error));
}
else
SQLITE_CHK (sqlite3_open (path.c_str (), &(me->conn)));
// Asynchronous for speed
const char* set_async_sql = "PRAGMA synchronous = 0;";
SQLITE_CHK (sqlite3_exec (me->conn,set_async_sql,0,0,&sqlite_error));
// prepare insert statement
const char *insert_sql =
"insert into store (ns, key, text) values(?,?,?)";
SQLITE_CHK (sqlite3_prepare_v2 (me->conn, insert_sql,-1,&(me->insert),0));
// prepare update statement
const char *update_sql =
"update store set value = ? where ns = ? and key = ?";
SQLITE_CHK (sqlite3_prepare_v2 (me->conn, update_sql,-1,&(me->update),0));
// prepare remove statement
const char *remove_all_sql =
"delete store where ns = ?";
SQLITE_CHK (sqlite3_prepare_v2 (me->conn, remove_all_sql,-1,&(me->remove),0));
// prepare remove statement
const char *remove_sql =
"delete store where ns = ? and key = ?";
SQLITE_CHK (sqlite3_prepare_v2 (me->conn, remove_sql,-1,&(me->remove),0));
}
SqliteStore::~SqliteStore ()
{
if (me->insert)
SQLITE_CHK (sqlite3_finalize (me->insert));
if (me->update)
SQLITE_CHK (sqlite3_finalize (me->update));
if (me->remove)
SQLITE_CHK (sqlite3_finalize (me->remove));
if (me->remove_all)
SQLITE_CHK (sqlite3_finalize (me->remove_all));
if (me->conn != 0)
SQLITE_CHK (sqlite3_close (me->conn));
}
SqliteStore::Data SqliteStore::retrieve_all()
{
SqliteStore::Data result;
SQLITE_CHK (sqlite3_exec (me->conn,
"select ns,key,text from store",
load_record,
&result,
&sqlite_error));
return result;
}
int SqliteStore::load_record (void *ptr, int n_cols, char **values, char**)
{
SqliteStore::Data& data = *(SqliteStore::Data*) ptr;
bts_assert (n_cols == 3);
std::string ns = values[0];
std::string key = values[1];
std::string val = values[2];
auto &r = data[ns][key];
r = std::move(val);
return 0;
}
void SqliteStore::remove (std::string const &ns, std::set<std::string> const &keys)
{
bts_assert(ns.size());
for (auto &k : keys) {
SQLITE_CHK (sqlite3_bind_text(me->remove, 1,ns.c_str(),-1,SQLITE_TRANSIENT));
SQLITE_CHK (sqlite3_bind_text(me->remove, 2,k.c_str(),-1,SQLITE_TRANSIENT));
bts_assert (sqlite3_step(me->remove) == SQLITE_DONE);
SQLITE_CHK (sqlite3_reset (me->remove));
}
}
void SqliteStore::remove_all (std::string const &ns) {
bts_assert(ns.size());
SQLITE_CHK (sqlite3_bind_text(me->remove_all, 1,ns.c_str(),-1,SQLITE_TRANSIENT));
bts_assert (sqlite3_step(me->remove_all) == SQLITE_DONE);
SQLITE_CHK (sqlite3_reset (me->remove_all));
}
void SqliteStore::insert (std::string const &ns, std::string const&k,std::string const&v)
{
SQLITE_CHK (sqlite3_bind_text(me->insert, 1,ns.c_str(),-1,SQLITE_TRANSIENT));
SQLITE_CHK (sqlite3_bind_text(me->insert, 2,k.c_str(),-1,SQLITE_TRANSIENT));
SQLITE_CHK (sqlite3_bind_text(me->insert, 3,v.c_str(),-1,SQLITE_TRANSIENT));
bts_assert (sqlite3_step(me->insert) == SQLITE_DONE);
SQLITE_CHK (sqlite3_reset (me->insert));
}
void SqliteStore::update (std::string const &ns, std::string const&k,std::string const&v)
{
SQLITE_CHK (sqlite3_bind_text(me->update, 1,ns.c_str(),-1,SQLITE_TRANSIENT));
SQLITE_CHK (sqlite3_bind_text(me->update, 2,k.c_str(),-1,SQLITE_TRANSIENT));
SQLITE_CHK (sqlite3_bind_text(me->update, 3,v.c_str(),-1,SQLITE_TRANSIENT));
bts_assert (sqlite3_step(me->update) == SQLITE_DONE);
SQLITE_CHK (sqlite3_reset (me->update));
}
}}
@@ -0,0 +1,27 @@
#pragma once
#include <memory>
#include "IStore.h"
namespace bts {
namespace persistence {
class SqliteStore : public IStore
{
public:
SqliteStore(std::string const&db);
~SqliteStore();
typedef std::map<std::string,std::map<std::string,std::string>> Data ;
Data retrieve_all() override ;
void remove (std::string const &ns, std::set<std::string> const &keys) override ;
void remove_all (std::string const &ns) override ;
void insert (std::string const &ns, std::string const&k, std::string const&v) override ;
void update (std::string const &ns, std::string const&k, std::string const&v) override ;
private:
struct PImpl;
std::unique_ptr<PImpl> me;
static int load_record (void *ptr, int n_cols, char **values, char**);
};
}}
@@ -0,0 +1,33 @@
#include "catch/catch.hpp"
#include "server/sparkd/persistence/PersistenceServiceServer.h"
#include "lib/api/public/ServiceContext.h"
#include "lib/utils/Uuid.h"
#include "lib/api/services/PersistenceService.client.pb.h"
namespace bts {
namespace persistence {
using namespace api::persistence;
TEST_CASE ("PersistenceService Basic Tests", "[persistence]")
{
api::ServiceContext ctx;
utils::Uuid id;
PersistenceServiceServer server(ctx,std::string("test_db") + id.to_text_string());
SECTION("smoke_test")
{
auto cli = ctx.locate<PersistenceService>("test_db");
RetrieveAllRequest req;
cli.retrieve_all(req,[](api::Result<RetrieveResponse> r) {
REQUIRE(r.result()->result() == RetrieveResponse::EMPTY_NS);
});
}
}
}}