#include "PersistenceServiceServer.h" #include "SqliteStore.h" #include "lib/logging/logging.h" #include "lib/os/os.h" #include "lib/utils/bts_assert.h" #include "lib/utils/bts_no_copy.h" #include "lib/utils/file_utils.h" namespace bts { namespace persistence { using namespace api::persistence; struct PersistenceServiceServer::NSRecord { NSRecord() : publish(), data() {} Publisher publish; std::map data; std::vector to_vec() { std::vector result; result.reserve(data.size()); for (auto const &kvp : data) { result.emplace_back(); auto &r = result.back(); r.set_key(kvp.first); r.set_value(kvp.second); } return result; } void publish_all() { auto v = to_vec(); publish(&v, nullptr, nullptr); } }; struct PersistenceServiceServer::PImpl { PImpl(std::string const &db) : db(db), store(db), data() {} std::string const db; SqliteStore store; std::map data; BTS_NO_COPY(PImpl); }; PersistenceServiceServer::PersistenceServiceServer( bts::api::ServiceContext &context, std::string const &db) : PersistenceServiceBase(context, db), me(new PImpl(db)) { LOG(persist, Warn, "Starting persistence service for {}", db); bts_assert(db.size()); // load data auto data = me->store.retrieve_all(); for (auto const &kvp : data) { auto const &ns = kvp.first; auto &ns_rec = me->data[ns]; ns_rec.publish = make_ns_publisher(ns); for (auto const &kvp : kvp.second) { auto const &k = kvp.first; auto const &v = kvp.second; ns_rec.data[k] = std::move(v); } ns_rec.publish_all(); } } PersistenceServiceServer::~PersistenceServiceServer() {} void PersistenceServiceServer::retrieve( const RetrieveRequest &request, const std::function &reply) { LOG(persist, Diag, "Retrieving {} rows from {}/{}", request.key().size(), me->db, request.ns()); RetrieveResponse response; if (request.ns().empty()) { LOG(persist, Error, "Client used empty namespace while trying to retrieve rows in " "database {}", me->db); response.set_result(RetrieveResponse::EMPTY_NS); reply(response); return; } auto it = me->data.find(request.ns()); if (it == me->data.end()) { LOG(persist, Warn, "Client tried to retrieve specific rows from unknown namespace {}/{}", me->db, request.ns()); response.set_result(RetrieveResponse::NO_SUCH_NS); reply(response); return; } auto &known_rows = it->second.data; std::set keys; for (auto &k : request.key()) { if (keys.count(k)) { LOG(persist, Error, "Client tried to retrieve row with key {} twice in {}/{}", k, me->db, request.ns()); response.set_result(RetrieveResponse::DUPLICATE_KEY); reply(response); return; } auto it = known_rows.find(k); if (it == known_rows.end()) { LOG(persist, Warn, "Client tried to retrieve unknown row with key {} in {}/{}", k, me->db, request.ns()); response.set_result(RetrieveResponse::DUPLICATE_KEY); reply(response); return; } Record r; r.set_key(k); r.set_value(it->second); *response.add_record() = r; } response.set_result(RetrieveResponse::OK); reply(response); LOG(persist, Diag, "Successfully retrieved {} rows from {}/{}", request.key().size(), me->db, request.ns()); } void PersistenceServiceServer::retrieve_all( const RetrieveAllRequest &request, const std::function &reply) { LOG(persist, Diag, "Retrieving all rows from {}/{}", me->db, request.ns()); RetrieveResponse response; if (request.ns().empty()) { LOG(persist, Warn, "Client used empty namespace while trying to retrieve rows in " "database {}", me->db); response.set_result(RetrieveResponse::EMPTY_NS); reply(response); return; } auto it = me->data.find(request.ns()); if (it == me->data.end()) { response.set_result(RetrieveResponse::NO_SUCH_NS); reply(response); return; } auto v = it->second.to_vec(); for (auto &r : v) *response.add_record() = r; response.set_result(RetrieveResponse::OK); reply(response); LOG(persist, Diag, "Successfully retrieved all {} rows from {}/{}", v.size(), me->db, request.ns()); } void PersistenceServiceServer::remove( const RemoveRequest &request, const std::function &reply) { LOG(persist, Diag, "Removing {} rows from {}/{}", request.key().size(), me->db, request.ns()); RemoveResponse response; if (request.ns().empty()) { LOG(persist, Error, "Client used empty namespace while trying to remove rows in database " "{}", me->db); response.set_result(RemoveResponse::EMPTY_NS); reply(response); return; } auto it = me->data.find(request.ns()); if (it == me->data.end()) { LOG(persist, Warn, "Client used unknown namespace {} while trying to remove rows in " "database {}", request.ns(), me->db); response.set_result(RemoveResponse::NO_SUCH_NS); reply(response); return; } auto &ns_rec = it->second; std::set keys; for (auto &k : request.key()) { if (keys.count(k)) { LOG(persist, Warn, "Client used duplicate key {} while trying to remove rows in " "{}/{}", k, me->db, request.ns()); response.set_result(RemoveResponse::DUPLICATE_KEY); reply(response); return; } keys.insert(k); if (k.empty()) { LOG(persist, Warn, "Client used empty key while trying to remove rows in {}/{}", me->db, request.ns()); response.set_result(RemoveResponse::EMPTY_KEY); reply(response); return; } if (ns_rec.data.find(k) == ns_rec.data.end()) { LOG(persist, Info, "Client tried to remove unknown row with key {} in {}/{}", k, me->db, request.ns()); response.set_result(RemoveResponse::NO_SUCH_KEY); reply(response); return; } } // update memory for (auto &k : keys) ns_rec.data.erase(k); // update store me->store.remove(request.ns(), keys); // update stream ns_rec.publish(nullptr, nullptr, &keys); response.set_result(RemoveResponse::OK); reply(response); LOG(persist, Diag, "Successfully removed {} rows from {}/{}", keys.size(), me->db, request.ns()); } void PersistenceServiceServer::remove_all( const RemoveAllRequest &request, const std::function &reply) { LOG(persist, Diag, "Removing all rows from {}/{}", me->db, request.ns()); RemoveResponse response; if (request.ns().empty()) { LOG(persist, Error, "Client used empty namespace while trying to remove all rows in " "database {}", me->db); response.set_result(RemoveResponse::EMPTY_NS); reply(response); return; } auto it = me->data.find(request.ns()); if (it == me->data.end()) { LOG(persist, Error, "Client used unknown namespace {} while trying to remove all rows in " "database {}", request.ns(), me->db); response.set_result(RemoveResponse::NO_SUCH_NS); reply(response); return; } auto &ns_rec = it->second; std::set keys; for (auto &kvp : ns_rec.data) keys.insert(kvp.first); // update memory ns_rec.data.clear(); // update store me->store.remove_all(request.ns()); // update stream ns_rec.publish(nullptr, nullptr, &keys); response.set_result(RemoveResponse::OK); reply(response); LOG(persist, Diag, "Successfully removed all {} rows from {}/{}", keys.size(), me->db, request.ns()); } void PersistenceServiceServer::store( const StoreRequest &request, const std::function &reply) { LOG(persist, Diag, "Storing {} rows in {}/{}", request.record().size(), me->db, request.ns()); StoreResponse response; if (request.ns().empty()) { LOG(persist, Error, "Client used empty namespace while trying to store rows in database " "{}", me->db); response.set_result(StoreResponse::EMPTY_NS); reply(response); return; } NSRecord *rec = nullptr; auto it = me->data.find(request.ns()); if (it == me->data.end()) { rec = &me->data[request.ns()]; rec->publish = make_ns_publisher(request.ns()); } else rec = &(it->second); std::set keys; for (auto &r : request.record()) { auto const &k = r.key(); if (keys.count(k)) { LOG(persist, Error, "Client used duplicate key {} while trying to store rows in {}/{}", k, me->db, request.ns()); response.set_result(StoreResponse::DUPLICATE_KEY); reply(response); return; } if (k.empty()) { LOG(persist, Error, "Client used empty key while trying to store rows in {}/{}", me->db, request.ns()); response.set_result(StoreResponse::EMPTY_KEY); reply(response); return; } } std::vector insert; std::vector update; for (auto &record : request.record()) { auto it = rec->data.find(record.key()); // update memory me->data[request.ns()].data[record.key()] = record.value(); if (it != rec->data.end()) { // update store update.push_back(record); me->store.update(request.ns(), record.key(), record.value()); } else { // insert in store insert.push_back(record); me->store.insert(request.ns(), record.key(), record.value()); } } rec->publish(&insert, &update, nullptr); response.set_result(StoreResponse::OK); reply(response); LOG(persist, Diag, "Successfully stored {} rows in {}/{}", request.record().size(), me->db, request.ns()); } } // namespace persistence } // namespace bts