Persistence layer for vols

This commit is contained in:
Taha Afzal authored and John Colvin committed 2021-12-09 11:40:50 -05:00
1 parent 533a5c8f4a
commit 0ee03a0d30
4 files changed
+211 -28

No files matched your search

@@ -1,32 +1,72 @@
#include "AnalyticsServiceServer.h"
#include "lib/api/apicore.pb.h"
#include "lib/api/services/AnalyticsService.pb.h"
#include "lib/api/services/AnalyticsService.server.pb.h"
#include "lib/logging/logging.h"
#include "lib/persistence/IStore.h"
#include "lib/persistence/SqliteStore.h"
#include "lib/utils/PBUtils.h"
#include "lib/utils/Puid.h"
#include "lib/utils/bts_assert.h"
#include <boost/tokenizer.hpp>
#include <chrono>
#include <cstdint>
#include <memory>
#include <string>
#include <tuple>
using namespace bts::api::analytics;
namespace bts::analytics {
// ns = vs_key_name
std::tuple<int, std::string> parse_vs_id_name(const std::string &ns) {
boost::char_separator<char> sep(".");
boost::tokenizer<boost::char_separator<char>> tokens(ns, sep);
std::vector<std::string> parts;
for (const auto &t : tokens) {
parts.push_back(t);
}
bts_assert(parts.size() == 3);
return {std::stoi(parts[1]), parts[2]};
}
struct AnalyticsServiceServer::PImpl {
bts::api::ServiceContext &context;
std::unique_ptr<persistence::IStore> vs_fill_store;
std::unique_ptr<persistence::IStore> vs_store;
std::unique_ptr<persistence::IStore> exp_params_store;
std::unique_ptr<persistence::IStore> vs_usage_store;
AnalyticsServiceBase::VolsurfacesStatefulPublisher vs_publisher;
AnalyticsServiceBase::Exp_parametersStatefulPublisher exp_params_publisher;
AnalyticsServiceBase::Volsurface_usageStatefulPublisher vs_usage_publisher;
PImpl(
bts::api::ServiceContext &context,
const AnalyticsServiceBase::VolsurfacesStatefulPublisher &vs_publisher)
const AnalyticsServiceBase::VolsurfacesStatefulPublisher &vs_publisher,
const AnalyticsServiceBase::Exp_parametersStatefulPublisher
&exp_params_publisher,
const AnalyticsServiceBase::Volsurface_usageStatefulPublisher
&vs_usage_publisher)
: context(context),
vs_fill_store(
std::make_unique<persistence::SqliteStore>("volsurfaces")),
vs_publisher(vs_publisher) {}
vs_store(std::make_unique<persistence::SqliteStore>("volsurfaces")),
exp_params_store(std::make_unique<persistence::SqliteStore>(
"analytics_expiration_params")),
vs_usage_store(
std::make_unique<persistence::SqliteStore>("volsurface_usage")),
vs_publisher(vs_publisher), exp_params_publisher(exp_params_publisher),
vs_usage_publisher(vs_usage_publisher) {}
};
AnalyticsServiceServer::AnalyticsServiceServer(
bts::api::ServiceContext &context)
: AnalyticsServiceBase(context),
me(new PImpl(context, make_volsurfaces_stateful_publisher())) {
auto all = me->vs_fill_store->retrieve_all();
LOG(persist, Info, "Finished loading all {} manual fills", all.size());
for (auto const &ns_vals : all) {
me(new PImpl(
context,
make_volsurfaces_stateful_publisher(),
make_exp_parameters_stateful_publisher(),
make_volsurface_usage_stateful_publisher())) {
// publishing all vol surfaces
auto vs_table = me->vs_store->retrieve_all();
for (auto const &ns_vals : vs_table) {
const std::string &vs_name = ns_vals.first;
const std::map<std::string, std::string> &vals = ns_vals.second;
LOG(OrderMan, Info, "vs:{} entries:{}", vs_name, vals.size());
@@ -34,31 +74,113 @@ AnalyticsServiceServer::AnalyticsServiceServer(
const std::string &key = kvp.first;
const std::string &val = kvp.second;
LOG(OrderMan, Info, "vs:{} key:{} val:{}", vs_name, key, val);
VolSurfaceEntry vs_entry;
utils::pb::from_string(val, &vs_entry);
me->vs_publisher.add_or_update(vs_entry);
}
}
me->vs_publisher.flush();
// exp params
auto exp_params_table = me->exp_params_store->retrieve_all();
bts_assert(exp_params_table.size() == 0 || exp_params_table.size() == 1);
for (const auto &ns_vals : exp_params_table) {
const std::map<std::string, std::string> &vals = ns_vals.second;
bts_assert(vals.size() == 0 || vals.size() == 1);
for (const auto &kvp : vals) {
ExpirationParameterEntryList exp_params;
utils::pb::from_string(kvp.second, &exp_params);
me->exp_params_publisher.add_or_update(exp_params);
}
}
me->exp_params_publisher.flush();
}
void AnalyticsServiceServer::get_new_volsurface_id(
const ::bts::api::VoidMessage &,
const bts::api::Response<
::bts::api::analytics::AnalyticsServiceCreateResponse> &) {}
const bts::api::Response<::bts::api::analytics::AnalyticsServiceCreateReply>
&response) {
auto all = me->vs_store->retrieve_all();
int max_id = 0;
std::string curr_vs_name = "";
for (const auto &kvp : all) {
const std::string &ns = kvp.first;
auto [vs_id, vs_name] = parse_vs_id_name(ns);
if (vs_id >= max_id) {
max_id = vs_id;
curr_vs_name = vs_name;
}
}
bts::api::analytics::AnalyticsServiceCreateReply rep;
rep.set_ok(true);
rep.set_new_id(max_id + 1);
response(rep);
}
void AnalyticsServiceServer::save_volsurface(
const ::bts::api::analytics::VolSurfaceEntry &,
const bts::api::analytics::VolSurfaceEntry &vs_entry_arg,
const api::Response<::bts::api::ResultMessage> &) {
// ns = vs_key_name
// key = posix save time
// val = VolSurfaceEntry
using namespace std::chrono;
milliseconds ms =
duration_cast<milliseconds>(system_clock::now().time_since_epoch());
uint64_t ms_u = ms.count();
VolSurfaceEntry vs_entry = vs_entry_arg;
vs_entry.set_save_time(ms_u);
// namespace: volsurface_<vs-name>_<vs-id>
// key: save time
// value: VolSurfaceEntry
std::string ns = "volsurface_" + std::to_string(vs_entry.vs_id()) + "_" +
vs_entry.vs_name();
std::string key = std::to_string(ms_u);
std::string value = utils::pb::to_string(vs_entry);
me->vs_store->insert(ns, key, value);
me->vs_publisher.add_or_update(vs_entry);
me->vs_publisher.flush();
}
void AnalyticsServiceServer::delete_volsurface(
const ::bts::api::analytics::IntWrapper &,
const api::Response<::bts::api::ResultMessage> &) {}
const bts::api::analytics::IntWrapper &,
const api::Response<::bts::api::ResultMessage> &) {
// TODO
}
void AnalyticsServiceServer::clean_up(
const ::bts::api::analytics::IntWrapper &,
const api::Response<::bts::api::VoidMessage> &) {}
const api::Response<::bts::api::VoidMessage> &) {
// TODO
}
void AnalyticsServiceServer::set_exp_parameters(
const bts::api::analytics::ExpirationParameterEntryList &exp_params_args,
const api::Response<::bts::api::ResultMessage> &response) {
ExpirationParameterEntryList exp_params = exp_params_args;
exp_params.set_id(1);
std::string ns = "analytics_expiration_params";
std::string key = std::to_string(1);
std::string value = utils::pb::to_string(exp_params);
// TODO: check to see if it adds or not
me->exp_params_store->update(ns, key, value);
me->exp_params_publisher.add_or_update(exp_params);
me->exp_params_publisher.flush();
api::ResultMessage res;
res.set_success(true);
response(res);
}
void AnalyticsServiceServer::set_volsurface_usage_list(
const ::bts::api::analytics::VolSurfaceUsageList &vs_usage_list_args,
const api::Response<::bts::api::ResultMessage> &response) {
VolSurfaceUsageList vs_usage_list = vs_usage_list_args;
vs_usage_list.set_id(1);
std::string ns = "volsurface_usage_list";
std::string key = std::to_string(1);
std::string value = utils::pb::to_string(vs_usage_list);
me->vs_usage_store->update(ns, key, value);
me->vs_usage_publisher.add_or_update(vs_usage_list);
me->vs_usage_publisher.flush();
api::ResultMessage res;
res.set_success(true);
response(res);
}
AnalyticsServiceServer::~AnalyticsServiceServer() {}