476 lines
17 KiB
C++
476 lines
17 KiB
C++
#include "AnalyticsServiceServer.h"
|
|
#include "lib/api/apicore.pb.h"
|
|
#include "lib/api/services/AnalyticsService.pb.h"
|
|
#include "lib/api/services/AnalyticsService.server.pb.h"
|
|
#include "lib/api/services/ShellLatencyService.pb.h"
|
|
#include "lib/logging/logging.h"
|
|
#include "lib/persistence/IStore.h"
|
|
#include "lib/persistence/SqliteStore.h"
|
|
#include "lib/utils/PBUtils.h"
|
|
#include "lib/utils/Puid.h"
|
|
#include "lib/utils/bts_assert.h"
|
|
#include "server/sparkd/algo/AlgoDefinitionServiceServer.h"
|
|
#include <boost/algorithm/string.hpp>
|
|
#include <boost/tokenizer.hpp>
|
|
#include <chrono>
|
|
#include <cstdint>
|
|
#include <locale>
|
|
#include <memory>
|
|
#include <string>
|
|
#include <tuple>
|
|
|
|
using namespace bts::api::analytics;
|
|
|
|
namespace bts::analytics {
|
|
|
|
uint64_t get_epoch() {
|
|
using namespace std::chrono;
|
|
milliseconds ms =
|
|
duration_cast<milliseconds>(system_clock::now().time_since_epoch());
|
|
uint64_t ms_u = ms.count();
|
|
return ms_u;
|
|
}
|
|
|
|
// assumes key are ints
|
|
int get_max_id(IStore *const store, const std::string &ns) {
|
|
auto ns_map = store->retrieve_all();
|
|
auto it = ns_map.find(ns);
|
|
if (it == ns_map.end()) {
|
|
return 0;
|
|
}
|
|
int max_id = -1;
|
|
for (const auto &kv : it->second) {
|
|
int key = std::stoi(kv.first);
|
|
max_id = key > max_id ? key : max_id;
|
|
}
|
|
return max_id;
|
|
}
|
|
|
|
void add_or_update_to_table(
|
|
persistence::IStore *store,
|
|
const std::string &ns,
|
|
const std::string &key,
|
|
const std::string &val) {
|
|
auto [exists, existing_val] = store->retrieve_one(ns, key);
|
|
LOG(Gen, Info,
|
|
"add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns,
|
|
key, val, exists, existing_val);
|
|
if (exists) {
|
|
store->update(ns, key, val);
|
|
} else {
|
|
store->insert(ns, key, val);
|
|
}
|
|
}
|
|
|
|
// parses out the vs_id and vs_name from the key
|
|
std::tuple<int, std::string> parse_vs_id_name(
|
|
const std::string &ns, const std::string &sep_str) {
|
|
std::vector<std::string> parts;
|
|
boost::split(parts, ns, boost::is_any_of(sep_str));
|
|
bts_assert(parts.size() == 3);
|
|
return {std::stoi(parts[1]), parts[2]};
|
|
}
|
|
|
|
void fire_cb(
|
|
const api::Response<::bts::api::ResultMessage> &resp,
|
|
bool is_success,
|
|
std::string const *msg = nullptr) {
|
|
api::ResultMessage res;
|
|
res.set_success(is_success);
|
|
if (msg != nullptr) {
|
|
res.set_error_message(*msg);
|
|
}
|
|
resp(res);
|
|
}
|
|
|
|
struct AnalyticsServiceServer::PImpl {
|
|
bts::api::ServiceContext &context;
|
|
const std::string vs_ns_sep = ".";
|
|
const std::string exp_params_ns = "analytics_expiration_params";
|
|
const std::string vs_usage_ns = "volsurface_usage";
|
|
const std::string reports_ns = "reports";
|
|
const std::string archived_theos_ns = "archived_theos";
|
|
std::unique_ptr<persistence::IStore> vs_store;
|
|
std::unique_ptr<persistence::IStore> exp_params_store;
|
|
std::unique_ptr<persistence::IStore> vs_usage_store;
|
|
std::unique_ptr<persistence::IStore> reports_store;
|
|
std::unique_ptr<persistence::IStore> theos_store;
|
|
AnalyticsServiceBase::VolsurfacesStatefulPublisher vs_publisher;
|
|
AnalyticsServiceBase::Exp_parametersStatefulPublisher exp_params_publisher;
|
|
AnalyticsServiceBase::Volsurface_usageStatefulPublisher vs_usage_publisher;
|
|
AnalyticsServiceBase::ReportsStatefulPublisher reports_publisher;
|
|
PImpl(
|
|
bts::api::ServiceContext &context,
|
|
const AnalyticsServiceBase::VolsurfacesStatefulPublisher &vs_publisher,
|
|
const AnalyticsServiceBase::Exp_parametersStatefulPublisher
|
|
&exp_params_publisher,
|
|
const AnalyticsServiceBase::Volsurface_usageStatefulPublisher
|
|
&vs_usage_publisher,
|
|
const AnalyticsServiceBase::ReportsStatefulPublisher &reports_publisher)
|
|
: context(context),
|
|
vs_store(std::make_unique<persistence::SqliteStore>("volsurfaces")),
|
|
exp_params_store(std::make_unique<persistence::SqliteStore>(
|
|
"analytics_expiration_params")),
|
|
vs_usage_store(
|
|
std::make_unique<persistence::SqliteStore>("volsurface_usage")),
|
|
reports_store(std::make_unique<persistence::SqliteStore>("reports")),
|
|
theos_store(
|
|
std::make_unique<persistence::SqliteStore>("archived_theos")),
|
|
vs_publisher(vs_publisher), exp_params_publisher(exp_params_publisher),
|
|
vs_usage_publisher(vs_usage_publisher),
|
|
reports_publisher(reports_publisher) {}
|
|
};
|
|
|
|
AnalyticsServiceServer::AnalyticsServiceServer(
|
|
bts::api::ServiceContext &context)
|
|
: AnalyticsServiceBase(context),
|
|
me(new PImpl(
|
|
context,
|
|
make_volsurfaces_stateful_publisher(),
|
|
make_exp_parameters_stateful_publisher(),
|
|
make_volsurface_usage_stateful_publisher(),
|
|
make_reports_stateful_publisher())) {
|
|
// publishing all vol surfaces
|
|
LOG(Gen, Info, "Starting AnalyticsServiceServer");
|
|
_clean_up_vss();
|
|
auto vs_table = me->vs_store->retrieve_all();
|
|
for (auto const &ns_vals : vs_table) {
|
|
const std::string &vs_name = ns_vals.first;
|
|
const std::map<std::string, std::string> &key_vals = ns_vals.second;
|
|
LOG(OrderMan, Info, "vs:{} entries:{}", vs_name, key_vals.size());
|
|
for (const auto &kvp : key_vals) {
|
|
const std::string &key = kvp.first;
|
|
const std::string &val = kvp.second;
|
|
LOG(OrderMan, Diag, "vs:{} key:{} val:{}", vs_name, key, val);
|
|
VolSurfaceEntry vs_entry;
|
|
utils::pb::from_string(val, &vs_entry);
|
|
me->vs_publisher.add_or_update(vs_entry);
|
|
}
|
|
}
|
|
me->vs_publisher.flush();
|
|
// exp params
|
|
auto exp_params_table = me->exp_params_store->retrieve_all();
|
|
bts_assert(exp_params_table.size() == 0 || exp_params_table.size() == 1);
|
|
for (const auto &ns_vals : exp_params_table) {
|
|
const std::map<std::string, std::string> &key_vals = ns_vals.second;
|
|
bts_assert(key_vals.size() == 0 || key_vals.size() == 1);
|
|
for (const auto &kvp : key_vals) {
|
|
ExpirationParameterEntryList exp_params;
|
|
utils::pb::from_string(kvp.second, &exp_params);
|
|
me->exp_params_publisher.add_or_update(exp_params);
|
|
}
|
|
}
|
|
me->exp_params_publisher.flush();
|
|
// vol surface usage
|
|
auto vs_usage_table = me->vs_usage_store->retrieve_all();
|
|
bts_assert(vs_usage_table.size() == 0 || vs_usage_table.size() == 1);
|
|
for (const auto &ns_vals : vs_usage_table) {
|
|
const std::map<std::string, std::string> &key_vals = ns_vals.second;
|
|
bts_assert(key_vals.size() == 0 || key_vals.size() == 1);
|
|
for (const auto &kvp : key_vals) {
|
|
VolSurfaceUsageList vs_usage_list;
|
|
utils::pb::from_string(kvp.second, &vs_usage_list);
|
|
me->vs_usage_publisher.add_or_update(vs_usage_list);
|
|
}
|
|
}
|
|
me->vs_usage_publisher.flush();
|
|
// reports
|
|
{
|
|
auto ns_table = me->reports_store->retrieve_all();
|
|
bts_assert(ns_table.size() == 0 || ns_table.size() == 1);
|
|
for (const auto &ns_vals : ns_table) {
|
|
for (const auto &kvp : ns_vals.second) {
|
|
Report report;
|
|
utils::pb::from_string(kvp.second, &report);
|
|
me->reports_publisher.add_or_update(report);
|
|
}
|
|
}
|
|
me->reports_publisher.flush();
|
|
}
|
|
}
|
|
|
|
void AnalyticsServiceServer::get_new_volsurface_id(
|
|
const ::bts::api::VoidMessage &,
|
|
const bts::api::Response<::bts::api::analytics::AnalyticsServiceCreateReply>
|
|
&response) {
|
|
auto all = me->vs_store->retrieve_all();
|
|
int max_id = 0;
|
|
std::string curr_vs_name = "";
|
|
for (const auto &ns_vals : all) {
|
|
const std::string &ns = ns_vals.first;
|
|
auto [vs_id, vs_name] = parse_vs_id_name(ns, me->vs_ns_sep);
|
|
if (vs_id >= max_id) {
|
|
max_id = vs_id;
|
|
curr_vs_name = vs_name;
|
|
}
|
|
}
|
|
bts::api::analytics::AnalyticsServiceCreateReply rep;
|
|
rep.set_ok(true);
|
|
rep.set_new_id(max_id + 1);
|
|
response(rep);
|
|
}
|
|
|
|
void AnalyticsServiceServer::save_volsurface(
|
|
const bts::api::analytics::VolSurfaceEntry &vs_entry_arg,
|
|
const api::Response<::bts::api::ResultMessage> &response) {
|
|
uint64_t ms_u = get_epoch();
|
|
VolSurfaceEntry vs_entry = vs_entry_arg;
|
|
vs_entry.set_save_time(ms_u);
|
|
// namespace: volsurface_<vs-name>_<vs-id>
|
|
// key: save_time epoch
|
|
// value: VolSurfaceEntry
|
|
std::string ns = "volsurface" + me->vs_ns_sep +
|
|
std::to_string(vs_entry.vs_id()) + me->vs_ns_sep +
|
|
vs_entry.vs_name();
|
|
std::string key = std::to_string(ms_u);
|
|
std::string value = utils::pb::to_string(vs_entry);
|
|
me->vs_store->insert(ns, key, value);
|
|
me->vs_publisher.add_or_update(vs_entry);
|
|
me->vs_publisher.flush();
|
|
api::ResultMessage ret;
|
|
ret.set_success(true);
|
|
response(ret);
|
|
}
|
|
|
|
void AnalyticsServiceServer::delete_volsurface(
|
|
const bts::api::analytics::IntWrapper &vs_id_w,
|
|
const api::Response<::bts::api::ResultMessage> &response) {
|
|
auto vs_id = vs_id_w.data();
|
|
auto vs_vals = me->vs_store->retrieve_all();
|
|
auto ns_substr = "volsurface." + std::to_string(vs_id) + ".";
|
|
|
|
size_t n_elems_removed = 0;
|
|
for (const auto &kv : vs_vals) {
|
|
const auto &ns = kv.first;
|
|
if (ns.find(ns_substr) != std::string::npos) {
|
|
n_elems_removed = kv.second.size();
|
|
// this is the ns we want to delete
|
|
// first delete it from the live stream
|
|
for (const auto &kv_inside : kv.second) {
|
|
const auto &key_str = kv_inside.first;
|
|
uint64_t key = std::stoull(key_str);
|
|
me->vs_publisher.erase(key);
|
|
}
|
|
me->vs_publisher.flush();
|
|
me->vs_store->remove_all(ns);
|
|
}
|
|
}
|
|
LOG(Gen, Info,
|
|
"AnalyticsServiceServer::delete_volsurface entries removed: {}",
|
|
n_elems_removed);
|
|
api::ResultMessage res;
|
|
if (n_elems_removed > 0) {
|
|
res.set_success(true);
|
|
} else {
|
|
res.set_success(false);
|
|
res.set_error_message("Could not find any entries");
|
|
}
|
|
response(res);
|
|
}
|
|
|
|
void AnalyticsServiceServer::clean_up(
|
|
const ::bts::api::analytics::IntWrapper &,
|
|
const api::Response<::bts::api::VoidMessage> &) {
|
|
// TODO
|
|
}
|
|
|
|
void AnalyticsServiceServer::set_exp_parameters(
|
|
const bts::api::analytics::ExpirationParameterEntryList &exp_params_args,
|
|
const api::Response<::bts::api::ResultMessage> &response) {
|
|
LOG(Gen, Info, "AnalyticsServiceServer::set_exp_parameters params: {}",
|
|
exp_params_args.param_list().size());
|
|
if (exp_params_args.param_list().empty()) {
|
|
std::string err_msg = "Empty params list, call failed";
|
|
LOG(Gen, Error, "AnalyticsServiceServer::set_exp_parameters {}", err_msg);
|
|
fire_cb(response, false, &err_msg);
|
|
return;
|
|
}
|
|
ExpirationParameterEntryList exp_params = exp_params_args;
|
|
exp_params.set_id(1);
|
|
std::string ns = me->exp_params_ns;
|
|
std::string key = std::to_string(1);
|
|
std::string value = utils::pb::to_string(exp_params);
|
|
add_or_update_to_table(me->exp_params_store.get(), ns, key, value);
|
|
me->exp_params_publisher.add_or_update(exp_params);
|
|
me->exp_params_publisher.flush();
|
|
fire_cb(response, true);
|
|
}
|
|
|
|
void AnalyticsServiceServer::set_volsurface_usage_list(
|
|
const ::bts::api::analytics::VolSurfaceUsageList &vs_usage_list_args,
|
|
const api::Response<::bts::api::ResultMessage> &response) {
|
|
VolSurfaceUsageList vs_usage_list = vs_usage_list_args;
|
|
vs_usage_list.set_id(1);
|
|
std::string ns = me->vs_usage_ns;
|
|
std::string key = std::to_string(1);
|
|
std::string value = utils::pb::to_string(vs_usage_list);
|
|
me->vs_usage_store->update(ns, key, value);
|
|
me->vs_usage_publisher.add_or_update(vs_usage_list);
|
|
me->vs_usage_publisher.flush();
|
|
api::ResultMessage res;
|
|
res.set_success(true);
|
|
response(res);
|
|
}
|
|
|
|
void AnalyticsServiceServer::get_new_report_id(
|
|
const ::bts::api::VoidMessage &,
|
|
const bts::api::Response<::bts::api::analytics::AnalyticsServiceCreateReply>
|
|
&response) {
|
|
auto max_id = get_max_id(me->reports_store.get(), me->reports_ns);
|
|
auto new_id = max_id + 1;
|
|
bts::api::analytics::AnalyticsServiceCreateReply rep;
|
|
rep.set_ok(true);
|
|
rep.set_new_id(new_id);
|
|
response(rep);
|
|
}
|
|
|
|
void AnalyticsServiceServer::save_report(
|
|
const ::bts::api::analytics::Report &report_arg,
|
|
const api::Response<::bts::api::ResultMessage> &resp) {
|
|
Report report = report_arg;
|
|
std::string key = std::to_string(report.id());
|
|
auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key);
|
|
if (!present) {
|
|
int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1;
|
|
report.set_id(new_id);
|
|
} else {
|
|
bts_assert(!val.empty());
|
|
}
|
|
std::string report_str = utils::pb::to_string(report);
|
|
add_or_update_to_table(
|
|
me->reports_store.get(), me->reports_ns, std::to_string(report.id()),
|
|
report_str);
|
|
me->reports_publisher.add_or_update(report);
|
|
me->reports_publisher.flush();
|
|
api::ResultMessage res;
|
|
res.set_success(true);
|
|
resp(res);
|
|
}
|
|
|
|
void AnalyticsServiceServer::delete_report(
|
|
const bts::api::analytics::IntWrapper &id,
|
|
const api::Response<::bts::api::ResultMessage> &resp) {
|
|
std::string key = std::to_string(id.data());
|
|
auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key);
|
|
api::ResultMessage res;
|
|
res.set_success(false);
|
|
if (!present) {
|
|
res.set_error_message("Could not find report");
|
|
resp(res);
|
|
return;
|
|
}
|
|
bts_assert(!val.empty());
|
|
me->reports_publisher.erase(id.data());
|
|
me->reports_publisher.flush();
|
|
me->reports_store->remove(me->reports_ns, {key});
|
|
res.set_success(true);
|
|
resp(res);
|
|
}
|
|
|
|
void AnalyticsServiceServer::archive_theos(
|
|
const bts::api::analytics::ArchivedTheoList &theo_list,
|
|
const api::Response<::bts::api::ResultMessage> &resp) {
|
|
LOG(Gen, Info,
|
|
"AnalyticsServiceServer::archive_theos sec_tups: {} sec_theo: {}",
|
|
theo_list.sec_tups().size(), theo_list.sec_theo().size());
|
|
if (theo_list.sec_theo().empty()) {
|
|
std::string err_msg = "Empty archived theo list, call failed";
|
|
LOG(Gen, Error, "AnalyticsServiceServer::archive_theos {}", err_msg);
|
|
fire_cb(resp, false, &err_msg);
|
|
return;
|
|
}
|
|
const std::string &ns = me->archived_theos_ns;
|
|
const std::string key = std::to_string(theo_list.save_time());
|
|
std::string value = utils::pb::to_string(theo_list);
|
|
me->theos_store->insert(ns, key, value);
|
|
fire_cb(resp, true);
|
|
}
|
|
|
|
void AnalyticsServiceServer::get_archived_theos(
|
|
const bts::api::analytics::ArchivedTheoRetrieval &ret,
|
|
const bts::api::Response<bts::api::analytics::ArchivedTheoList> &resp) {
|
|
LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {}",
|
|
utils::pb::to_string(ret));
|
|
auto contains_st = [&](const ArchivedTheoList &tl) {
|
|
for (const auto &st : tl.sec_tups()) {
|
|
// TODO: compare all, just not symbol
|
|
if (st.symbol() == ret.sec_tup().symbol()) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
};
|
|
ArchivedTheoList rv = {};
|
|
const auto &rows = me->theos_store->retrieve_all();
|
|
uint64_t max_diff = std::numeric_limits<uint64_t>::max();
|
|
std::string key = "";
|
|
// this is rather slow, we can speed up by sorting the keys
|
|
// and searching based on distance from ret.save_time()
|
|
for (const auto &ns : rows) {
|
|
for (const auto &kv : ns.second) {
|
|
ArchivedTheoList tl;
|
|
utils::pb::from_string(kv.second, &tl);
|
|
if (contains_st(tl)) {
|
|
uint64_t ts = std::stoull(kv.first);
|
|
int64_t diff = ts - ret.save_time();
|
|
uint64_t abs_diff = std::abs(diff);
|
|
if (abs_diff < max_diff) {
|
|
key = kv.first;
|
|
max_diff = abs_diff;
|
|
rv = tl;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {} key: {}",
|
|
utils::pb::to_string(ret), key);
|
|
resp(rv);
|
|
}
|
|
|
|
AnalyticsServiceServer::~AnalyticsServiceServer() {}
|
|
|
|
void AnalyticsServiceServer::_clean_up_vss() {
|
|
const size_t final_sz = 500;
|
|
auto all_rows = me->vs_store->retrieve_all();
|
|
for (const auto &ns_k : all_rows) {
|
|
const auto &vs = ns_k.first;
|
|
const auto &vs_row_map = ns_k.second;
|
|
if (vs_row_map.size() <= final_sz) {
|
|
LOG(Gen, Info,
|
|
"AnalyticsServiceServer::_clean_up_vss ignoring vs: {} because it "
|
|
"has only {} rows",
|
|
vs, vs_row_map.size());
|
|
continue;
|
|
}
|
|
std::vector<uint64_t> keys_uint;
|
|
for (const auto &r : vs_row_map) {
|
|
const auto &k = r.first;
|
|
try {
|
|
keys_uint.push_back(std::stoull(k));
|
|
} catch (int) {
|
|
LOG(Gen, Error,
|
|
"AnalyticsServiceServer::_clean_up_vss could not convert "
|
|
"key: "
|
|
"{} to uint",
|
|
k);
|
|
}
|
|
}
|
|
// sort and remove
|
|
std::sort(keys_uint.begin(), keys_uint.end());
|
|
size_t n_del_rows = keys_uint.size() - final_sz;
|
|
std::vector<uint64_t> keys_uint_to_del = std::vector<uint64_t>(
|
|
keys_uint.begin(), keys_uint.begin() + n_del_rows);
|
|
std::set<std::string> keys_to_del;
|
|
for (auto k : keys_uint_to_del) {
|
|
keys_to_del.insert(std::to_string(k));
|
|
}
|
|
LOG(Gen, Info,
|
|
"AnalyticsServiceServer::_clean_up_vss vs: {} n_rows: {} removing {} "
|
|
"rows",
|
|
vs, keys_uint.size(), keys_to_del.size());
|
|
me->vs_store->remove(vs, keys_to_del);
|
|
}
|
|
}
|
|
|
|
} // namespace bts::analytics
|