Files
spark/src/server/sparkd/analytics/AnalyticsServiceServer.cpp
T
John Colvin bd8c97244a refactor: Return row id in IStore::retrieve_one
This is a prerequisite to synchronizing security ids stored via IDGenerator
(the "new" way to store ids, added when we implemented Deribit), and
SecurityDB (the original way from spark days). The problem was that these
2 implementations were using separate databases. We need to unify the
interfaces between these storage services to avoid breaking existing code.
2024-11-20 22:17:08 +00:00

571 lines
21 KiB
C++
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#include "AnalyticsServiceServer.h"
#include "lib/api/apicore.pb.h"
#include "lib/api/services/AnalyticsService.pb.h"
#include "lib/api/services/AnalyticsService.server.pb.h"
#include "lib/api/services/ShellLatencyService.pb.h"
#include "lib/logging/logging.h"
#include "lib/persistence/IStore.h"
#include "lib/persistence/SqliteStore.h"
#include "lib/utils/PBUtils.h"
#include "lib/utils/Puid.h"
#include "lib/utils/TimeStamp.h"
#include "lib/utils/bts_assert.h"
#include "server/sparkd/algo/AlgoDefinitionServiceServer.h"
#include <boost/algorithm/string.hpp>
#include <boost/tokenizer.hpp>
#include <chrono>
#include <cstdint>
#include <locale>
#include <memory>
#include <string>
#include <tuple>
using namespace bts::api::analytics;
namespace bts::analytics {
uint64_t get_epoch() {
using namespace std::chrono;
milliseconds ms =
duration_cast<milliseconds>(system_clock::now().time_since_epoch());
uint64_t ms_u = ms.count();
return ms_u;
}
// assumes key are ints
int get_max_id(IStore *const store, const std::string &ns) {
auto ns_map = store->retrieve_all();
auto it = ns_map.find(ns);
if (it == ns_map.end()) {
return 0;
}
int max_id = -1;
for (const auto &kv : it->second) {
int key = std::stoi(kv.first);
max_id = key > max_id ? key : max_id;
}
return max_id;
}
void add_or_update_to_table(
persistence::IStore *store,
const std::string &ns,
const std::string &key,
const std::string &val) {
auto [exists, rowid, existing_val] = store->retrieve_one(ns, key);
LOG(Gen, Info,
"add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns,
key, val, exists, existing_val);
if (exists) {
store->update(ns, key, val);
} else {
store->insert(ns, key, val);
}
}
// parses out the vs_id and vs_name from the key
std::tuple<int, std::string> parse_vs_id_name(
const std::string &ns, const std::string &sep_str) {
std::vector<std::string> parts;
boost::split(parts, ns, boost::is_any_of(sep_str));
bts_assert(parts.size() == 3);
return {std::stoi(parts[1]), parts[2]};
}
void fire_cb(
const api::Response<::bts::api::ResultMessage> &resp,
bool is_success,
std::string const *msg = nullptr) {
api::ResultMessage res;
res.set_success(is_success);
if (msg != nullptr) {
res.set_error_message(*msg);
}
resp(res);
}
struct AnalyticsServiceServer::PImpl {
bts::api::ServiceContext &context;
const std::string vs_ns_sep = ".";
const std::string exp_params_ns = "analytics_expiration_params";
const std::string vs_usage_ns = "volsurface_usage";
const std::string reports_ns = "reports";
const std::string archived_theos_ns = "archived_theos";
// this is the total number of vs data that will be stored in the db,
// beyond that, old values will be recycled
const size_t vs_history_sz = 500;
// anologous to above for archived theos (however unlike the above this is
// recycled periodically, and is not published out)
const size_t archived_theos_history_sz = 5000;
const size_t n_theos_archives_before_cleanup = 100;
size_t n_theos_archives = 0;
std::unique_ptr<persistence::IStore> vs_store;
std::unique_ptr<persistence::IStore> exp_params_store;
std::unique_ptr<persistence::IStore> vs_usage_store;
std::unique_ptr<persistence::IStore> reports_store;
std::unique_ptr<persistence::IStore> theos_store;
AnalyticsServiceBase::VolsurfacesStatefulPublisher vs_publisher;
AnalyticsServiceBase::Exp_parametersStatefulPublisher exp_params_publisher;
AnalyticsServiceBase::Volsurface_usageStatefulPublisher vs_usage_publisher;
AnalyticsServiceBase::ReportsStatefulPublisher reports_publisher;
PImpl(
bts::api::ServiceContext &context,
const AnalyticsServiceBase::VolsurfacesStatefulPublisher &vs_publisher,
const AnalyticsServiceBase::Exp_parametersStatefulPublisher
&exp_params_publisher,
const AnalyticsServiceBase::Volsurface_usageStatefulPublisher
&vs_usage_publisher,
const AnalyticsServiceBase::ReportsStatefulPublisher &reports_publisher)
: context(context),
vs_store(std::make_unique<persistence::SqliteStore>("volsurfaces")),
exp_params_store(std::make_unique<persistence::SqliteStore>(
"analytics_expiration_params")),
vs_usage_store(
std::make_unique<persistence::SqliteStore>("volsurface_usage")),
reports_store(std::make_unique<persistence::SqliteStore>("reports")),
theos_store(
std::make_unique<persistence::SqliteStore>("archived_theos")),
vs_publisher(vs_publisher), exp_params_publisher(exp_params_publisher),
vs_usage_publisher(vs_usage_publisher),
reports_publisher(reports_publisher) {}
};
AnalyticsServiceServer::AnalyticsServiceServer(
bts::api::ServiceContext &context)
: AnalyticsServiceBase(context),
me(new PImpl(
context,
bts::api::StatefulPublisher<VolSurfaceEntry, uint64_t>(
make_volsurfaces_publisher(),
[](const VolSurfaceEntry &v) { return v.save_time(); },
true),
// make_volsurfaces_stateful_publisher(),
make_exp_parameters_stateful_publisher(),
make_volsurface_usage_stateful_publisher(),
make_reports_stateful_publisher())) {
// publishing all vol surfaces
LOG(Gen, Info, "Starting AnalyticsServiceServer");
_clean_up_vss();
_clean_up_archived_theos();
auto vs_table = me->vs_store->retrieve_all();
for (auto const &ns_vals : vs_table) {
const std::string &vs_name = ns_vals.first;
const std::map<std::string, std::string> &key_vals = ns_vals.second;
LOG(OrderMan, Info, "vs: {} entries: {}", vs_name, key_vals.size());
for (const auto &kvp : key_vals) {
const std::string &key = kvp.first;
const std::string &val = kvp.second;
LOG(OrderMan, Info,
"AnalyticsServiceServer::cons loaded from db vs: {} key: {}",
vs_name, key);
VolSurfaceEntry vs_entry;
utils::pb::from_string(val, &vs_entry);
LOG(Gen, Diag,
"AnalyticsServiceServer::cons loading to vs_publisher ns: {} "
"key: {} vs_name: {} vs_id: {} vs.save_time: {}",
vs_name, key, vs_entry.vs_name(), vs_entry.vs_id(),
vs_entry.save_time());
me->vs_publisher.add_or_update(vs_entry);
}
}
me->vs_publisher.flush();
// exp params
auto exp_params_table = me->exp_params_store->retrieve_all();
bts_assert(exp_params_table.size() == 0 || exp_params_table.size() == 1);
for (const auto &ns_vals : exp_params_table) {
const std::map<std::string, std::string> &key_vals = ns_vals.second;
bts_assert(key_vals.size() == 0 || key_vals.size() == 1);
for (const auto &kvp : key_vals) {
ExpirationParameterEntryList exp_params;
utils::pb::from_string(kvp.second, &exp_params);
me->exp_params_publisher.add_or_update(exp_params);
}
}
me->exp_params_publisher.flush();
// vol surface usage
auto vs_usage_table = me->vs_usage_store->retrieve_all();
bts_assert(vs_usage_table.size() == 0 || vs_usage_table.size() == 1);
for (const auto &ns_vals : vs_usage_table) {
const std::map<std::string, std::string> &key_vals = ns_vals.second;
bts_assert(key_vals.size() == 0 || key_vals.size() == 1);
for (const auto &kvp : key_vals) {
VolSurfaceUsageList vs_usage_list;
utils::pb::from_string(kvp.second, &vs_usage_list);
me->vs_usage_publisher.add_or_update(vs_usage_list);
}
}
me->vs_usage_publisher.flush();
// reports
{
auto ns_table = me->reports_store->retrieve_all();
bts_assert(ns_table.size() == 0 || ns_table.size() == 1);
for (const auto &ns_vals : ns_table) {
for (const auto &kvp : ns_vals.second) {
Report report;
utils::pb::from_string(kvp.second, &report);
me->reports_publisher.add_or_update(report);
}
}
me->reports_publisher.flush();
}
}
void AnalyticsServiceServer::get_new_volsurface_id(
const ::bts::api::VoidMessage &,
const bts::api::Response<::bts::api::analytics::AnalyticsServiceCreateReply>
&response) {
auto all = me->vs_store->retrieve_all();
int max_id = 0;
std::string curr_vs_name = "";
for (const auto &ns_vals : all) {
const std::string &ns = ns_vals.first;
auto [vs_id, vs_name] = parse_vs_id_name(ns, me->vs_ns_sep);
if (vs_id >= max_id) {
max_id = vs_id;
curr_vs_name = vs_name;
}
}
bts::api::analytics::AnalyticsServiceCreateReply rep;
rep.set_ok(true);
rep.set_new_id(max_id + 1);
response(rep);
}
void AnalyticsServiceServer::save_volsurface(
const bts::api::analytics::VolSurfaceEntry &vs_entry_arg,
const api::Response<::bts::api::ResultMessage> &response) {
auto t0 = utils::TimeStamp::now().to_uint64();
auto remove_oldest = [&](const std::string &vs_ns) {
// return;
int vs_id = vs_entry_arg.vs_id();
// find all the keys belonging to this vs
std::vector<uint64_t> vs_keys = {};
for (const auto &kv : me->vs_publisher.state_data()) {
if (kv.second.vs_id() == vs_id) {
vs_keys.push_back(kv.first);
}
}
LOG(Gen, Info,
"AnalyticsServiceServer::save_volsurface vs_id: {} n_entries: {}",
vs_entry_arg.vs_id(), vs_keys.size());
if (vs_keys.size() <= me->vs_history_sz)
return;
// since we are already truncating it at the the start, we'll just
// find the earliest key and delete it         
uint64_t key_to_delete = std::numeric_limits<uint64_t>::max();
for (uint64_t k : vs_keys) {
if (k < key_to_delete) {
key_to_delete = k;
}
}
LOG(Gen, Info,
"AnalyticsServiceServer::save_volsurface removing row ns: {} key: {}",
vs_ns, key_to_delete);
me->vs_store->remove(vs_ns, {std::to_string(key_to_delete)});
me->vs_publisher.erase(key_to_delete);
};
uint64_t ms_u = get_epoch();
VolSurfaceEntry vs_entry = vs_entry_arg;
vs_entry.set_save_time(ms_u);
// namespace: volsurface_<vs-name>_<vs-id>
// key: save_time epoch
// value: VolSurfaceEntry
std::string ns = "volsurface" + me->vs_ns_sep +
std::to_string(vs_entry.vs_id()) + me->vs_ns_sep +
vs_entry.vs_name();
std::string key = std::to_string(ms_u);
std::string value = utils::pb::to_string(vs_entry);
me->vs_store->insert(ns, key, value);
me->vs_publisher.add_or_update(vs_entry);
remove_oldest(ns);
me->vs_publisher.flush();
api::ResultMessage ret;
ret.set_success(true);
auto t1 = utils::TimeStamp::now().to_uint64();
LOG(Gen, Info,
"AnalyticsServiceServer::save_volsurface ns: {} key: {} n_entries: {} "
"time: {} ns",
ns, key, me->vs_publisher.state_data().size(), t1 - t0);
response(ret);
}
void AnalyticsServiceServer::delete_volsurface(
const bts::api::analytics::IntWrapper &vs_id_w,
const api::Response<::bts::api::ResultMessage> &response) {
auto vs_id = vs_id_w.data();
auto vs_vals = me->vs_store->retrieve_all();
auto ns_substr = "volsurface." + std::to_string(vs_id) + ".";
size_t n_elems_removed = 0;
for (const auto &kv : vs_vals) {
const auto &ns = kv.first;
if (ns.find(ns_substr) != std::string::npos) {
n_elems_removed = kv.second.size();
// this is the ns we want to delete
// first delete it from the live stream
for (const auto &kv_inside : kv.second) {
const auto &key_str = kv_inside.first;
uint64_t key = std::stoull(key_str);
me->vs_publisher.erase(key);
}
me->vs_publisher.flush();
me->vs_store->remove_all(ns);
}
}
LOG(Gen, Info,
"AnalyticsServiceServer::delete_volsurface entries removed: {}",
n_elems_removed);
api::ResultMessage res;
if (n_elems_removed > 0) {
res.set_success(true);
} else {
res.set_success(false);
res.set_error_message("Could not find any entries");
}
response(res);
}
void AnalyticsServiceServer::clean_up(
const ::bts::api::analytics::IntWrapper &,
const api::Response<::bts::api::VoidMessage> &) {
// TODO
}
void AnalyticsServiceServer::set_exp_parameters(
const bts::api::analytics::ExpirationParameterEntryList &exp_params_args,
const api::Response<::bts::api::ResultMessage> &response) {
LOG(Gen, Info, "AnalyticsServiceServer::set_exp_parameters params: {}",
exp_params_args.param_list().size());
if (exp_params_args.param_list().empty()) {
std::string err_msg = "Empty params list, call failed";
LOG(Gen, Error, "AnalyticsServiceServer::set_exp_parameters {}", err_msg);
fire_cb(response, false, &err_msg);
return;
}
ExpirationParameterEntryList exp_params = exp_params_args;
exp_params.set_id(1);
std::string ns = me->exp_params_ns;
std::string key = std::to_string(1);
std::string value = utils::pb::to_string(exp_params);
add_or_update_to_table(me->exp_params_store.get(), ns, key, value);
me->exp_params_publisher.add_or_update(exp_params);
me->exp_params_publisher.flush();
fire_cb(response, true);
}
void AnalyticsServiceServer::set_volsurface_usage_list(
const ::bts::api::analytics::VolSurfaceUsageList &vs_usage_list_args,
const api::Response<::bts::api::ResultMessage> &response) {
VolSurfaceUsageList vs_usage_list = vs_usage_list_args;
vs_usage_list.set_id(1);
std::string ns = me->vs_usage_ns;
std::string key = std::to_string(1);
std::string value = utils::pb::to_string(vs_usage_list);
me->vs_usage_store->update(ns, key, value);
me->vs_usage_publisher.add_or_update(vs_usage_list);
me->vs_usage_publisher.flush();
api::ResultMessage res;
res.set_success(true);
response(res);
}
void AnalyticsServiceServer::get_new_report_id(
const ::bts::api::VoidMessage &,
const bts::api::Response<::bts::api::analytics::AnalyticsServiceCreateReply>
&response) {
auto max_id = get_max_id(me->reports_store.get(), me->reports_ns);
auto new_id = max_id + 1;
bts::api::analytics::AnalyticsServiceCreateReply rep;
rep.set_ok(true);
rep.set_new_id(new_id);
response(rep);
}
void AnalyticsServiceServer::save_report(
const ::bts::api::analytics::Report &report_arg,
const api::Response<::bts::api::ResultMessage> &resp) {
Report report = report_arg;
std::string key = std::to_string(report.id());
auto [present, rowid, val] =
me->reports_store->retrieve_one(me->reports_ns, key);
if (!present) {
int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1;
report.set_id(new_id);
} else {
bts_assert(!val.empty());
}
std::string report_str = utils::pb::to_string(report);
add_or_update_to_table(
me->reports_store.get(), me->reports_ns, std::to_string(report.id()),
report_str);
me->reports_publisher.add_or_update(report);
me->reports_publisher.flush();
api::ResultMessage res;
res.set_success(true);
resp(res);
}
void AnalyticsServiceServer::delete_report(
const bts::api::analytics::IntWrapper &id,
const api::Response<::bts::api::ResultMessage> &resp) {
std::string key = std::to_string(id.data());
auto [present, rowid, val] =
me->reports_store->retrieve_one(me->reports_ns, key);
api::ResultMessage res;
res.set_success(false);
if (!present) {
res.set_error_message("Could not find report");
resp(res);
return;
}
bts_assert(!val.empty());
me->reports_publisher.erase(id.data());
me->reports_publisher.flush();
me->reports_store->remove(me->reports_ns, {key});
res.set_success(true);
resp(res);
}
void AnalyticsServiceServer::archive_theos(
const bts::api::analytics::ArchivedTheoList &theo_list,
const api::Response<::bts::api::ResultMessage> &resp) {
me->n_theos_archives++;
uint64_t t0 = utils::TimeStamp::now().to_uint64();
if (theo_list.sec_theo().empty()) {
std::string err_msg = "Empty archived theo list, call failed";
LOG(Gen, Error, "AnalyticsServiceServer::archive_theos {}", err_msg);
fire_cb(resp, false, &err_msg);
return;
}
const std::string &ns = me->archived_theos_ns;
const std::string key = std::to_string(theo_list.save_time());
std::string value = utils::pb::to_string(theo_list);
me->theos_store->insert(ns, key, value);
uint64_t t1 = utils::TimeStamp::now().to_uint64();
uint64_t t_diff = t1 - t0;
LOG(Gen, Info,
"AnalyticsServiceServer::archive_theos sec_tups: {} sec_theo: {} time: "
"{} ns",
theo_list.sec_tups().size(), theo_list.sec_theo().size(), t_diff);
if (me->n_theos_archives >= me->n_theos_archives_before_cleanup) {
me->n_theos_archives = 0;
_clean_up_archived_theos();
}
fire_cb(resp, true);
}
void AnalyticsServiceServer::get_archived_theos(
const bts::api::analytics::ArchivedTheoRetrieval &ret,
const bts::api::Response<bts::api::analytics::ArchivedTheoList> &resp) {
uint64_t t0 = utils::TimeStamp::now().to_uint64();
LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {}",
utils::pb::to_string(ret));
auto contains_st = [&](const ArchivedTheoList &tl) {
for (const auto &st : tl.sec_tups()) {
// TODO: compare all, just not symbol
if (st.symbol() == ret.sec_tup().symbol()) {
return true;
}
}
return false;
};
ArchivedTheoList rv = {};
const auto &rows = me->theos_store->retrieve_all();
uint64_t max_diff = std::numeric_limits<uint64_t>::max();
std::string key = "";
// this is rather slow, we can speed up by sorting the keys
// and searching based on distance from ret.save_time()
for (const auto &ns : rows) {
for (const auto &kv : ns.second) {
ArchivedTheoList tl;
utils::pb::from_string(kv.second, &tl);
if (contains_st(tl)) {
uint64_t ts = std::stoull(kv.first);
int64_t diff = ts - ret.save_time();
uint64_t abs_diff = std::abs(diff);
if (abs_diff < max_diff) {
key = kv.first;
max_diff = abs_diff;
rv = tl;
}
}
}
}
uint64_t t1 = utils::TimeStamp::now().to_uint64();
uint64_t t_diff = t1 - t0;
LOG(Gen, Info,
"AnalyticsServiceServer::get_archived_theos ret: {} key: {} time: {} ns",
utils::pb::to_string(ret), key, t_diff);
resp(rv);
}
AnalyticsServiceServer::~AnalyticsServiceServer() {}
// for a map where the key is the epoch timestamp, this returns the keys
// to delete such that final size is final_sz
std::set<std::string> get_keys_to_delete(
const std::map<std::string, std::string> &time_keyed_map, size_t final_sz) {
if (time_keyed_map.size() <= final_sz) {
return {};
}
std::vector<uint64_t> keys_uint;
for (const auto &kv : time_keyed_map) {
try {
keys_uint.push_back(std::stoull(kv.first));
} catch (int) {
LOG(Gen, Error,
"AnalyticsServiceServer::get_keys_to_delete could not "
"convert "
"key: {} to uint",
kv.first);
}
}
// this is sorted lowest to highest
std::sort(keys_uint.begin(), keys_uint.end());
size_t n_del_rows = keys_uint.size() - final_sz;
std::vector<uint64_t> keys_uint_to_del =
std::vector<uint64_t>(keys_uint.begin(), keys_uint.begin() + n_del_rows);
std::set<std::string> keys_to_del;
for (auto k : keys_uint_to_del) {
keys_to_del.insert(std::to_string(k));
}
return keys_to_del;
}
void AnalyticsServiceServer::_clean_up_vss() {
const size_t final_sz = me->vs_history_sz;
auto all_rows = me->vs_store->retrieve_all();
for (const auto &ns_k : all_rows) {
auto t0 = utils::TimeStamp::now().to_uint64();
const auto &vs = ns_k.first;
const auto &vs_row_map = ns_k.second;
auto keys_to_del = get_keys_to_delete(vs_row_map, final_sz);
me->vs_store->remove(vs, keys_to_del);
auto t1 = utils::TimeStamp::now().to_uint64();
LOG(Gen, Info,
"AnalyticsServiceServer::_clean_up_vss vs: {} n_rows: {} "
"n_dels {} time: {} ns",
vs, vs_row_map.size(), keys_to_del.size(), t1 - t0);
}
}
void AnalyticsServiceServer::_clean_up_archived_theos() {
uint64_t t0 = utils::TimeStamp::now().to_uint64();
std::map<std::string, std::map<std::string, std::string>> all_data =
me->theos_store->retrieve_all();
auto it = all_data.find(me->archived_theos_ns);
if (it == all_data.end())
return;
const std::map<std::string, std::string> &theo_data = it->second;
std::set<std::string> keys_to_del =
get_keys_to_delete(theo_data, me->archived_theos_history_sz);
me->theos_store->remove(me->archived_theos_ns, keys_to_del);
uint64_t t1 = utils::TimeStamp::now().to_uint64();
uint64_t t_diff = t1 - t0;
LOG(Gen, Info,
"AnalyticsServiceServer::_clean_up_archived_theos n_dels: {} "
"time: {} ns",
keys_to_del.size(), t_diff);
}
} // namespace bts::analytics