VolSurfaces and achived theo history size is not mantained at runtime

This commit is contained in:
Taha Afzal committed 2023-07-04 03:13:46 +00:00
1 parent ad1a45cf49
commit abda2dbf97
3 files changed
+95 -22

No files matched your search

@@ -93,9 +93,12 @@ struct AnalyticsServiceServer::PImpl {
const std::string archived_theos_ns = "archived_theos";
// this is the total number of vs data that will be stored in the db,
// beyond that, old values will be recycled
const size_t vs_history_sz = 5000;
// anologous to above for archived theos
const size_t archived_theos_history_sz = 1000;
const size_t vs_history_sz = 500;
// anologous to above for archived theos (however unlike the above this is
// recycled periodically, and is not published out)
const size_t archived_theos_history_sz = 5000;
const size_t n_theos_archives_before_cleanup = 100;
size_t n_theos_archives = 0;
std::unique_ptr<persistence::IStore> vs_store;
std::unique_ptr<persistence::IStore> exp_params_store;
std::unique_ptr<persistence::IStore> vs_usage_store;
@@ -132,7 +135,11 @@ AnalyticsServiceServer::AnalyticsServiceServer(
: AnalyticsServiceBase(context),
me(new PImpl(
context,
make_volsurfaces_stateful_publisher(),
bts::api::StatefulPublisher<VolSurfaceEntry, uint64_t>(
make_volsurfaces_publisher(),
[](const VolSurfaceEntry &v) { return v.save_time(); },
true),
// make_volsurfaces_stateful_publisher(),
make_exp_parameters_stateful_publisher(),
make_volsurface_usage_stateful_publisher(),
make_reports_stateful_publisher())) {
@@ -144,13 +151,20 @@ AnalyticsServiceServer::AnalyticsServiceServer(
for (auto const &ns_vals : vs_table) {
const std::string &vs_name = ns_vals.first;
const std::map<std::string, std::string> &key_vals = ns_vals.second;
LOG(OrderMan, Info, "vs:{} entries:{}", vs_name, key_vals.size());
LOG(OrderMan, Info, "vs: {} entries: {}", vs_name, key_vals.size());
for (const auto &kvp : key_vals) {
const std::string &key = kvp.first;
const std::string &val = kvp.second;
LOG(OrderMan, Diag, "vs:{} key:{} val:{}", vs_name, key, val);
LOG(OrderMan, Info,
"AnalyticsServiceServer::cons loaded from db vs: {} key: {}",
vs_name, key);
VolSurfaceEntry vs_entry;
utils::pb::from_string(val, &vs_entry);
LOG(Gen, Diag,
"AnalyticsServiceServer::cons loading to vs_publisher ns: {} "
"key: {} vs_name: {} vs_id: {} vs.save_time: {}",
vs_name, key, vs_entry.vs_name(), vs_entry.vs_id(),
vs_entry.save_time());
me->vs_publisher.add_or_update(vs_entry);
}
}
@@ -220,6 +234,36 @@ void AnalyticsServiceServer::get_new_volsurface_id(
void AnalyticsServiceServer::save_volsurface(
const bts::api::analytics::VolSurfaceEntry &vs_entry_arg,
const api::Response<::bts::api::ResultMessage> &response) {
auto t0 = utils::TimeStamp::now().to_uint64();
auto remove_oldest = [&](const std::string &vs_ns) {
// return;
int vs_id = vs_entry_arg.vs_id();
// find all the keys belonging to this vs
std::vector<uint64_t> vs_keys = {};
for (const auto &kv : me->vs_publisher.state_data()) {
if (kv.second.vs_id() == vs_id) {
vs_keys.push_back(kv.first);
}
}
LOG(Gen, Info,
"AnalyticsServiceServer::save_volsurface vs_id: {} n_entries: {}",
vs_entry_arg.vs_id(), vs_keys.size());
if (vs_keys.size() <= me->vs_history_sz)
return;
// since we are already truncating it at the the start, we'll just
// find the earliest key and delete it         
uint64_t key_to_delete = std::numeric_limits<uint64_t>::max();
for (uint64_t k : vs_keys) {
if (k < key_to_delete) {
key_to_delete = k;
}
}
LOG(Gen, Info,
"AnalyticsServiceServer::save_volsurface removing row ns: {} key: {}",
vs_ns, key_to_delete);
me->vs_store->remove(vs_ns, {std::to_string(key_to_delete)});
me->vs_publisher.erase(key_to_delete);
};
uint64_t ms_u = get_epoch();
VolSurfaceEntry vs_entry = vs_entry_arg;
vs_entry.set_save_time(ms_u);
@@ -233,9 +277,15 @@ void AnalyticsServiceServer::save_volsurface(
std::string value = utils::pb::to_string(vs_entry);
me->vs_store->insert(ns, key, value);
me->vs_publisher.add_or_update(vs_entry);
remove_oldest(ns);
me->vs_publisher.flush();
api::ResultMessage ret;
ret.set_success(true);
auto t1 = utils::TimeStamp::now().to_uint64();
LOG(Gen, Info,
"AnalyticsServiceServer::save_volsurface ns: {} key: {} n_entries: {} "
"time: {} ns",
ns, key, me->vs_publisher.state_data().size(), t1 - t0);
response(ret);
}
@@ -377,9 +427,8 @@ void AnalyticsServiceServer::delete_report(
void AnalyticsServiceServer::archive_theos(
const bts::api::analytics::ArchivedTheoList &theo_list,
const api::Response<::bts::api::ResultMessage> &resp) {
LOG(Gen, Info,
"AnalyticsServiceServer::archive_theos sec_tups: {} sec_theo: {}",
theo_list.sec_tups().size(), theo_list.sec_theo().size());
me->n_theos_archives++;
uint64_t t0 = utils::TimeStamp::now().to_uint64();
if (theo_list.sec_theo().empty()) {
std::string err_msg = "Empty archived theo list, call failed";
LOG(Gen, Error, "AnalyticsServiceServer::archive_theos {}", err_msg);
@@ -390,12 +439,23 @@ void AnalyticsServiceServer::archive_theos(
const std::string key = std::to_string(theo_list.save_time());
std::string value = utils::pb::to_string(theo_list);
me->theos_store->insert(ns, key, value);
uint64_t t1 = utils::TimeStamp::now().to_uint64();
uint64_t t_diff = t1 - t0;
LOG(Gen, Info,
"AnalyticsServiceServer::archive_theos sec_tups: {} sec_theo: {} time: "
"{} ns",
theo_list.sec_tups().size(), theo_list.sec_theo().size(), t_diff);
if (me->n_theos_archives >= me->n_theos_archives_before_cleanup) {
me->n_theos_archives = 0;
_clean_up_archived_theos();
}
fire_cb(resp, true);
}
void AnalyticsServiceServer::get_archived_theos(
const bts::api::analytics::ArchivedTheoRetrieval &ret,
const bts::api::Response<bts::api::analytics::ArchivedTheoList> &resp) {
uint64_t t0 = utils::TimeStamp::now().to_uint64();
LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {}",
utils::pb::to_string(ret));
auto contains_st = [&](const ArchivedTheoList &tl) {
@@ -429,15 +489,18 @@ void AnalyticsServiceServer::get_archived_theos(
}
}
}
LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {} key: {}",
utils::pb::to_string(ret), key);
uint64_t t1 = utils::TimeStamp::now().to_uint64();
uint64_t t_diff = t1 - t0;
LOG(Gen, Info,
"AnalyticsServiceServer::get_archived_theos ret: {} key: {} time: {} ns",
utils::pb::to_string(ret), key, t_diff);
resp(rv);
}
AnalyticsServiceServer::~AnalyticsServiceServer() {}
// for a map where the key is the epoch timestamp, this returns the keys to
// delete such that final size is final_sz
// for a map where the key is the epoch timestamp, this returns the keys
// to delete such that final size is final_sz
std::set<std::string> get_keys_to_delete(
const std::map<std::string, std::string> &time_keyed_map, size_t final_sz) {
if (time_keyed_map.size() <= final_sz) {
@@ -449,7 +512,8 @@ std::set<std::string> get_keys_to_delete(
keys_uint.push_back(std::stoull(kv.first));
} catch (int) {
LOG(Gen, Error,
"AnalyticsServiceServer::get_keys_to_delete could not convert "
"AnalyticsServiceServer::get_keys_to_delete could not "
"convert "
"key: {} to uint",
kv.first);
}
@@ -477,14 +541,14 @@ void AnalyticsServiceServer::_clean_up_vss() {
me->vs_store->remove(vs, keys_to_del);
auto t1 = utils::TimeStamp::now().to_uint64();
LOG(Gen, Info,
"AnalyticsServiceServer::_clean_up_vss vs: {} n_rows: {} n_dels {} "
"time: {} us",
"AnalyticsServiceServer::_clean_up_vss vs: {} n_rows: {} "
"n_dels {} time: {} ns",
vs, vs_row_map.size(), keys_to_del.size(), t1 - t0);
}
}
void AnalyticsServiceServer::_clean_up_archived_theos() {
auto t0 = utils::TimeStamp::now().to_uint64();
uint64_t t0 = utils::TimeStamp::now().to_uint64();
std::map<std::string, std::map<std::string, std::string>> all_data =
me->theos_store->retrieve_all();
auto it = all_data.find(me->archived_theos_ns);
@@ -494,11 +558,11 @@ void AnalyticsServiceServer::_clean_up_archived_theos() {
std::set<std::string> keys_to_del =
get_keys_to_delete(theo_data, me->archived_theos_history_sz);
me->theos_store->remove(me->archived_theos_ns, keys_to_del);
auto t1 = utils::TimeStamp::now().to_uint64();
uint64_t t1 = utils::TimeStamp::now().to_uint64();
uint64_t t_diff = t1 - t0;
LOG(Gen, Info,
"AnalyticsServiceServer::_clean_up_archived_theos n_dels: {} time: {} "
"us",
keys_to_del.size(), t1 - t0);
"AnalyticsServiceServer::_clean_up_archived_theos n_dels: {} "
"time: {} ns",
keys_to_del.size(), t_diff);
}
} // namespace bts::analytics