VolSurfaces and achived theo history size is not mantained at runtime
This commit is contained in:
1 parent
ad1a45cf49
commit
abda2dbf97
3 files changed
+95
-22
No files matched your search
@@ -93,9 +93,12 @@ struct AnalyticsServiceServer::PImpl {
|
||||
const std::string archived_theos_ns = "archived_theos";
|
||||
// this is the total number of vs data that will be stored in the db,
|
||||
// beyond that, old values will be recycled
|
||||
const size_t vs_history_sz = 5000;
|
||||
// anologous to above for archived theos
|
||||
const size_t archived_theos_history_sz = 1000;
|
||||
const size_t vs_history_sz = 500;
|
||||
// anologous to above for archived theos (however unlike the above this is
|
||||
// recycled periodically, and is not published out)
|
||||
const size_t archived_theos_history_sz = 5000;
|
||||
const size_t n_theos_archives_before_cleanup = 100;
|
||||
size_t n_theos_archives = 0;
|
||||
std::unique_ptr<persistence::IStore> vs_store;
|
||||
std::unique_ptr<persistence::IStore> exp_params_store;
|
||||
std::unique_ptr<persistence::IStore> vs_usage_store;
|
||||
@@ -132,7 +135,11 @@ AnalyticsServiceServer::AnalyticsServiceServer(
|
||||
: AnalyticsServiceBase(context),
|
||||
me(new PImpl(
|
||||
context,
|
||||
make_volsurfaces_stateful_publisher(),
|
||||
bts::api::StatefulPublisher<VolSurfaceEntry, uint64_t>(
|
||||
make_volsurfaces_publisher(),
|
||||
[](const VolSurfaceEntry &v) { return v.save_time(); },
|
||||
true),
|
||||
// make_volsurfaces_stateful_publisher(),
|
||||
make_exp_parameters_stateful_publisher(),
|
||||
make_volsurface_usage_stateful_publisher(),
|
||||
make_reports_stateful_publisher())) {
|
||||
@@ -144,13 +151,20 @@ AnalyticsServiceServer::AnalyticsServiceServer(
|
||||
for (auto const &ns_vals : vs_table) {
|
||||
const std::string &vs_name = ns_vals.first;
|
||||
const std::map<std::string, std::string> &key_vals = ns_vals.second;
|
||||
LOG(OrderMan, Info, "vs:{} entries:{}", vs_name, key_vals.size());
|
||||
LOG(OrderMan, Info, "vs: {} entries: {}", vs_name, key_vals.size());
|
||||
for (const auto &kvp : key_vals) {
|
||||
const std::string &key = kvp.first;
|
||||
const std::string &val = kvp.second;
|
||||
LOG(OrderMan, Diag, "vs:{} key:{} val:{}", vs_name, key, val);
|
||||
LOG(OrderMan, Info,
|
||||
"AnalyticsServiceServer::cons loaded from db vs: {} key: {}",
|
||||
vs_name, key);
|
||||
VolSurfaceEntry vs_entry;
|
||||
utils::pb::from_string(val, &vs_entry);
|
||||
LOG(Gen, Diag,
|
||||
"AnalyticsServiceServer::cons loading to vs_publisher ns: {} "
|
||||
"key: {} vs_name: {} vs_id: {} vs.save_time: {}",
|
||||
vs_name, key, vs_entry.vs_name(), vs_entry.vs_id(),
|
||||
vs_entry.save_time());
|
||||
me->vs_publisher.add_or_update(vs_entry);
|
||||
}
|
||||
}
|
||||
@@ -220,6 +234,36 @@ void AnalyticsServiceServer::get_new_volsurface_id(
|
||||
void AnalyticsServiceServer::save_volsurface(
|
||||
const bts::api::analytics::VolSurfaceEntry &vs_entry_arg,
|
||||
const api::Response<::bts::api::ResultMessage> &response) {
|
||||
auto t0 = utils::TimeStamp::now().to_uint64();
|
||||
auto remove_oldest = [&](const std::string &vs_ns) {
|
||||
// return;
|
||||
int vs_id = vs_entry_arg.vs_id();
|
||||
// find all the keys belonging to this vs
|
||||
std::vector<uint64_t> vs_keys = {};
|
||||
for (const auto &kv : me->vs_publisher.state_data()) {
|
||||
if (kv.second.vs_id() == vs_id) {
|
||||
vs_keys.push_back(kv.first);
|
||||
}
|
||||
}
|
||||
LOG(Gen, Info,
|
||||
"AnalyticsServiceServer::save_volsurface vs_id: {} n_entries: {}",
|
||||
vs_entry_arg.vs_id(), vs_keys.size());
|
||||
if (vs_keys.size() <= me->vs_history_sz)
|
||||
return;
|
||||
// since we are already truncating it at the the start, we'll just
|
||||
// find the earliest key and delete it
|
||||
uint64_t key_to_delete = std::numeric_limits<uint64_t>::max();
|
||||
for (uint64_t k : vs_keys) {
|
||||
if (k < key_to_delete) {
|
||||
key_to_delete = k;
|
||||
}
|
||||
}
|
||||
LOG(Gen, Info,
|
||||
"AnalyticsServiceServer::save_volsurface removing row ns: {} key: {}",
|
||||
vs_ns, key_to_delete);
|
||||
me->vs_store->remove(vs_ns, {std::to_string(key_to_delete)});
|
||||
me->vs_publisher.erase(key_to_delete);
|
||||
};
|
||||
uint64_t ms_u = get_epoch();
|
||||
VolSurfaceEntry vs_entry = vs_entry_arg;
|
||||
vs_entry.set_save_time(ms_u);
|
||||
@@ -233,9 +277,15 @@ void AnalyticsServiceServer::save_volsurface(
|
||||
std::string value = utils::pb::to_string(vs_entry);
|
||||
me->vs_store->insert(ns, key, value);
|
||||
me->vs_publisher.add_or_update(vs_entry);
|
||||
remove_oldest(ns);
|
||||
me->vs_publisher.flush();
|
||||
api::ResultMessage ret;
|
||||
ret.set_success(true);
|
||||
auto t1 = utils::TimeStamp::now().to_uint64();
|
||||
LOG(Gen, Info,
|
||||
"AnalyticsServiceServer::save_volsurface ns: {} key: {} n_entries: {} "
|
||||
"time: {} ns",
|
||||
ns, key, me->vs_publisher.state_data().size(), t1 - t0);
|
||||
response(ret);
|
||||
}
|
||||
|
||||
@@ -377,9 +427,8 @@ void AnalyticsServiceServer::delete_report(
|
||||
void AnalyticsServiceServer::archive_theos(
|
||||
const bts::api::analytics::ArchivedTheoList &theo_list,
|
||||
const api::Response<::bts::api::ResultMessage> &resp) {
|
||||
LOG(Gen, Info,
|
||||
"AnalyticsServiceServer::archive_theos sec_tups: {} sec_theo: {}",
|
||||
theo_list.sec_tups().size(), theo_list.sec_theo().size());
|
||||
me->n_theos_archives++;
|
||||
uint64_t t0 = utils::TimeStamp::now().to_uint64();
|
||||
if (theo_list.sec_theo().empty()) {
|
||||
std::string err_msg = "Empty archived theo list, call failed";
|
||||
LOG(Gen, Error, "AnalyticsServiceServer::archive_theos {}", err_msg);
|
||||
@@ -390,12 +439,23 @@ void AnalyticsServiceServer::archive_theos(
|
||||
const std::string key = std::to_string(theo_list.save_time());
|
||||
std::string value = utils::pb::to_string(theo_list);
|
||||
me->theos_store->insert(ns, key, value);
|
||||
uint64_t t1 = utils::TimeStamp::now().to_uint64();
|
||||
uint64_t t_diff = t1 - t0;
|
||||
LOG(Gen, Info,
|
||||
"AnalyticsServiceServer::archive_theos sec_tups: {} sec_theo: {} time: "
|
||||
"{} ns",
|
||||
theo_list.sec_tups().size(), theo_list.sec_theo().size(), t_diff);
|
||||
if (me->n_theos_archives >= me->n_theos_archives_before_cleanup) {
|
||||
me->n_theos_archives = 0;
|
||||
_clean_up_archived_theos();
|
||||
}
|
||||
fire_cb(resp, true);
|
||||
}
|
||||
|
||||
void AnalyticsServiceServer::get_archived_theos(
|
||||
const bts::api::analytics::ArchivedTheoRetrieval &ret,
|
||||
const bts::api::Response<bts::api::analytics::ArchivedTheoList> &resp) {
|
||||
uint64_t t0 = utils::TimeStamp::now().to_uint64();
|
||||
LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {}",
|
||||
utils::pb::to_string(ret));
|
||||
auto contains_st = [&](const ArchivedTheoList &tl) {
|
||||
@@ -429,15 +489,18 @@ void AnalyticsServiceServer::get_archived_theos(
|
||||
}
|
||||
}
|
||||
}
|
||||
LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {} key: {}",
|
||||
utils::pb::to_string(ret), key);
|
||||
uint64_t t1 = utils::TimeStamp::now().to_uint64();
|
||||
uint64_t t_diff = t1 - t0;
|
||||
LOG(Gen, Info,
|
||||
"AnalyticsServiceServer::get_archived_theos ret: {} key: {} time: {} ns",
|
||||
utils::pb::to_string(ret), key, t_diff);
|
||||
resp(rv);
|
||||
}
|
||||
|
||||
AnalyticsServiceServer::~AnalyticsServiceServer() {}
|
||||
|
||||
// for a map where the key is the epoch timestamp, this returns the keys to
|
||||
// delete such that final size is final_sz
|
||||
// for a map where the key is the epoch timestamp, this returns the keys
|
||||
// to delete such that final size is final_sz
|
||||
std::set<std::string> get_keys_to_delete(
|
||||
const std::map<std::string, std::string> &time_keyed_map, size_t final_sz) {
|
||||
if (time_keyed_map.size() <= final_sz) {
|
||||
@@ -449,7 +512,8 @@ std::set<std::string> get_keys_to_delete(
|
||||
keys_uint.push_back(std::stoull(kv.first));
|
||||
} catch (int) {
|
||||
LOG(Gen, Error,
|
||||
"AnalyticsServiceServer::get_keys_to_delete could not convert "
|
||||
"AnalyticsServiceServer::get_keys_to_delete could not "
|
||||
"convert "
|
||||
"key: {} to uint",
|
||||
kv.first);
|
||||
}
|
||||
@@ -477,14 +541,14 @@ void AnalyticsServiceServer::_clean_up_vss() {
|
||||
me->vs_store->remove(vs, keys_to_del);
|
||||
auto t1 = utils::TimeStamp::now().to_uint64();
|
||||
LOG(Gen, Info,
|
||||
"AnalyticsServiceServer::_clean_up_vss vs: {} n_rows: {} n_dels {} "
|
||||
"time: {} us",
|
||||
"AnalyticsServiceServer::_clean_up_vss vs: {} n_rows: {} "
|
||||
"n_dels {} time: {} ns",
|
||||
vs, vs_row_map.size(), keys_to_del.size(), t1 - t0);
|
||||
}
|
||||
}
|
||||
|
||||
void AnalyticsServiceServer::_clean_up_archived_theos() {
|
||||
auto t0 = utils::TimeStamp::now().to_uint64();
|
||||
uint64_t t0 = utils::TimeStamp::now().to_uint64();
|
||||
std::map<std::string, std::map<std::string, std::string>> all_data =
|
||||
me->theos_store->retrieve_all();
|
||||
auto it = all_data.find(me->archived_theos_ns);
|
||||
@@ -494,11 +558,11 @@ void AnalyticsServiceServer::_clean_up_archived_theos() {
|
||||
std::set<std::string> keys_to_del =
|
||||
get_keys_to_delete(theo_data, me->archived_theos_history_sz);
|
||||
me->theos_store->remove(me->archived_theos_ns, keys_to_del);
|
||||
auto t1 = utils::TimeStamp::now().to_uint64();
|
||||
uint64_t t1 = utils::TimeStamp::now().to_uint64();
|
||||
uint64_t t_diff = t1 - t0;
|
||||
LOG(Gen, Info,
|
||||
"AnalyticsServiceServer::_clean_up_archived_theos n_dels: {} time: {} "
|
||||
"us",
|
||||
keys_to_del.size(), t1 - t0);
|
||||
"AnalyticsServiceServer::_clean_up_archived_theos n_dels: {} "
|
||||
"time: {} ns",
|
||||
keys_to_del.size(), t_diff);
|
||||
}
|
||||
|
||||
} // namespace bts::analytics
|
||||
Reference in new issue
Block a user