From abda2dbf97b9445f91f0a16704c8fc6c223d6fcc Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Tue, 4 Jul 2023 03:12:04 +0000 Subject: [PATCH] VolSurfaces and achived theo history size is not mantained at runtime --- src/lib/api/public/StatefulPublisher.h | 7 ++ .../analytics/AnalyticsServiceServer.cpp | 108 ++++++++++++++---- src/server/sparkd/cmds/md/cmds/Run.cpp | 2 + 3 files changed, 95 insertions(+), 22 deletions(-) diff --git a/src/lib/api/public/StatefulPublisher.h b/src/lib/api/public/StatefulPublisher.h index cd8711147..68ecdba60 100644 --- a/src/lib/api/public/StatefulPublisher.h +++ b/src/lib/api/public/StatefulPublisher.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace bts { namespace api { @@ -55,8 +56,14 @@ class StatefulPublisher { } } void erase(TKey m) { + // there is an assumption that we are not dealing with that within a + // transaction, add_or_update and erase is called just once a key if (_has_key(m)) { _erased.insert(m); + if (_keep_state) + _state_data.erase(m); + else + _state.erase(m); } } void flush() { diff --git a/src/server/sparkd/analytics/AnalyticsServiceServer.cpp b/src/server/sparkd/analytics/AnalyticsServiceServer.cpp index 65d5b857d..7cd0cc84a 100644 --- a/src/server/sparkd/analytics/AnalyticsServiceServer.cpp +++ b/src/server/sparkd/analytics/AnalyticsServiceServer.cpp @@ -93,9 +93,12 @@ struct AnalyticsServiceServer::PImpl { const std::string archived_theos_ns = "archived_theos"; // this is the total number of vs data that will be stored in the db, // beyond that, old values will be recycled - const size_t vs_history_sz = 5000; - // anologous to above for archived theos - const size_t archived_theos_history_sz = 1000; + const size_t vs_history_sz = 500; + // anologous to above for archived theos (however unlike the above this is + // recycled periodically, and is not published out) + const size_t archived_theos_history_sz = 5000; + const size_t n_theos_archives_before_cleanup = 100; + size_t n_theos_archives = 0; std::unique_ptr vs_store; std::unique_ptr exp_params_store; std::unique_ptr vs_usage_store; @@ -132,7 +135,11 @@ AnalyticsServiceServer::AnalyticsServiceServer( : AnalyticsServiceBase(context), me(new PImpl( context, - make_volsurfaces_stateful_publisher(), + bts::api::StatefulPublisher( + make_volsurfaces_publisher(), + [](const VolSurfaceEntry &v) { return v.save_time(); }, + true), + // make_volsurfaces_stateful_publisher(), make_exp_parameters_stateful_publisher(), make_volsurface_usage_stateful_publisher(), make_reports_stateful_publisher())) { @@ -144,13 +151,20 @@ AnalyticsServiceServer::AnalyticsServiceServer( for (auto const &ns_vals : vs_table) { const std::string &vs_name = ns_vals.first; const std::map &key_vals = ns_vals.second; - LOG(OrderMan, Info, "vs:{} entries:{}", vs_name, key_vals.size()); + LOG(OrderMan, Info, "vs: {} entries: {}", vs_name, key_vals.size()); for (const auto &kvp : key_vals) { const std::string &key = kvp.first; const std::string &val = kvp.second; - LOG(OrderMan, Diag, "vs:{} key:{} val:{}", vs_name, key, val); + LOG(OrderMan, Info, + "AnalyticsServiceServer::cons loaded from db vs: {} key: {}", + vs_name, key); VolSurfaceEntry vs_entry; utils::pb::from_string(val, &vs_entry); + LOG(Gen, Diag, + "AnalyticsServiceServer::cons loading to vs_publisher ns: {} " + "key: {} vs_name: {} vs_id: {} vs.save_time: {}", + vs_name, key, vs_entry.vs_name(), vs_entry.vs_id(), + vs_entry.save_time()); me->vs_publisher.add_or_update(vs_entry); } } @@ -220,6 +234,36 @@ void AnalyticsServiceServer::get_new_volsurface_id( void AnalyticsServiceServer::save_volsurface( const bts::api::analytics::VolSurfaceEntry &vs_entry_arg, const api::Response<::bts::api::ResultMessage> &response) { + auto t0 = utils::TimeStamp::now().to_uint64(); + auto remove_oldest = [&](const std::string &vs_ns) { + // return; + int vs_id = vs_entry_arg.vs_id(); + // find all the keys belonging to this vs + std::vector vs_keys = {}; + for (const auto &kv : me->vs_publisher.state_data()) { + if (kv.second.vs_id() == vs_id) { + vs_keys.push_back(kv.first); + } + } + LOG(Gen, Info, + "AnalyticsServiceServer::save_volsurface vs_id: {} n_entries: {}", + vs_entry_arg.vs_id(), vs_keys.size()); + if (vs_keys.size() <= me->vs_history_sz) + return; + // since we are already truncating it at the the start, we'll just + // find the earliest key and delete it          + uint64_t key_to_delete = std::numeric_limits::max(); + for (uint64_t k : vs_keys) { + if (k < key_to_delete) { + key_to_delete = k; + } + } + LOG(Gen, Info, + "AnalyticsServiceServer::save_volsurface removing row ns: {} key: {}", + vs_ns, key_to_delete); + me->vs_store->remove(vs_ns, {std::to_string(key_to_delete)}); + me->vs_publisher.erase(key_to_delete); + }; uint64_t ms_u = get_epoch(); VolSurfaceEntry vs_entry = vs_entry_arg; vs_entry.set_save_time(ms_u); @@ -233,9 +277,15 @@ void AnalyticsServiceServer::save_volsurface( std::string value = utils::pb::to_string(vs_entry); me->vs_store->insert(ns, key, value); me->vs_publisher.add_or_update(vs_entry); + remove_oldest(ns); me->vs_publisher.flush(); api::ResultMessage ret; ret.set_success(true); + auto t1 = utils::TimeStamp::now().to_uint64(); + LOG(Gen, Info, + "AnalyticsServiceServer::save_volsurface ns: {} key: {} n_entries: {} " + "time: {} ns", + ns, key, me->vs_publisher.state_data().size(), t1 - t0); response(ret); } @@ -377,9 +427,8 @@ void AnalyticsServiceServer::delete_report( void AnalyticsServiceServer::archive_theos( const bts::api::analytics::ArchivedTheoList &theo_list, const api::Response<::bts::api::ResultMessage> &resp) { - LOG(Gen, Info, - "AnalyticsServiceServer::archive_theos sec_tups: {} sec_theo: {}", - theo_list.sec_tups().size(), theo_list.sec_theo().size()); + me->n_theos_archives++; + uint64_t t0 = utils::TimeStamp::now().to_uint64(); if (theo_list.sec_theo().empty()) { std::string err_msg = "Empty archived theo list, call failed"; LOG(Gen, Error, "AnalyticsServiceServer::archive_theos {}", err_msg); @@ -390,12 +439,23 @@ void AnalyticsServiceServer::archive_theos( const std::string key = std::to_string(theo_list.save_time()); std::string value = utils::pb::to_string(theo_list); me->theos_store->insert(ns, key, value); + uint64_t t1 = utils::TimeStamp::now().to_uint64(); + uint64_t t_diff = t1 - t0; + LOG(Gen, Info, + "AnalyticsServiceServer::archive_theos sec_tups: {} sec_theo: {} time: " + "{} ns", + theo_list.sec_tups().size(), theo_list.sec_theo().size(), t_diff); + if (me->n_theos_archives >= me->n_theos_archives_before_cleanup) { + me->n_theos_archives = 0; + _clean_up_archived_theos(); + } fire_cb(resp, true); } void AnalyticsServiceServer::get_archived_theos( const bts::api::analytics::ArchivedTheoRetrieval &ret, const bts::api::Response &resp) { + uint64_t t0 = utils::TimeStamp::now().to_uint64(); LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {}", utils::pb::to_string(ret)); auto contains_st = [&](const ArchivedTheoList &tl) { @@ -429,15 +489,18 @@ void AnalyticsServiceServer::get_archived_theos( } } } - LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {} key: {}", - utils::pb::to_string(ret), key); + uint64_t t1 = utils::TimeStamp::now().to_uint64(); + uint64_t t_diff = t1 - t0; + LOG(Gen, Info, + "AnalyticsServiceServer::get_archived_theos ret: {} key: {} time: {} ns", + utils::pb::to_string(ret), key, t_diff); resp(rv); } AnalyticsServiceServer::~AnalyticsServiceServer() {} -// for a map where the key is the epoch timestamp, this returns the keys to -// delete such that final size is final_sz +// for a map where the key is the epoch timestamp, this returns the keys +// to delete such that final size is final_sz std::set get_keys_to_delete( const std::map &time_keyed_map, size_t final_sz) { if (time_keyed_map.size() <= final_sz) { @@ -449,7 +512,8 @@ std::set get_keys_to_delete( keys_uint.push_back(std::stoull(kv.first)); } catch (int) { LOG(Gen, Error, - "AnalyticsServiceServer::get_keys_to_delete could not convert " + "AnalyticsServiceServer::get_keys_to_delete could not " + "convert " "key: {} to uint", kv.first); } @@ -477,14 +541,14 @@ void AnalyticsServiceServer::_clean_up_vss() { me->vs_store->remove(vs, keys_to_del); auto t1 = utils::TimeStamp::now().to_uint64(); LOG(Gen, Info, - "AnalyticsServiceServer::_clean_up_vss vs: {} n_rows: {} n_dels {} " - "time: {} us", + "AnalyticsServiceServer::_clean_up_vss vs: {} n_rows: {} " + "n_dels {} time: {} ns", vs, vs_row_map.size(), keys_to_del.size(), t1 - t0); } } void AnalyticsServiceServer::_clean_up_archived_theos() { - auto t0 = utils::TimeStamp::now().to_uint64(); + uint64_t t0 = utils::TimeStamp::now().to_uint64(); std::map> all_data = me->theos_store->retrieve_all(); auto it = all_data.find(me->archived_theos_ns); @@ -494,11 +558,11 @@ void AnalyticsServiceServer::_clean_up_archived_theos() { std::set keys_to_del = get_keys_to_delete(theo_data, me->archived_theos_history_sz); me->theos_store->remove(me->archived_theos_ns, keys_to_del); - auto t1 = utils::TimeStamp::now().to_uint64(); + uint64_t t1 = utils::TimeStamp::now().to_uint64(); + uint64_t t_diff = t1 - t0; LOG(Gen, Info, - "AnalyticsServiceServer::_clean_up_archived_theos n_dels: {} time: {} " - "us", - keys_to_del.size(), t1 - t0); + "AnalyticsServiceServer::_clean_up_archived_theos n_dels: {} " + "time: {} ns", + keys_to_del.size(), t_diff); } - } // namespace bts::analytics diff --git a/src/server/sparkd/cmds/md/cmds/Run.cpp b/src/server/sparkd/cmds/md/cmds/Run.cpp index 6efe761af..2e6ac22a1 100644 --- a/src/server/sparkd/cmds/md/cmds/Run.cpp +++ b/src/server/sparkd/cmds/md/cmds/Run.cpp @@ -115,6 +115,7 @@ void Run::run( auto impl = [&mbo_writer, &tss_dispatcher, &type, &cleanup]() { LOG(md, Info, "Run::impl lambda t_id:{} t_name", os::get_thread_id(), get_thread_name()); + return; if (cleanup) md::cmds::Cleanup::execute(); if (type == Run::MDType::Configured) { @@ -148,6 +149,7 @@ void Run::run( threading::make_thread("Emumba", []() { run_emumba(); }).detach(); } if (bts::config::get().use_sim()) { + return; threading::make_thread("ExchangeSimGLBX", [&mbo_writer]() { sim::start_exchange_sim(api::Venue::VENUE_SIM, mbo_writer); }).detach();