#include "AnalyticsServiceServer.h" #include "lib/api/apicore.pb.h" #include "lib/api/services/AnalyticsService.pb.h" #include "lib/api/services/AnalyticsService.server.pb.h" #include "lib/api/services/ShellLatencyService.pb.h" #include "lib/logging/logging.h" #include "lib/persistence/IStore.h" #include "lib/persistence/SqliteStore.h" #include "lib/utils/PBUtils.h" #include "lib/utils/Puid.h" #include "lib/utils/TimeStamp.h" #include "lib/utils/bts_assert.h" #include "server/sparkd/algo/AlgoDefinitionServiceServer.h" #include #include #include #include #include #include #include #include using namespace bts::api::analytics; namespace bts::analytics { uint64_t get_epoch() { using namespace std::chrono; milliseconds ms = duration_cast(system_clock::now().time_since_epoch()); uint64_t ms_u = ms.count(); return ms_u; } // assumes key are ints int get_max_id(IStore *const store, const std::string &ns) { auto ns_map = store->retrieve_all(); auto it = ns_map.find(ns); if (it == ns_map.end()) { return 0; } int max_id = -1; for (const auto &kv : it->second) { int key = std::stoi(kv.first); max_id = key > max_id ? key : max_id; } return max_id; } void add_or_update_to_table( persistence::IStore *store, const std::string &ns, const std::string &key, const std::string &val) { auto [exists, rowid, existing_val] = store->retrieve_one(ns, key); LOG(Gen, Info, "add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns, key, val, exists, existing_val); if (exists) { store->update(ns, key, val); } else { store->insert(ns, key, val); } } // parses out the vs_id and vs_name from the key std::tuple parse_vs_id_name( const std::string &ns, const std::string &sep_str) { std::vector parts; boost::split(parts, ns, boost::is_any_of(sep_str)); bts_assert(parts.size() == 3); return {std::stoi(parts[1]), parts[2]}; } void fire_cb( const api::Response<::bts::api::ResultMessage> &resp, bool is_success, std::string const *msg = nullptr) { api::ResultMessage res; res.set_success(is_success); if (msg != nullptr) { res.set_error_message(*msg); } resp(res); } struct AnalyticsServiceServer::PImpl { bts::api::ServiceContext &context; const std::string vs_ns_sep = "."; const std::string exp_params_ns = "analytics_expiration_params"; const std::string vs_usage_ns = "volsurface_usage"; const std::string reports_ns = "reports"; const std::string archived_theos_ns = "archived_theos"; // this is the total number of vs data that will be stored in the db, // beyond that, old values will be recycled const size_t vs_history_sz = 500; // anologous to above for archived theos (however unlike the above this is // recycled periodically, and is not published out) const size_t archived_theos_history_sz = 5000; const size_t n_theos_archives_before_cleanup = 100; size_t n_theos_archives = 0; std::unique_ptr vs_store; std::unique_ptr exp_params_store; std::unique_ptr vs_usage_store; std::unique_ptr reports_store; std::unique_ptr theos_store; AnalyticsServiceBase::VolsurfacesStatefulPublisher vs_publisher; AnalyticsServiceBase::Exp_parametersStatefulPublisher exp_params_publisher; AnalyticsServiceBase::Volsurface_usageStatefulPublisher vs_usage_publisher; AnalyticsServiceBase::ReportsStatefulPublisher reports_publisher; PImpl( bts::api::ServiceContext &context, const AnalyticsServiceBase::VolsurfacesStatefulPublisher &vs_publisher, const AnalyticsServiceBase::Exp_parametersStatefulPublisher &exp_params_publisher, const AnalyticsServiceBase::Volsurface_usageStatefulPublisher &vs_usage_publisher, const AnalyticsServiceBase::ReportsStatefulPublisher &reports_publisher) : context(context), vs_store(std::make_unique("volsurfaces")), exp_params_store(std::make_unique( "analytics_expiration_params")), vs_usage_store( std::make_unique("volsurface_usage")), reports_store(std::make_unique("reports")), theos_store( std::make_unique("archived_theos")), vs_publisher(vs_publisher), exp_params_publisher(exp_params_publisher), vs_usage_publisher(vs_usage_publisher), reports_publisher(reports_publisher) {} }; AnalyticsServiceServer::AnalyticsServiceServer( bts::api::ServiceContext &context) : AnalyticsServiceBase(context), me(new PImpl( context, bts::api::StatefulPublisher( make_volsurfaces_publisher(), [](const VolSurfaceEntry &v) { return v.save_time(); }, true), // make_volsurfaces_stateful_publisher(), make_exp_parameters_stateful_publisher(), make_volsurface_usage_stateful_publisher(), make_reports_stateful_publisher())) { // publishing all vol surfaces LOG(Gen, Info, "Starting AnalyticsServiceServer"); _clean_up_vss(); _clean_up_archived_theos(); auto vs_table = me->vs_store->retrieve_all(); for (auto const &ns_vals : vs_table) { const std::string &vs_name = ns_vals.first; const std::map &key_vals = ns_vals.second; LOG(OrderMan, Info, "vs: {} entries: {}", vs_name, key_vals.size()); for (const auto &kvp : key_vals) { const std::string &key = kvp.first; const std::string &val = kvp.second; LOG(OrderMan, Info, "AnalyticsServiceServer::cons loaded from db vs: {} key: {}", vs_name, key); VolSurfaceEntry vs_entry; utils::pb::from_string(val, &vs_entry); LOG(Gen, Diag, "AnalyticsServiceServer::cons loading to vs_publisher ns: {} " "key: {} vs_name: {} vs_id: {} vs.save_time: {}", vs_name, key, vs_entry.vs_name(), vs_entry.vs_id(), vs_entry.save_time()); me->vs_publisher.add_or_update(vs_entry); } } me->vs_publisher.flush(); // exp params auto exp_params_table = me->exp_params_store->retrieve_all(); bts_assert(exp_params_table.size() == 0 || exp_params_table.size() == 1); for (const auto &ns_vals : exp_params_table) { const std::map &key_vals = ns_vals.second; bts_assert(key_vals.size() == 0 || key_vals.size() == 1); for (const auto &kvp : key_vals) { ExpirationParameterEntryList exp_params; utils::pb::from_string(kvp.second, &exp_params); me->exp_params_publisher.add_or_update(exp_params); } } me->exp_params_publisher.flush(); // vol surface usage auto vs_usage_table = me->vs_usage_store->retrieve_all(); bts_assert(vs_usage_table.size() == 0 || vs_usage_table.size() == 1); for (const auto &ns_vals : vs_usage_table) { const std::map &key_vals = ns_vals.second; bts_assert(key_vals.size() == 0 || key_vals.size() == 1); for (const auto &kvp : key_vals) { VolSurfaceUsageList vs_usage_list; utils::pb::from_string(kvp.second, &vs_usage_list); me->vs_usage_publisher.add_or_update(vs_usage_list); } } me->vs_usage_publisher.flush(); // reports { auto ns_table = me->reports_store->retrieve_all(); bts_assert(ns_table.size() == 0 || ns_table.size() == 1); for (const auto &ns_vals : ns_table) { for (const auto &kvp : ns_vals.second) { Report report; utils::pb::from_string(kvp.second, &report); me->reports_publisher.add_or_update(report); } } me->reports_publisher.flush(); } } void AnalyticsServiceServer::get_new_volsurface_id( const ::bts::api::VoidMessage &, const bts::api::Response<::bts::api::analytics::AnalyticsServiceCreateReply> &response) { auto all = me->vs_store->retrieve_all(); int max_id = 0; std::string curr_vs_name = ""; for (const auto &ns_vals : all) { const std::string &ns = ns_vals.first; auto [vs_id, vs_name] = parse_vs_id_name(ns, me->vs_ns_sep); if (vs_id >= max_id) { max_id = vs_id; curr_vs_name = vs_name; } } bts::api::analytics::AnalyticsServiceCreateReply rep; rep.set_ok(true); rep.set_new_id(max_id + 1); response(rep); } void AnalyticsServiceServer::save_volsurface( const bts::api::analytics::VolSurfaceEntry &vs_entry_arg, const api::Response<::bts::api::ResultMessage> &response) { auto t0 = utils::TimeStamp::now().to_uint64(); auto remove_oldest = [&](const std::string &vs_ns) { // return; int vs_id = vs_entry_arg.vs_id(); // find all the keys belonging to this vs std::vector vs_keys = {}; for (const auto &kv : me->vs_publisher.state_data()) { if (kv.second.vs_id() == vs_id) { vs_keys.push_back(kv.first); } } LOG(Gen, Info, "AnalyticsServiceServer::save_volsurface vs_id: {} n_entries: {}", vs_entry_arg.vs_id(), vs_keys.size()); if (vs_keys.size() <= me->vs_history_sz) return; // since we are already truncating it at the the start, we'll just // find the earliest key and delete it          uint64_t key_to_delete = std::numeric_limits::max(); for (uint64_t k : vs_keys) { if (k < key_to_delete) { key_to_delete = k; } } LOG(Gen, Info, "AnalyticsServiceServer::save_volsurface removing row ns: {} key: {}", vs_ns, key_to_delete); me->vs_store->remove(vs_ns, {std::to_string(key_to_delete)}); me->vs_publisher.erase(key_to_delete); }; uint64_t ms_u = get_epoch(); VolSurfaceEntry vs_entry = vs_entry_arg; vs_entry.set_save_time(ms_u); // namespace: volsurface__ // key: save_time epoch // value: VolSurfaceEntry std::string ns = "volsurface" + me->vs_ns_sep + std::to_string(vs_entry.vs_id()) + me->vs_ns_sep + vs_entry.vs_name(); std::string key = std::to_string(ms_u); std::string value = utils::pb::to_string(vs_entry); me->vs_store->insert(ns, key, value); me->vs_publisher.add_or_update(vs_entry); remove_oldest(ns); me->vs_publisher.flush(); api::ResultMessage ret; ret.set_success(true); auto t1 = utils::TimeStamp::now().to_uint64(); LOG(Gen, Info, "AnalyticsServiceServer::save_volsurface ns: {} key: {} n_entries: {} " "time: {} ns", ns, key, me->vs_publisher.state_data().size(), t1 - t0); response(ret); } void AnalyticsServiceServer::delete_volsurface( const bts::api::analytics::IntWrapper &vs_id_w, const api::Response<::bts::api::ResultMessage> &response) { auto vs_id = vs_id_w.data(); auto vs_vals = me->vs_store->retrieve_all(); auto ns_substr = "volsurface." + std::to_string(vs_id) + "."; size_t n_elems_removed = 0; for (const auto &kv : vs_vals) { const auto &ns = kv.first; if (ns.find(ns_substr) != std::string::npos) { n_elems_removed = kv.second.size(); // this is the ns we want to delete // first delete it from the live stream for (const auto &kv_inside : kv.second) { const auto &key_str = kv_inside.first; uint64_t key = std::stoull(key_str); me->vs_publisher.erase(key); } me->vs_publisher.flush(); me->vs_store->remove_all(ns); } } LOG(Gen, Info, "AnalyticsServiceServer::delete_volsurface entries removed: {}", n_elems_removed); api::ResultMessage res; if (n_elems_removed > 0) { res.set_success(true); } else { res.set_success(false); res.set_error_message("Could not find any entries"); } response(res); } void AnalyticsServiceServer::clean_up( const ::bts::api::analytics::IntWrapper &, const api::Response<::bts::api::VoidMessage> &) { // TODO } void AnalyticsServiceServer::set_exp_parameters( const bts::api::analytics::ExpirationParameterEntryList &exp_params_args, const api::Response<::bts::api::ResultMessage> &response) { LOG(Gen, Info, "AnalyticsServiceServer::set_exp_parameters params: {}", exp_params_args.param_list().size()); if (exp_params_args.param_list().empty()) { std::string err_msg = "Empty params list, call failed"; LOG(Gen, Error, "AnalyticsServiceServer::set_exp_parameters {}", err_msg); fire_cb(response, false, &err_msg); return; } ExpirationParameterEntryList exp_params = exp_params_args; exp_params.set_id(1); std::string ns = me->exp_params_ns; std::string key = std::to_string(1); std::string value = utils::pb::to_string(exp_params); add_or_update_to_table(me->exp_params_store.get(), ns, key, value); me->exp_params_publisher.add_or_update(exp_params); me->exp_params_publisher.flush(); fire_cb(response, true); } void AnalyticsServiceServer::set_volsurface_usage_list( const ::bts::api::analytics::VolSurfaceUsageList &vs_usage_list_args, const api::Response<::bts::api::ResultMessage> &response) { VolSurfaceUsageList vs_usage_list = vs_usage_list_args; vs_usage_list.set_id(1); std::string ns = me->vs_usage_ns; std::string key = std::to_string(1); std::string value = utils::pb::to_string(vs_usage_list); me->vs_usage_store->update(ns, key, value); me->vs_usage_publisher.add_or_update(vs_usage_list); me->vs_usage_publisher.flush(); api::ResultMessage res; res.set_success(true); response(res); } void AnalyticsServiceServer::get_new_report_id( const ::bts::api::VoidMessage &, const bts::api::Response<::bts::api::analytics::AnalyticsServiceCreateReply> &response) { auto max_id = get_max_id(me->reports_store.get(), me->reports_ns); auto new_id = max_id + 1; bts::api::analytics::AnalyticsServiceCreateReply rep; rep.set_ok(true); rep.set_new_id(new_id); response(rep); } void AnalyticsServiceServer::save_report( const ::bts::api::analytics::Report &report_arg, const api::Response<::bts::api::ResultMessage> &resp) { Report report = report_arg; std::string key = std::to_string(report.id()); auto [present, rowid, val] = me->reports_store->retrieve_one(me->reports_ns, key); if (!present) { int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1; report.set_id(new_id); } else { bts_assert(!val.empty()); } std::string report_str = utils::pb::to_string(report); add_or_update_to_table( me->reports_store.get(), me->reports_ns, std::to_string(report.id()), report_str); me->reports_publisher.add_or_update(report); me->reports_publisher.flush(); api::ResultMessage res; res.set_success(true); resp(res); } void AnalyticsServiceServer::delete_report( const bts::api::analytics::IntWrapper &id, const api::Response<::bts::api::ResultMessage> &resp) { std::string key = std::to_string(id.data()); auto [present, rowid, val] = me->reports_store->retrieve_one(me->reports_ns, key); api::ResultMessage res; res.set_success(false); if (!present) { res.set_error_message("Could not find report"); resp(res); return; } bts_assert(!val.empty()); me->reports_publisher.erase(id.data()); me->reports_publisher.flush(); me->reports_store->remove(me->reports_ns, {key}); res.set_success(true); resp(res); } void AnalyticsServiceServer::archive_theos( const bts::api::analytics::ArchivedTheoList &theo_list, const api::Response<::bts::api::ResultMessage> &resp) { me->n_theos_archives++; uint64_t t0 = utils::TimeStamp::now().to_uint64(); if (theo_list.sec_theo().empty()) { std::string err_msg = "Empty archived theo list, call failed"; LOG(Gen, Error, "AnalyticsServiceServer::archive_theos {}", err_msg); fire_cb(resp, false, &err_msg); return; } const std::string &ns = me->archived_theos_ns; const std::string key = std::to_string(theo_list.save_time()); std::string value = utils::pb::to_string(theo_list); me->theos_store->insert(ns, key, value); uint64_t t1 = utils::TimeStamp::now().to_uint64(); uint64_t t_diff = t1 - t0; LOG(Gen, Info, "AnalyticsServiceServer::archive_theos sec_tups: {} sec_theo: {} time: " "{} ns", theo_list.sec_tups().size(), theo_list.sec_theo().size(), t_diff); if (me->n_theos_archives >= me->n_theos_archives_before_cleanup) { me->n_theos_archives = 0; _clean_up_archived_theos(); } fire_cb(resp, true); } void AnalyticsServiceServer::get_archived_theos( const bts::api::analytics::ArchivedTheoRetrieval &ret, const bts::api::Response &resp) { uint64_t t0 = utils::TimeStamp::now().to_uint64(); LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {}", utils::pb::to_string(ret)); auto contains_st = [&](const ArchivedTheoList &tl) { for (const auto &st : tl.sec_tups()) { // TODO: compare all, just not symbol if (st.symbol() == ret.sec_tup().symbol()) { return true; } } return false; }; ArchivedTheoList rv = {}; const auto &rows = me->theos_store->retrieve_all(); uint64_t max_diff = std::numeric_limits::max(); std::string key = ""; // this is rather slow, we can speed up by sorting the keys // and searching based on distance from ret.save_time() for (const auto &ns : rows) { for (const auto &kv : ns.second) { ArchivedTheoList tl; utils::pb::from_string(kv.second, &tl); if (contains_st(tl)) { uint64_t ts = std::stoull(kv.first); int64_t diff = ts - ret.save_time(); uint64_t abs_diff = std::abs(diff); if (abs_diff < max_diff) { key = kv.first; max_diff = abs_diff; rv = tl; } } } } uint64_t t1 = utils::TimeStamp::now().to_uint64(); uint64_t t_diff = t1 - t0; LOG(Gen, Info, "AnalyticsServiceServer::get_archived_theos ret: {} key: {} time: {} ns", utils::pb::to_string(ret), key, t_diff); resp(rv); } AnalyticsServiceServer::~AnalyticsServiceServer() {} // for a map where the key is the epoch timestamp, this returns the keys // to delete such that final size is final_sz std::set get_keys_to_delete( const std::map &time_keyed_map, size_t final_sz) { if (time_keyed_map.size() <= final_sz) { return {}; } std::vector keys_uint; for (const auto &kv : time_keyed_map) { try { keys_uint.push_back(std::stoull(kv.first)); } catch (int) { LOG(Gen, Error, "AnalyticsServiceServer::get_keys_to_delete could not " "convert " "key: {} to uint", kv.first); } } // this is sorted lowest to highest std::sort(keys_uint.begin(), keys_uint.end()); size_t n_del_rows = keys_uint.size() - final_sz; std::vector keys_uint_to_del = std::vector(keys_uint.begin(), keys_uint.begin() + n_del_rows); std::set keys_to_del; for (auto k : keys_uint_to_del) { keys_to_del.insert(std::to_string(k)); } return keys_to_del; } void AnalyticsServiceServer::_clean_up_vss() { const size_t final_sz = me->vs_history_sz; auto all_rows = me->vs_store->retrieve_all(); for (const auto &ns_k : all_rows) { auto t0 = utils::TimeStamp::now().to_uint64(); const auto &vs = ns_k.first; const auto &vs_row_map = ns_k.second; auto keys_to_del = get_keys_to_delete(vs_row_map, final_sz); me->vs_store->remove(vs, keys_to_del); auto t1 = utils::TimeStamp::now().to_uint64(); LOG(Gen, Info, "AnalyticsServiceServer::_clean_up_vss vs: {} n_rows: {} " "n_dels {} time: {} ns", vs, vs_row_map.size(), keys_to_del.size(), t1 - t0); } } void AnalyticsServiceServer::_clean_up_archived_theos() { uint64_t t0 = utils::TimeStamp::now().to_uint64(); std::map> all_data = me->theos_store->retrieve_all(); auto it = all_data.find(me->archived_theos_ns); if (it == all_data.end()) return; const std::map &theo_data = it->second; std::set keys_to_del = get_keys_to_delete(theo_data, me->archived_theos_history_sz); me->theos_store->remove(me->archived_theos_ns, keys_to_del); uint64_t t1 = utils::TimeStamp::now().to_uint64(); uint64_t t_diff = t1 - t0; LOG(Gen, Info, "AnalyticsServiceServer::_clean_up_archived_theos n_dels: {} " "time: {} ns", keys_to_del.size(), t_diff); } } // namespace bts::analytics