#include "lib/tracing/PerfLog.h" #include "lib/config/config.h" #include "lib/config/protos/perflog_cfg.pb.h" #include "lib/config/protos/system.pb.h" #include "lib/persistence/persistence_utils.h" #include "lib/utils/bts_array_size.h" #include "lib/utils/datetime.h" #include "lib/utils/stats.h" #include "lib/utils/thread_manager.h" #include #include #include #include #include #include #include #include #include #include namespace bts { namespace trace { const char *PerfLog::get_thread_name() { char *thread_name_buffer = new char[17]; pthread_getname_np(pthread_self(), thread_name_buffer, 16); return thread_name_buffer; } thread_local char const *PerfLog::_thread_name_ptr = get_thread_name(); auto insert_trace = "insert into trace (" "session_start," "ts," "hts," "channel," "entry_kind," "txt_1," "uint64_1," "thread," "cor_event," "sn," "uint32_1" ") values (" ":session_start," ":ts," ":hts," ":channel," ":entry_kind," ":txt_1," ":uint64_1," ":thread," ":cor_event," ":sn," ":uint32_1" ")"; auto insert_session = "insert into session (" "version," "host," "build_type," "session_start" ") values (" ":version," ":host," ":build_type," ":session_start" ")"; auto requote_query = "select * from requote_trace order by trace_time desc limit ?"; auto hedge_query = "select * from hedge_trace order by trace_time desc limit ?"; struct PerfLog::PImpl { PImpl(std::string const &loc) : db(loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE), insert_trace_statement(), insert_session_statement(), requote_query_statement(), hedge_query_statement(), session_start(), entry_batch(config::get().entry_batch_size()), lock() {} SQLite::Database db; std::unique_ptr insert_trace_statement; std::unique_ptr insert_session_statement; std::unique_ptr requote_query_statement; std::unique_ptr hedge_query_statement; std::string session_start; boost::circular_buffer entry_batch; std::mutex lock; std::thread init_thread; std::atomic init_completed; }; PerfLog::PerfLog() : _pending(), _disabled(false), me() { _disabled = !config::get().active(); if (_disabled) return; auto loc = init_storage_location(); if (loc.empty()) { _disabled = true; return; } try { me = std::make_unique(loc); } catch (SQLite::Exception const &e) { LOG(persist, Error, "Error initializing perf_trace DB {}. Error: {}", loc, std::string(e.getErrorStr())); _disabled = true; return; } me->init_completed = false; me->init_thread = threading::make_thread("perf_log_ini", [loc, this]() { try { LOG(persist, Info, "Started initialization of perf_log db"); init_db(); me->insert_trace_statement = std::make_unique(me->db, insert_trace); me->insert_session_statement = std::make_unique(me->db, insert_session); me->requote_query_statement = std::make_unique(me->db, requote_query); me->hedge_query_statement = std::make_unique(me->db, hedge_query); me->session_start = datetime::FIX_timestamp(datetime::local_now()); me->insert_session_statement->bind( ":version", config::get().version()); me->insert_session_statement->bind( ":host", config::get().host_name()); #ifdef DEBUG me->insert_session_statement->bind(":build_type", "debug"); #else me->insert_session_statement->bind(":build_type", "release"); #endif me->insert_session_statement->bind( ":session_start", me->session_start); me->insert_session_statement->exec(); me->insert_session_statement->reset(); } catch (SQLite::Exception const &e) { LOG(persist, Error, "Error initializing perf_trace DB (2) {}. Error: {}", loc, std::string(e.getErrorStr())); _disabled = true; } LOG(persist, Info, "Finished initialization of perf_log db"); me->init_completed = true; }); } PerfLog::~PerfLog() { me->init_thread.join(); } std::string PerfLog::init_storage_location() { auto loc = persistence::storage_location("perf_log"); auto parent = boost::filesystem::path(loc).parent_path(); if (!boost::filesystem::is_directory(parent)) { if (boost::filesystem::create_directories(parent)) return loc; else return ""; } else return loc; } size_t PerfLog::store_log_entries() { if (!me->init_completed || _disabled) { _pending.clear(); return 0; } PerfLogEntry ev; size_t count = 0; while (_pending.try_pop(ev)) { if (is_flush_signal(ev)) count += flush_current_batch_to_disk(); else me->entry_batch.push_front(ev); } return count; } void PerfLog::init_db() { auto sql_path = os::sparkd_wd() + "/data_files/perf_trace/perf_log.sql"; std::ifstream f(sql_path); std::string str( (std::istreambuf_iterator(f)), (std::istreambuf_iterator())); config::PerfLogCfg defaults; boost::replace_all( str, std::to_string(defaults.max_rows()), std::to_string(config::get().max_rows())); me->db.exec(str); } std::string encode_string(SQLite::Column const &c) { return fmt::format("\"{}\"", std::string(c)); } std::string encode_number(SQLite::Column const &c) { return fmt::format("{}", (long long)c); } std::vector> PerfLog::get_latest_requote_traces(size_t n) { std::vector> result; if (!me->init_completed) return result; std::lock_guard l(me->lock); try { me->requote_query_statement->bind(1, (long long)n); while (me->requote_query_statement->executeStep()) { auto &row = result.emplace_back(); row["version"] = encode_string(me->requote_query_statement->getColumn("version")); row["host"] = encode_string(me->requote_query_statement->getColumn("host")); row["build_type"] = encode_string( me->requote_query_statement->getColumn("build_type")); row["session_start"] = encode_string( me->requote_query_statement->getColumn("session_start")); row["trace_time"] = encode_string( me->requote_query_statement->getColumn("trace_time")); row["order_key"] = encode_number(me->requote_query_statement->getColumn("order_key")); row["total"] = encode_number(me->requote_query_statement->getColumn("total")); } me->requote_query_statement->reset(); } catch (SQLite::Exception const &e) { LOG(persist, Error, "Unable to retrieve quote traces {}. Error: {}", std::string(me->db.getFilename()), std::string(e.getErrorStr())); } return result; } std::vector> PerfLog::get_latest_hedge_traces(size_t n) { std::vector> result; if (!me->init_completed) return result; std::lock_guard l(me->lock); try { me->hedge_query_statement->bind(1, (long long)n); while (me->hedge_query_statement->executeStep()) { auto &row = result.emplace_back(); row["version"] = encode_string(me->hedge_query_statement->getColumn("version")); row["host"] = encode_string(me->hedge_query_statement->getColumn("host")); row["build_type"] = encode_string(me->hedge_query_statement->getColumn("build_type")); row["session_start"] = encode_string( me->hedge_query_statement->getColumn("session_start")); row["trace_time"] = encode_string(me->hedge_query_statement->getColumn("trace_time")); row["order_key"] = encode_number(me->hedge_query_statement->getColumn("order_key")); row["total"] = encode_number(me->hedge_query_statement->getColumn("total")); } me->hedge_query_statement->reset(); } catch (SQLite::Exception const &e) { LOG(persist, Error, "Unable to retrieve hedge traces. Error: {}", std::string(e.getErrorStr())); } return result; } bool PerfLog::is_flush_signal(PerfLogEntry const &ev) { return ev.entry_kind == nullptr; } size_t PerfLog::flush_current_batch_to_disk() { size_t count = 0; try { for (auto const &ev : me->entry_batch) { if (count == 0) me->db.exec("begin"); me->insert_trace_statement->bind(":session_start", me->session_start); me->insert_trace_statement->bind(":ts", (long long)ev.ts.to_uint64()); me->insert_trace_statement->bind( ":hts", datetime::FIX_timestamp(datetime::timestamp_to_ptime(ev.ts))); me->insert_trace_statement->bind(":channel", ev.channel); me->insert_trace_statement->bind(":entry_kind", ev.entry_kind); me->insert_trace_statement->bind(":txt_1", ev.txt_1); me->insert_trace_statement->bind(":sn", ev.sn); me->insert_trace_statement->bind(":uint32_1", ev.uint32_1); me->insert_trace_statement->bind(":uint64_1", (long long)ev.uint64_1); me->insert_trace_statement->bind( ":cor_event", (long long)ev.cor_event.to_uint64()); me->insert_trace_statement->bind(":thread", ev.thread_name); me->insert_trace_statement->exec(); count++; me->insert_trace_statement->reset(); } if (count) me->db.exec("commit"); } catch (SQLite::Exception const &e) { if (count) try { me->db.exec("rollback"); } catch (SQLite::Exception const &e2) { } LOG(persist, Error, "Unable to insert perf trace in. Error: {}", std::string(e.getErrorStr())); } me->entry_batch.clear(); return count; } namespace perf_stat { using Row = std::map; struct Summary { std::string name; double min; double max; double median; double ninenty_nine; double stddev; std::vector values; }; struct Session { std::string start; std::string host; std::string version; std::string build_type; uint32_t n_rows; std::map summaries; std::vector rows; }; } // namespace perf_stat std::map structure_quote_traces( std::vector> "es) { std::map result; for (auto &q : quotes) { auto &session = result[q["session_start"]]; if (session.start.empty()) { session.start = q["session_start"]; session.host = q["host"]; session.version = q["version"]; session.build_type = q["build_type"]; } for (auto st : {"total"}) { auto &summary = session.summaries[st]; if (summary.name.empty()) summary.name = st; summary.values.push_back(std::stod(q[st])); } session.rows.push_back(q); session.n_rows++; } for (auto &session_kvp : result) for (auto &stat_kvp : session_kvp.second.summaries) { auto &summary = stat_kvp.second; summary.min = stats::min(summary.values); summary.max = stats::max(summary.values); summary.median = stats::percentile(summary.values, 0.5); summary.ninenty_nine = stats::percentile(summary.values, 0.99); summary.stddev = stats::stdev(summary.values); } return result; } std::map structure_hedge_traces( std::vector> "es) { std::map result; for (auto &q : quotes) { auto &session = result[q["session_start"]]; if (session.start.empty()) { session.start = q["session_start"]; session.host = q["host"]; session.version = q["version"]; session.build_type = q["build_type"]; } for (auto st : {"total"}) { auto &summary = session.summaries[st]; if (summary.name.empty()) summary.name = st; summary.values.push_back(std::stod(q[st])); } session.rows.push_back(q); session.n_rows++; } for (auto &session_kvp : result) for (auto &stat_kvp : session_kvp.second.summaries) { auto &summary = stat_kvp.second; summary.min = stats::min(summary.values); summary.max = stats::max(summary.values); summary.median = stats::percentile(summary.values, 0.5); summary.ninenty_nine = stats::percentile(summary.values, 0.99); summary.stddev = stats::stdev(summary.values); } return result; } void print_summary(std::ostream &ss, perf_stat::Summary const &s, bool last) { ss << " { " << "\"value\":" << " \"" << s.name << "\"" << ", \"min\":" << s.min << ", \"max\":" << s.max << ", \"median\":" << s.median << ", \"ninenty_nine\":" << s.ninenty_nine << ", \"stdev\":" << s.stddev; if (!last) ss << " },\n"; else ss << " }\n"; } void print_row(std::ostream &ss, perf_stat::Row const &r, bool last) { ss << " { "; size_t i = 0; for (auto const &kvp : r) if (kvp.first != "version" && kvp.first != "host" && kvp.first != "build_type" && kvp.first != "session_start") { ss << "\"" << kvp.first << "\":" << kvp.second; if (++i != r.size() - 4) ss << ","; } if (!last) ss << " },\n"; else ss << " }\n"; } void print_session(std::ostream &ss, perf_stat::Session const &s, bool last) { ss << "{\n" << " \"session_name\": " << s.start << ",\n" << " \"host\": " << s.host << ",\n" << " \"version\": " << s.version << ",\n" << " \"build_type\": " << s.build_type << ",\n" << " \"n_rows\": " << s.n_rows << ",\n"; ss << " \"summaries\":\n [\n"; size_t i = 0; for (auto &summaries_kvp : s.summaries) print_summary(ss, summaries_kvp.second, ++i == s.summaries.size()); ss << " ],\n"; ss << " \"rows\":\n [\n"; i = 0; for (auto &r : s.rows) print_row(ss, r, ++i == s.rows.size()); ss << " ]\n"; if (!last) ss << "},\n"; else ss << "}\n"; } void PerfLog::print_latest_traces(std::ostream &ss, size_t n) { if (_disabled) return; { auto rows = get_latest_requote_traces(n); auto structured = structure_quote_traces(rows); ss << "{\n\"requotes\": [\n"; size_t i = 0; for (auto &session_kvp : boost::adaptors::reverse(structured)) print_session(ss, session_kvp.second, ++i == structured.size()); ss << "],\n"; } { auto rows = get_latest_hedge_traces(n); auto structured = structure_hedge_traces(rows); ss << "\"hedges\": [\n"; size_t i = 0; for (auto &session_kvp : boost::adaptors::reverse(structured)) print_session(ss, session_kvp.second, ++i == structured.size()); ss << "]\n}\n"; } } } // namespace trace } // namespace bts