509 lines
16 KiB
C++
509 lines
16 KiB
C++
#include "lib/tracing/PerfLog.h"
|
|
#include "lib/config/config.h"
|
|
#include "lib/config/protos/perflog_cfg.pb.h"
|
|
#include "lib/config/protos/system.pb.h"
|
|
#include "lib/persistence/persistence_utils.h"
|
|
#include "lib/utils/bts_array_size.h"
|
|
#include "lib/utils/datetime.h"
|
|
#include "lib/utils/stats.h"
|
|
#include "lib/utils/thread_manager.h"
|
|
|
|
#include <SQLiteCpp/SQLiteCpp.h>
|
|
#include <boost/algorithm/string/replace.hpp>
|
|
#include <boost/circular_buffer.hpp>
|
|
#include <boost/date_time/posix_time/ptime.hpp>
|
|
#include <boost/filesystem.hpp>
|
|
#include <boost/range/adaptor/reversed.hpp>
|
|
#include <fmt/format.h>
|
|
#include <mutex>
|
|
#include <pthread.h>
|
|
|
|
#include <thread>
|
|
|
|
namespace bts {
|
|
namespace trace {
|
|
|
|
const char *PerfLog::get_thread_name() {
|
|
char *thread_name_buffer = new char[17];
|
|
pthread_getname_np(pthread_self(), thread_name_buffer, 16);
|
|
return thread_name_buffer;
|
|
}
|
|
|
|
thread_local char const *PerfLog::_thread_name_ptr = get_thread_name();
|
|
|
|
auto insert_trace = "insert into trace ("
|
|
"session_start,"
|
|
"ts,"
|
|
"hts,"
|
|
"channel,"
|
|
"entry_kind,"
|
|
"txt_1,"
|
|
"uint64_1,"
|
|
"thread,"
|
|
"cor_event,"
|
|
"sn,"
|
|
"uint32_1"
|
|
") values ("
|
|
":session_start,"
|
|
":ts,"
|
|
":hts,"
|
|
":channel,"
|
|
":entry_kind,"
|
|
":txt_1,"
|
|
":uint64_1,"
|
|
":thread,"
|
|
":cor_event,"
|
|
":sn,"
|
|
":uint32_1"
|
|
")";
|
|
|
|
auto insert_session = "insert into session ("
|
|
"version,"
|
|
"host,"
|
|
"build_type,"
|
|
"session_start"
|
|
") values ("
|
|
":version,"
|
|
":host,"
|
|
":build_type,"
|
|
":session_start"
|
|
")";
|
|
|
|
auto requote_query =
|
|
"select * from requote_trace order by trace_time desc limit ?";
|
|
|
|
auto hedge_query = "select * from hedge_trace order by trace_time desc limit ?";
|
|
|
|
struct PerfLog::PImpl {
|
|
PImpl(std::string const &loc)
|
|
: db(loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE),
|
|
insert_trace_statement(), insert_session_statement(),
|
|
requote_query_statement(), hedge_query_statement(), session_start(),
|
|
entry_batch(config::get<config::PerfLogCfg>().entry_batch_size()),
|
|
lock() {}
|
|
|
|
SQLite::Database db;
|
|
std::unique_ptr<SQLite::Statement> insert_trace_statement;
|
|
std::unique_ptr<SQLite::Statement> insert_session_statement;
|
|
std::unique_ptr<SQLite::Statement> requote_query_statement;
|
|
std::unique_ptr<SQLite::Statement> hedge_query_statement;
|
|
std::string session_start;
|
|
boost::circular_buffer<PerfLogEntry> entry_batch;
|
|
std::mutex lock;
|
|
std::thread init_thread;
|
|
std::atomic<bool> init_completed;
|
|
};
|
|
|
|
PerfLog::PerfLog() : _pending(), _disabled(false), me() {
|
|
_disabled = !config::get<config::PerfLogCfg>().active();
|
|
|
|
if (_disabled)
|
|
return;
|
|
|
|
auto loc = init_storage_location();
|
|
|
|
if (loc.empty()) {
|
|
_disabled = true;
|
|
return;
|
|
}
|
|
|
|
try {
|
|
me = std::make_unique<PImpl>(loc);
|
|
} catch (SQLite::Exception const &e) {
|
|
LOG(persist, Error, "Error initializing perf_trace DB {}. Error: {}", loc,
|
|
std::string(e.getErrorStr()));
|
|
_disabled = true;
|
|
return;
|
|
}
|
|
|
|
me->init_completed = false;
|
|
me->init_thread = threading::make_thread("perf_log_ini", [loc, this]() {
|
|
try {
|
|
LOG(persist, Info, "Started initialization of perf_log db");
|
|
init_db();
|
|
|
|
me->insert_trace_statement =
|
|
std::make_unique<SQLite::Statement>(me->db, insert_trace);
|
|
me->insert_session_statement =
|
|
std::make_unique<SQLite::Statement>(me->db, insert_session);
|
|
me->requote_query_statement =
|
|
std::make_unique<SQLite::Statement>(me->db, requote_query);
|
|
me->hedge_query_statement =
|
|
std::make_unique<SQLite::Statement>(me->db, hedge_query);
|
|
me->session_start = datetime::FIX_timestamp(datetime::local_now());
|
|
|
|
me->insert_session_statement->bind(
|
|
":version", config::get<config::System>().version());
|
|
me->insert_session_statement->bind(
|
|
":host", config::get<config::System>().host_name());
|
|
#ifdef DEBUG
|
|
me->insert_session_statement->bind(":build_type", "debug");
|
|
#else
|
|
me->insert_session_statement->bind(":build_type", "release");
|
|
#endif
|
|
me->insert_session_statement->bind(
|
|
":session_start", me->session_start);
|
|
me->insert_session_statement->exec();
|
|
me->insert_session_statement->reset();
|
|
} catch (SQLite::Exception const &e) {
|
|
LOG(persist, Error,
|
|
"Error initializing perf_trace DB (2) {}. Error: {}", loc,
|
|
std::string(e.getErrorStr()));
|
|
_disabled = true;
|
|
}
|
|
LOG(persist, Info, "Finished initialization of perf_log db");
|
|
me->init_completed = true;
|
|
});
|
|
}
|
|
|
|
PerfLog::~PerfLog() { me->init_thread.join(); }
|
|
|
|
std::string PerfLog::init_storage_location() {
|
|
auto loc = persistence::storage_location("perf_log");
|
|
auto parent = boost::filesystem::path(loc).parent_path();
|
|
|
|
if (!boost::filesystem::is_directory(parent)) {
|
|
if (boost::filesystem::create_directories(parent))
|
|
return loc;
|
|
else
|
|
return "";
|
|
} else
|
|
return loc;
|
|
}
|
|
|
|
size_t PerfLog::store_log_entries() {
|
|
if (!me->init_completed || _disabled) {
|
|
_pending.clear();
|
|
return 0;
|
|
}
|
|
|
|
PerfLogEntry ev;
|
|
size_t count = 0;
|
|
while (_pending.try_pop(ev)) {
|
|
if (is_flush_signal(ev))
|
|
count += flush_current_batch_to_disk();
|
|
else
|
|
me->entry_batch.push_front(ev);
|
|
}
|
|
return count;
|
|
}
|
|
|
|
void PerfLog::init_db() {
|
|
auto sql_path = os::sparkd_wd() + "/data_files/perf_trace/perf_log.sql";
|
|
std::ifstream f(sql_path);
|
|
std::string str(
|
|
(std::istreambuf_iterator<char>(f)), (std::istreambuf_iterator<char>()));
|
|
config::PerfLogCfg defaults;
|
|
boost::replace_all(
|
|
str, std::to_string(defaults.max_rows()),
|
|
std::to_string(config::get<config::PerfLogCfg>().max_rows()));
|
|
me->db.exec(str);
|
|
}
|
|
|
|
std::string encode_string(SQLite::Column const &c) {
|
|
return fmt::format("\"{}\"", std::string(c));
|
|
}
|
|
|
|
std::string encode_number(SQLite::Column const &c) {
|
|
return fmt::format("{}", (long long)c);
|
|
}
|
|
|
|
std::vector<std::map<std::string, std::string>>
|
|
PerfLog::get_latest_requote_traces(size_t n) {
|
|
std::vector<std::map<std::string, std::string>> result;
|
|
if (!me->init_completed)
|
|
return result;
|
|
|
|
std::lock_guard<std::mutex> l(me->lock);
|
|
try {
|
|
me->requote_query_statement->bind(1, (long long)n);
|
|
|
|
while (me->requote_query_statement->executeStep()) {
|
|
auto &row = result.emplace_back();
|
|
row["version"] =
|
|
encode_string(me->requote_query_statement->getColumn("version"));
|
|
row["host"] =
|
|
encode_string(me->requote_query_statement->getColumn("host"));
|
|
row["build_type"] = encode_string(
|
|
me->requote_query_statement->getColumn("build_type"));
|
|
row["session_start"] = encode_string(
|
|
me->requote_query_statement->getColumn("session_start"));
|
|
row["trace_time"] = encode_string(
|
|
me->requote_query_statement->getColumn("trace_time"));
|
|
|
|
row["order_key"] =
|
|
encode_number(me->requote_query_statement->getColumn("order_key"));
|
|
row["total"] =
|
|
encode_number(me->requote_query_statement->getColumn("total"));
|
|
}
|
|
me->requote_query_statement->reset();
|
|
} catch (SQLite::Exception const &e) {
|
|
LOG(persist, Error, "Unable to retrieve quote traces {}. Error: {}",
|
|
std::string(me->db.getFilename()), std::string(e.getErrorStr()));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
std::vector<std::map<std::string, std::string>>
|
|
PerfLog::get_latest_hedge_traces(size_t n) {
|
|
std::vector<std::map<std::string, std::string>> result;
|
|
if (!me->init_completed)
|
|
return result;
|
|
|
|
std::lock_guard<std::mutex> l(me->lock);
|
|
try {
|
|
me->hedge_query_statement->bind(1, (long long)n);
|
|
|
|
while (me->hedge_query_statement->executeStep()) {
|
|
auto &row = result.emplace_back();
|
|
row["version"] =
|
|
encode_string(me->hedge_query_statement->getColumn("version"));
|
|
row["host"] =
|
|
encode_string(me->hedge_query_statement->getColumn("host"));
|
|
row["build_type"] =
|
|
encode_string(me->hedge_query_statement->getColumn("build_type"));
|
|
row["session_start"] = encode_string(
|
|
me->hedge_query_statement->getColumn("session_start"));
|
|
row["trace_time"] =
|
|
encode_string(me->hedge_query_statement->getColumn("trace_time"));
|
|
|
|
row["order_key"] =
|
|
encode_number(me->hedge_query_statement->getColumn("order_key"));
|
|
row["total"] =
|
|
encode_number(me->hedge_query_statement->getColumn("total"));
|
|
}
|
|
me->hedge_query_statement->reset();
|
|
} catch (SQLite::Exception const &e) {
|
|
LOG(persist, Error, "Unable to retrieve hedge traces. Error: {}",
|
|
std::string(e.getErrorStr()));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
bool PerfLog::is_flush_signal(PerfLogEntry const &ev) {
|
|
return ev.entry_kind == nullptr;
|
|
}
|
|
|
|
size_t PerfLog::flush_current_batch_to_disk() {
|
|
size_t count = 0;
|
|
try {
|
|
for (auto const &ev : me->entry_batch) {
|
|
if (count == 0)
|
|
me->db.exec("begin");
|
|
me->insert_trace_statement->bind(":session_start", me->session_start);
|
|
me->insert_trace_statement->bind(":ts", (long long)ev.ts.to_uint64());
|
|
me->insert_trace_statement->bind(
|
|
":hts",
|
|
datetime::FIX_timestamp(datetime::timestamp_to_ptime(ev.ts)));
|
|
me->insert_trace_statement->bind(":channel", ev.channel);
|
|
me->insert_trace_statement->bind(":entry_kind", ev.entry_kind);
|
|
me->insert_trace_statement->bind(":txt_1", ev.txt_1);
|
|
me->insert_trace_statement->bind(":sn", ev.sn);
|
|
me->insert_trace_statement->bind(":uint32_1", ev.uint32_1);
|
|
me->insert_trace_statement->bind(":uint64_1", (long long)ev.uint64_1);
|
|
me->insert_trace_statement->bind(
|
|
":cor_event", (long long)ev.cor_event.to_uint64());
|
|
me->insert_trace_statement->bind(":thread", ev.thread_name);
|
|
me->insert_trace_statement->exec();
|
|
count++;
|
|
me->insert_trace_statement->reset();
|
|
}
|
|
if (count)
|
|
me->db.exec("commit");
|
|
} catch (SQLite::Exception const &e) {
|
|
if (count)
|
|
try {
|
|
me->db.exec("rollback");
|
|
} catch (SQLite::Exception const &e2) {
|
|
}
|
|
|
|
LOG(persist, Error, "Unable to insert perf trace in. Error: {}",
|
|
std::string(e.getErrorStr()));
|
|
}
|
|
me->entry_batch.clear();
|
|
return count;
|
|
}
|
|
|
|
namespace perf_stat {
|
|
|
|
using Row = std::map<std::string, std::string>;
|
|
|
|
struct Summary {
|
|
std::string name;
|
|
double min;
|
|
double max;
|
|
double median;
|
|
double ninenty_nine;
|
|
double stddev;
|
|
std::vector<double> values;
|
|
};
|
|
|
|
struct Session {
|
|
std::string start;
|
|
std::string host;
|
|
std::string version;
|
|
std::string build_type;
|
|
uint32_t n_rows;
|
|
std::map<std::string, Summary> summaries;
|
|
std::vector<Row> rows;
|
|
};
|
|
|
|
} // namespace perf_stat
|
|
|
|
std::map<std::string, perf_stat::Session> structure_quote_traces(
|
|
std::vector<std::map<std::string, std::string>> "es) {
|
|
|
|
std::map<std::string, perf_stat::Session> result;
|
|
|
|
for (auto &q : quotes) {
|
|
auto &session = result[q["session_start"]];
|
|
if (session.start.empty()) {
|
|
session.start = q["session_start"];
|
|
session.host = q["host"];
|
|
session.version = q["version"];
|
|
session.build_type = q["build_type"];
|
|
}
|
|
for (auto st : {"total"}) {
|
|
auto &summary = session.summaries[st];
|
|
if (summary.name.empty())
|
|
summary.name = st;
|
|
summary.values.push_back(std::stod(q[st]));
|
|
}
|
|
session.rows.push_back(q);
|
|
session.n_rows++;
|
|
}
|
|
|
|
for (auto &session_kvp : result)
|
|
for (auto &stat_kvp : session_kvp.second.summaries) {
|
|
auto &summary = stat_kvp.second;
|
|
summary.min = stats::min(summary.values);
|
|
summary.max = stats::max(summary.values);
|
|
summary.median = stats::percentile(summary.values, 0.5);
|
|
summary.ninenty_nine = stats::percentile(summary.values, 0.99);
|
|
summary.stddev = stats::stdev(summary.values);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
std::map<std::string, perf_stat::Session> structure_hedge_traces(
|
|
std::vector<std::map<std::string, std::string>> "es) {
|
|
|
|
std::map<std::string, perf_stat::Session> result;
|
|
|
|
for (auto &q : quotes) {
|
|
auto &session = result[q["session_start"]];
|
|
if (session.start.empty()) {
|
|
session.start = q["session_start"];
|
|
session.host = q["host"];
|
|
session.version = q["version"];
|
|
session.build_type = q["build_type"];
|
|
}
|
|
for (auto st : {"total"}) {
|
|
auto &summary = session.summaries[st];
|
|
if (summary.name.empty())
|
|
summary.name = st;
|
|
summary.values.push_back(std::stod(q[st]));
|
|
}
|
|
session.rows.push_back(q);
|
|
session.n_rows++;
|
|
}
|
|
|
|
for (auto &session_kvp : result)
|
|
for (auto &stat_kvp : session_kvp.second.summaries) {
|
|
auto &summary = stat_kvp.second;
|
|
summary.min = stats::min(summary.values);
|
|
summary.max = stats::max(summary.values);
|
|
summary.median = stats::percentile(summary.values, 0.5);
|
|
summary.ninenty_nine = stats::percentile(summary.values, 0.99);
|
|
summary.stddev = stats::stdev(summary.values);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
void print_summary(std::ostream &ss, perf_stat::Summary const &s, bool last) {
|
|
ss << " { "
|
|
<< "\"value\":"
|
|
<< " \"" << s.name << "\""
|
|
<< ", \"min\":" << s.min << ", \"max\":" << s.max
|
|
<< ", \"median\":" << s.median << ", \"ninenty_nine\":" << s.ninenty_nine
|
|
<< ", \"stdev\":" << s.stddev;
|
|
if (!last)
|
|
ss << " },\n";
|
|
else
|
|
ss << " }\n";
|
|
}
|
|
|
|
void print_row(std::ostream &ss, perf_stat::Row const &r, bool last) {
|
|
ss << " { ";
|
|
size_t i = 0;
|
|
for (auto const &kvp : r)
|
|
if (kvp.first != "version" && kvp.first != "host" &&
|
|
kvp.first != "build_type" && kvp.first != "session_start") {
|
|
ss << "\"" << kvp.first << "\":" << kvp.second;
|
|
if (++i != r.size() - 4)
|
|
ss << ",";
|
|
}
|
|
if (!last)
|
|
ss << " },\n";
|
|
else
|
|
ss << " }\n";
|
|
}
|
|
|
|
void print_session(std::ostream &ss, perf_stat::Session const &s, bool last) {
|
|
ss << "{\n"
|
|
<< " \"session_name\": " << s.start << ",\n"
|
|
<< " \"host\": " << s.host << ",\n"
|
|
<< " \"version\": " << s.version << ",\n"
|
|
<< " \"build_type\": " << s.build_type << ",\n"
|
|
<< " \"n_rows\": " << s.n_rows << ",\n";
|
|
|
|
ss << " \"summaries\":\n [\n";
|
|
size_t i = 0;
|
|
for (auto &summaries_kvp : s.summaries)
|
|
print_summary(ss, summaries_kvp.second, ++i == s.summaries.size());
|
|
ss << " ],\n";
|
|
|
|
ss << " \"rows\":\n [\n";
|
|
i = 0;
|
|
for (auto &r : s.rows)
|
|
print_row(ss, r, ++i == s.rows.size());
|
|
ss << " ]\n";
|
|
if (!last)
|
|
ss << "},\n";
|
|
else
|
|
ss << "}\n";
|
|
}
|
|
|
|
void PerfLog::print_latest_traces(std::ostream &ss, size_t n) {
|
|
if (_disabled)
|
|
return;
|
|
|
|
{
|
|
auto rows = get_latest_requote_traces(n);
|
|
auto structured = structure_quote_traces(rows);
|
|
|
|
ss << "{\n\"requotes\": [\n";
|
|
size_t i = 0;
|
|
for (auto &session_kvp : boost::adaptors::reverse(structured))
|
|
print_session(ss, session_kvp.second, ++i == structured.size());
|
|
ss << "],\n";
|
|
}
|
|
|
|
{
|
|
auto rows = get_latest_hedge_traces(n);
|
|
auto structured = structure_hedge_traces(rows);
|
|
|
|
ss << "\"hedges\": [\n";
|
|
|
|
size_t i = 0;
|
|
for (auto &session_kvp : boost::adaptors::reverse(structured))
|
|
print_session(ss, session_kvp.second, ++i == structured.size());
|
|
ss << "]\n}\n";
|
|
}
|
|
}
|
|
|
|
} // namespace trace
|
|
} // namespace bts
|