diag: Periodically log ActorThread work cycle speed

So we can better detect slowness, blockage, etc.
This commit is contained in:
John Colvin committed 2025-02-19 22:49:30 +00:00
1 parent b5d97c0b1b
commit 93ace52bfa
1 file changed
+45 -1
+45 -1
View File
@@ -8,6 +8,7 @@
#include "lib/config/config.h"
#include "lib/config/protos/jetfix.pb.h"
#include "lib/config/protos/orderman_cfg.pb.h"
#include "lib/core/FMT.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/domain_model/common.h"
#include "lib/logging/logging.h"
@@ -33,6 +34,7 @@
#include "lib/orderman/order_factory/DynamicOrderRegistry.h"
#include "lib/orderman/orderman_utils/ExchangeOpenOrders.h"
#include "lib/orderman/orders/Order.h"
#include "lib/os/os.h"
#include "lib/os/spark_data_directory.h"
#include "lib/plugin/SparkServices.h"
#include "lib/plugin/plugin.h"
@@ -71,16 +73,58 @@ struct ActorThread {
void start(trace::PerfLog &log) {
if (cfg.poll())
thread = threading::make_thread(cfg.thread_name(), [this, &log]() {
for (;;)
const core::TimeSpan log_interval =
core::TimeSpan::from_seconds(30);
core::TimeStamp last_log_time{0};
core::TimeStamp now;
core::TimeSpan elapsed;
std::uint32_t last_count = 0;
for (;;) {
now = core::TimeStamp::now();
elapsed = now - last_log_time;
if (elapsed > log_interval) {
LOG(OrderMan, Info,
"OrderServiceServer ActorThread work speed is {:.2f} "
"cycles/sec",
static_cast<float>(last_count) / elapsed.total_seconds());
last_log_time = now;
last_count = 0;
}
executor.do_work(log);
last_count++;
}
});
else {
thread = threading::make_thread(cfg.thread_name(), [this, &log]() {
// for logging purposes
const std::string t_name = bts::os::get_thread_name();
const std::string actors_desc = FMT_JOIN(cfg.hosted_actor(), ", ");
const core::TimeSpan log_interval =
core::TimeSpan::from_seconds(30);
const auto inner_iter = 1000;
bts::core::TimeStamp last_log_time{0};
core::TimeStamp now;
core::TimeSpan elapsed;
std::uint32_t last_count = 0;
while (!done) {
now = core::TimeStamp::now();
elapsed = now - last_log_time;
auto log_elapsed = now - last_log_time;
if (log_elapsed > log_interval) {
LOG(OrderMan, Info,
"OrderServiceServer ActorThread {} work speed is {:.2f} "
"cycles/sec ({})",
t_name,
static_cast<float>(last_count) / elapsed.total_seconds(),
actors_desc);
last_log_time = now;
last_count = 0;
}
auto start = Timer::now();
for (auto i = 0; i < inner_iter; i++)
executor.do_work(log);
last_count += inner_iter;
auto stop = Timer::now();
loop_metric->observe(ns_diff(stop, start) / inner_iter);
std::this_thread::sleep_for(10ms);