fix: Resolve main logging dir in new SparkDataDirectory class

This commit is contained in:
John Colvin committed 2024-11-29 15:05:12 +00:00
1 parent 7c0d51411a
commit 7cc2a1ec3e
10 files changed
+114 -53

No files matched your search

+2 -2
View File
@@ -6,7 +6,7 @@
#include "spark/api/service_helpers/AdminServiceUtils.h"
#include "spark/api/services/AdminService.client.pb.h"
#include "system/logging/logging.h"
#include "spark/utils/init_spark_lib.h"
#include "system/protobufs/logging.pb.h"
#include <algorithm>
@@ -19,7 +19,7 @@ namespace bts::cmds {
void md_report() {
auto cfg = config::get<logging::protobufs::Logging>();
cfg.mutable_console()->set_default_level("Fatal");
logging::configure_logging(cfg);
utils::SparkLibInit spark_lib{cfg, false, true};
std::map<api::CMEMics, std::string> mic_map;
mic_map[api::CMEMics::CME] = "XCME";
@@ -1,6 +1,7 @@
#define CATCH_CONFIG_RUNNER
#include "catch_test_runner.h"
#include "spark/config/config.h"
#include "spark/spark_data_directory.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/is_unittest_environment.h"
#include "system/logging/logging.h"
@@ -22,7 +23,8 @@ int run_tests(std::vector<std::string> const &args) {
auto cfg = config::get<logging::protobufs::Logging>();
cfg.mutable_console()->set_default_level("Fatal");
logging::configure_logging(cfg);
const spark::SparkDataDirectory data_dir;
logging::configure_logging(cfg, {}, data_dir.log_file().string());
bts_assert(args.size() < 100);
char const *argv[100];
+42 -8
View File
@@ -1,23 +1,57 @@
#include "init_spark_lib.h"
#include "spark/utils/bts_assert.h"
#include "spark/spark_data_directory.h"
#include "spark/utils/thread_manager.h"
#include "system/logging/logging.h"
#include "system/logging/protobufs/logging.h"
#include "system/output_switch.h"
#include "system/protobufs/config/system_config.pb.h"
namespace bts {
namespace utils {
static logging::LoggingConfigPoller *poller = nullptr;
using boost::filesystem::path;
using bts::logging::LoggingConfigPoller;
using bts::system::threading::ThreadManager;
void init_spark_lib() {
static LoggingConfigPoller *poller = nullptr;
static void init_spark_lib(
const logging::protobufs::Logging &logging_cfg,
bool hot_reload,
bool quiet_console) {
system::OutputSwitch cout{&std::cout, quiet_console};
// Write to stdout instead of normal log, because the logging system hasn't
// been configured yet (that's one of the things we're about to do).
std::cout << "Initializing spark_lib\n";
std::cout << "Configuring thread 'main'" << std::endl;
cout << "Initializing spark_lib\n";
cout << "Configuring thread 'main'" << std::endl;
threading::configure_current_thread("main");
std::cout << "Starting default logging thread" << std::endl;
cout << "Starting default logging thread" << std::endl;
logging::start_default_logging_thread();
std::cout << "Initializing LoggingConfigPoller" << std::endl;
poller = new logging::LoggingConfigPoller();
cout << "Initializing LoggingConfigPoller" << std::endl;
ThreadManager thread_man{ThreadManager::default_thread_config()};
auto lcfg = thread_man.get_thread_cfg("logger");
const spark::SparkDataDirectory data_dir;
if (hot_reload) {
poller = new LoggingConfigPoller(
logging_cfg, thread_man.get_thread_cfg("logger"),
data_dir.log_file().string(), data_dir.log_cache_file().string());
} else {
logging::configure_logging(
logging_cfg, lcfg, data_dir.log_file().string());
}
}
SparkLibInit::SparkLibInit(
const logging::protobufs::Logging &logging_cfg,
bool hot_reload,
bool quiet_console) {
init_spark_lib(logging_cfg, hot_reload, quiet_console);
}
SparkLibInit::~SparkLibInit() { finit_spark_lib(); }
void init_spark_lib() {
init_spark_lib(logging::protobufs::default_logging_config(), true, false);
}
void finit_spark_lib() {
+8 -2
View File
@@ -1,5 +1,7 @@
#pragma once
#include "bts_export.h"
#include "system/logging/protobufs/logging.h"
#include "system/protobufs/logging.pb.h"
// this is needed because we cannot relly on static initialization do
// work reliably in DLL's on Windows. See:
@@ -13,8 +15,12 @@ extern "C" BTS_EXPORT void init_spark_lib();
extern "C" BTS_EXPORT void finit_spark_lib();
struct SparkLibInit {
SparkLibInit() { init_spark_lib(); }
~SparkLibInit() { finit_spark_lib(); }
explicit SparkLibInit(
const logging::protobufs::Logging &logging_cfg =
logging::protobufs::default_logging_config(),
bool hot_reload = true,
bool quiet_console = false);
~SparkLibInit();
};
} // namespace utils
+13 -27
View File
@@ -4,7 +4,6 @@
#include "system/logging/logging_config.h"
#include "system/logging/protobufs/logging.h"
#include "system/os.h"
#include "system/protobufs/config/system_config.pb.h"
#include "system/protobufs/config/threads.pb.h"
#include "system/protobufs/logging.pb.h"
#include "system/threading/dispatcher.h"
@@ -31,7 +30,6 @@ namespace bts::logging {
using bts::logging::protobufs::Logging;
using bts::logging::protobufs::SinkConfig;
using bts::system::protobufs::SystemConfig;
using bts::system::protobufs::ThreadGroupCfg;
using Path = boost::filesystem::path;
@@ -47,13 +45,6 @@ typename M::const_iterator find_key(
return m.end();
}
Path log_path() {
// TODO: get path from SystemConfig (need to migrate system_config.proto over to lib_system)
return Path{"/home/bts/.local/share/bts/sparkd/data/logging.sparkd"};
// return Path{config::get<SystemConfig>().data_path()} /
// fmt::format("logging.{}/", os::get_application_name());
}
void start_default_logging_thread() { _default_logger_hub.start(); }
Filter get_filter(LoggingConfig &cfg, SinkConfig const &scfg) {
@@ -91,10 +82,11 @@ Filter get_filter(LoggingConfig &cfg, SinkConfig const &scfg) {
void configure_logging(const LoggingConfig &cfg) {
_default_logger_hub.config(cfg);
}
void configure_logging(const Logging &cfg) {
configure_logging(cfg, ThreadGroupCfg());
}
void configure_logging(const Logging &cfg, const ThreadGroupCfg &thread_cfg) {
void configure_logging(
const Logging &cfg,
const ThreadGroupCfg &thread_cfg,
const std::string &log_file) {
LoggingConfig real_cfg;
// real_cfg.deployment_type = config::get<config::DeploymentType>().type();
real_cfg.logger_initial_queue_size = cfg.initial_queue_size();
@@ -124,29 +116,23 @@ void configure_logging(const Logging &cfg, const ThreadGroupCfg &thread_cfg) {
real_cfg.sinks["syslog"] = std::make_pair(
syslog_sink(real_cfg), get_filter(real_cfg, cfg.syslog()));
real_cfg.sinks["file"] =
std::make_pair(file_sink(real_cfg), get_filter(real_cfg, cfg.file()));
std::make_pair(file_sink(real_cfg, log_file), get_filter(real_cfg, cfg.file()));
real_cfg.thread_cfg = thread_cfg;
configure_logging(real_cfg);
}
Path log_cache_file_path() { return log_path() / "log_control"; }
LoggingConfigPoller::LoggingConfigPoller(
const Logging &cfg, const ThreadGroupCfg &thread_cfg)
const Logging &cfg,
const ThreadGroupCfg &thread_cfg,
const std::string &log_file,
const std::string &log_cache_file)
: _poller(
utils::Dispatcher::background(), 1000, [=, this]() { poll_once(); }),
_last_ts(std::time(nullptr)), _log_cache_file(log_cache_file_path()),
_last_ts(std::time(nullptr)), _log_file(log_file), _log_cache_file(log_cache_file),
_done(false) {
init(cfg, thread_cfg);
}
LoggingConfigPoller::LoggingConfigPoller()
: LoggingConfigPoller(
logging::protobufs::default_logging_config(),
system::threading::ThreadManager{
system::threading::ThreadManager::default_thread_config()}
.get_thread_cfg("logger")) {}
void LoggingConfigPoller::init(
Logging const &cfg, const ThreadGroupCfg &thread_cfg) {
auto parent = _log_cache_file.parent_path();
@@ -158,7 +144,7 @@ void LoggingConfigPoller::init(
google::protobuf::TextFormat::Print(cfg, &gs);
// initialize logging
configure_logging(cfg, thread_cfg);
configure_logging(cfg, thread_cfg, _log_file.string());
_poller.start();
}
@@ -175,7 +161,7 @@ void LoggingConfigPoller::poll_once() {
if (google::protobuf::TextFormat::Parse(&gs, &cfg)) {
LOG(Log, Info,
"Detected logging configuration change. Reconfiguring logging.");
configure_logging(cfg);
configure_logging(cfg, ThreadGroupCfg{}, _log_file.string());
} else
LOG(Log, Error, "Detected invalid logging configuration. Ignoring.");
}
+6 -5
View File
@@ -87,8 +87,8 @@ extern thread_local LoggerHandle _thread_logger;
void configure_logging(LoggingConfig const &cfg);
void configure_logging(
const logging::protobufs::Logging &cfg,
const system::protobufs::ThreadGroupCfg &thread_cfg);
void configure_logging(const bts::logging::protobufs::Logging &cfg);
const system::protobufs::ThreadGroupCfg &thread_cfg,
const std::string &log_file);
void start_default_logging_thread();
class LoggingConfigPoller {
@@ -96,7 +96,9 @@ class LoggingConfigPoller {
LoggingConfigPoller();
LoggingConfigPoller(
const logging::protobufs::Logging &cfg,
const system::protobufs::ThreadGroupCfg &thread_cfg);
const system::protobufs::ThreadGroupCfg &thread_cfg,
const std::string &log_file,
const std::string &log_cache_file);
~LoggingConfigPoller();
private:
@@ -106,12 +108,11 @@ class LoggingConfigPoller {
void poll_once();
utils::DispatcherTimer _poller;
std::time_t _last_ts;
boost::filesystem::path _log_file;
boost::filesystem::path _log_cache_file;
std::atomic<bool> _done;
};
boost::filesystem::path log_path();
#define LOG(subsystem, level, ...) \
bts::logging::_thread_logger.logger->is_loggable( \
bts::logging::subsystem, level) && \
+3 -5
View File
@@ -96,16 +96,14 @@ std::ifstream::pos_type filesize(std::string const &filename) {
}
}
Sink file_sink(LoggingConfig const &cfg) {
Sink file_sink(LoggingConfig const &cfg, const std::string &file) {
static std::ofstream stream;
static uint64_t bytes_written = 0;
const auto max_sz = 120 * 1024 * 1024;
auto f = formatter(cfg);
auto path = log_path();
path /= "log";
auto parent = path.parent_path();
boost::filesystem::path path{file};
auto parent = path.parent_path();
if (!boost::filesystem::is_directory(parent))
bts_assert(boost::filesystem::create_directories(parent));
+1 -1
View File
@@ -31,7 +31,7 @@ GCC_DIAG_RESTORE()
using Sink = std::function<void(LogMsg const &)>;
Sink console_sink(LoggingConfig const &cfg);
Sink syslog_sink(LoggingConfig const &cfg);
Sink file_sink(LoggingConfig const &cfg);
Sink file_sink(LoggingConfig const &cfg, const std::string &file);
Sink null_sink();
} // namespace bts::logging
@@ -1,6 +1,8 @@
#include "system/bts_chrono.h"
#include "system/logging/logging.h"
#include "system/logging/protobufs/logging.h"
#include "system/os.h"
#include "system/protobufs/config/system_config.pb.h"
#include "system/protobufs/logging.pb.h"
#include <catch2/catch_all.hpp>
@@ -10,8 +12,9 @@
namespace bts {
int logging_tests;
TEST_CASE("Logging Tests", "[logging] [benchmark]") {
using boost::filesystem::path;
TEST_CASE("Logging Tests", "[logging] [benchmark]") {
auto sz = 1000000;
logging::Filter f;
for (auto &l : f)
@@ -73,9 +76,12 @@ TEST_CASE("Logging Tests", "[logging] [benchmark]") {
LOG(Testing, Info, "Logging: ns/msg for log skipping: {} tt: {} ",
bts::ns_diff(stop, start) / sz, bts::ns_diff(stop, start));
}
const path log_file = os::app_data_path() / "data" /
fmt::format("logging.{}", os::get_application_name()) /
"log";
auto cfg2 = logging::protobufs::default_logging_config();
cfg2.mutable_console()->set_default_level("Fatal");
logging::configure_logging(cfg2);
logging::configure_logging(cfg2, {}, log_file.string());
}
} // namespace bts
+28
View File
@@ -0,0 +1,28 @@
#pragma once
#include <ostream>
namespace bts::system {
struct OutputSwitch {
std::ostream *out = nullptr;
bool quiet = false;
template <class... Args>
OutputSwitch &operator<<(Args &&...args) {
if (!quiet) {
(*out << ... << std::forward<Args>(args));
}
return *this;
}
// Specialization for manipulators like std::endl
OutputSwitch &operator<<(std::ostream &(*manip)(std::ostream &)) {
if (!quiet) {
manip(*out);
}
return *this;
}
};
} // namespace bts::system