fix: Resolve main logging dir in new SparkDataDirectory class
This commit is contained in:
1 parent
7c0d51411a
commit
7cc2a1ec3e
10 files changed
+114
-53
No files matched your search
@@ -6,7 +6,7 @@
|
||||
|
||||
#include "spark/api/service_helpers/AdminServiceUtils.h"
|
||||
#include "spark/api/services/AdminService.client.pb.h"
|
||||
#include "system/logging/logging.h"
|
||||
#include "spark/utils/init_spark_lib.h"
|
||||
#include "system/protobufs/logging.pb.h"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -19,7 +19,7 @@ namespace bts::cmds {
|
||||
void md_report() {
|
||||
auto cfg = config::get<logging::protobufs::Logging>();
|
||||
cfg.mutable_console()->set_default_level("Fatal");
|
||||
logging::configure_logging(cfg);
|
||||
utils::SparkLibInit spark_lib{cfg, false, true};
|
||||
|
||||
std::map<api::CMEMics, std::string> mic_map;
|
||||
mic_map[api::CMEMics::CME] = "XCME";
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#define CATCH_CONFIG_RUNNER
|
||||
#include "catch_test_runner.h"
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/spark_data_directory.h"
|
||||
#include "spark/utils/bts_assert.h"
|
||||
#include "spark/utils/is_unittest_environment.h"
|
||||
#include "system/logging/logging.h"
|
||||
@@ -22,7 +23,8 @@ int run_tests(std::vector<std::string> const &args) {
|
||||
|
||||
auto cfg = config::get<logging::protobufs::Logging>();
|
||||
cfg.mutable_console()->set_default_level("Fatal");
|
||||
logging::configure_logging(cfg);
|
||||
const spark::SparkDataDirectory data_dir;
|
||||
logging::configure_logging(cfg, {}, data_dir.log_file().string());
|
||||
|
||||
bts_assert(args.size() < 100);
|
||||
char const *argv[100];
|
||||
|
||||
@@ -1,23 +1,57 @@
|
||||
#include "init_spark_lib.h"
|
||||
#include "spark/utils/bts_assert.h"
|
||||
#include "spark/spark_data_directory.h"
|
||||
#include "spark/utils/thread_manager.h"
|
||||
#include "system/logging/logging.h"
|
||||
#include "system/logging/protobufs/logging.h"
|
||||
#include "system/output_switch.h"
|
||||
#include "system/protobufs/config/system_config.pb.h"
|
||||
|
||||
namespace bts {
|
||||
namespace utils {
|
||||
|
||||
static logging::LoggingConfigPoller *poller = nullptr;
|
||||
using boost::filesystem::path;
|
||||
using bts::logging::LoggingConfigPoller;
|
||||
using bts::system::threading::ThreadManager;
|
||||
|
||||
void init_spark_lib() {
|
||||
static LoggingConfigPoller *poller = nullptr;
|
||||
|
||||
static void init_spark_lib(
|
||||
const logging::protobufs::Logging &logging_cfg,
|
||||
bool hot_reload,
|
||||
bool quiet_console) {
|
||||
system::OutputSwitch cout{&std::cout, quiet_console};
|
||||
// Write to stdout instead of normal log, because the logging system hasn't
|
||||
// been configured yet (that's one of the things we're about to do).
|
||||
std::cout << "Initializing spark_lib\n";
|
||||
std::cout << "Configuring thread 'main'" << std::endl;
|
||||
cout << "Initializing spark_lib\n";
|
||||
cout << "Configuring thread 'main'" << std::endl;
|
||||
threading::configure_current_thread("main");
|
||||
std::cout << "Starting default logging thread" << std::endl;
|
||||
cout << "Starting default logging thread" << std::endl;
|
||||
logging::start_default_logging_thread();
|
||||
std::cout << "Initializing LoggingConfigPoller" << std::endl;
|
||||
poller = new logging::LoggingConfigPoller();
|
||||
cout << "Initializing LoggingConfigPoller" << std::endl;
|
||||
ThreadManager thread_man{ThreadManager::default_thread_config()};
|
||||
auto lcfg = thread_man.get_thread_cfg("logger");
|
||||
const spark::SparkDataDirectory data_dir;
|
||||
if (hot_reload) {
|
||||
poller = new LoggingConfigPoller(
|
||||
logging_cfg, thread_man.get_thread_cfg("logger"),
|
||||
data_dir.log_file().string(), data_dir.log_cache_file().string());
|
||||
} else {
|
||||
logging::configure_logging(
|
||||
logging_cfg, lcfg, data_dir.log_file().string());
|
||||
}
|
||||
}
|
||||
|
||||
SparkLibInit::SparkLibInit(
|
||||
const logging::protobufs::Logging &logging_cfg,
|
||||
bool hot_reload,
|
||||
bool quiet_console) {
|
||||
init_spark_lib(logging_cfg, hot_reload, quiet_console);
|
||||
}
|
||||
|
||||
SparkLibInit::~SparkLibInit() { finit_spark_lib(); }
|
||||
|
||||
void init_spark_lib() {
|
||||
init_spark_lib(logging::protobufs::default_logging_config(), true, false);
|
||||
}
|
||||
|
||||
void finit_spark_lib() {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#pragma once
|
||||
#include "bts_export.h"
|
||||
#include "system/logging/protobufs/logging.h"
|
||||
#include "system/protobufs/logging.pb.h"
|
||||
|
||||
// this is needed because we cannot relly on static initialization do
|
||||
// work reliably in DLL's on Windows. See:
|
||||
@@ -13,8 +15,12 @@ extern "C" BTS_EXPORT void init_spark_lib();
|
||||
extern "C" BTS_EXPORT void finit_spark_lib();
|
||||
|
||||
struct SparkLibInit {
|
||||
SparkLibInit() { init_spark_lib(); }
|
||||
~SparkLibInit() { finit_spark_lib(); }
|
||||
explicit SparkLibInit(
|
||||
const logging::protobufs::Logging &logging_cfg =
|
||||
logging::protobufs::default_logging_config(),
|
||||
bool hot_reload = true,
|
||||
bool quiet_console = false);
|
||||
~SparkLibInit();
|
||||
};
|
||||
|
||||
} // namespace utils
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
#include "system/logging/logging_config.h"
|
||||
#include "system/logging/protobufs/logging.h"
|
||||
#include "system/os.h"
|
||||
#include "system/protobufs/config/system_config.pb.h"
|
||||
#include "system/protobufs/config/threads.pb.h"
|
||||
#include "system/protobufs/logging.pb.h"
|
||||
#include "system/threading/dispatcher.h"
|
||||
@@ -31,7 +30,6 @@ namespace bts::logging {
|
||||
|
||||
using bts::logging::protobufs::Logging;
|
||||
using bts::logging::protobufs::SinkConfig;
|
||||
using bts::system::protobufs::SystemConfig;
|
||||
using bts::system::protobufs::ThreadGroupCfg;
|
||||
using Path = boost::filesystem::path;
|
||||
|
||||
@@ -47,13 +45,6 @@ typename M::const_iterator find_key(
|
||||
return m.end();
|
||||
}
|
||||
|
||||
Path log_path() {
|
||||
// TODO: get path from SystemConfig (need to migrate system_config.proto over to lib_system)
|
||||
return Path{"/home/bts/.local/share/bts/sparkd/data/logging.sparkd"};
|
||||
// return Path{config::get<SystemConfig>().data_path()} /
|
||||
// fmt::format("logging.{}/", os::get_application_name());
|
||||
}
|
||||
|
||||
void start_default_logging_thread() { _default_logger_hub.start(); }
|
||||
|
||||
Filter get_filter(LoggingConfig &cfg, SinkConfig const &scfg) {
|
||||
@@ -91,10 +82,11 @@ Filter get_filter(LoggingConfig &cfg, SinkConfig const &scfg) {
|
||||
void configure_logging(const LoggingConfig &cfg) {
|
||||
_default_logger_hub.config(cfg);
|
||||
}
|
||||
void configure_logging(const Logging &cfg) {
|
||||
configure_logging(cfg, ThreadGroupCfg());
|
||||
}
|
||||
void configure_logging(const Logging &cfg, const ThreadGroupCfg &thread_cfg) {
|
||||
|
||||
void configure_logging(
|
||||
const Logging &cfg,
|
||||
const ThreadGroupCfg &thread_cfg,
|
||||
const std::string &log_file) {
|
||||
LoggingConfig real_cfg;
|
||||
// real_cfg.deployment_type = config::get<config::DeploymentType>().type();
|
||||
real_cfg.logger_initial_queue_size = cfg.initial_queue_size();
|
||||
@@ -124,29 +116,23 @@ void configure_logging(const Logging &cfg, const ThreadGroupCfg &thread_cfg) {
|
||||
real_cfg.sinks["syslog"] = std::make_pair(
|
||||
syslog_sink(real_cfg), get_filter(real_cfg, cfg.syslog()));
|
||||
real_cfg.sinks["file"] =
|
||||
std::make_pair(file_sink(real_cfg), get_filter(real_cfg, cfg.file()));
|
||||
std::make_pair(file_sink(real_cfg, log_file), get_filter(real_cfg, cfg.file()));
|
||||
real_cfg.thread_cfg = thread_cfg;
|
||||
configure_logging(real_cfg);
|
||||
}
|
||||
|
||||
Path log_cache_file_path() { return log_path() / "log_control"; }
|
||||
|
||||
LoggingConfigPoller::LoggingConfigPoller(
|
||||
const Logging &cfg, const ThreadGroupCfg &thread_cfg)
|
||||
const Logging &cfg,
|
||||
const ThreadGroupCfg &thread_cfg,
|
||||
const std::string &log_file,
|
||||
const std::string &log_cache_file)
|
||||
: _poller(
|
||||
utils::Dispatcher::background(), 1000, [=, this]() { poll_once(); }),
|
||||
_last_ts(std::time(nullptr)), _log_cache_file(log_cache_file_path()),
|
||||
_last_ts(std::time(nullptr)), _log_file(log_file), _log_cache_file(log_cache_file),
|
||||
_done(false) {
|
||||
init(cfg, thread_cfg);
|
||||
}
|
||||
|
||||
LoggingConfigPoller::LoggingConfigPoller()
|
||||
: LoggingConfigPoller(
|
||||
logging::protobufs::default_logging_config(),
|
||||
system::threading::ThreadManager{
|
||||
system::threading::ThreadManager::default_thread_config()}
|
||||
.get_thread_cfg("logger")) {}
|
||||
|
||||
void LoggingConfigPoller::init(
|
||||
Logging const &cfg, const ThreadGroupCfg &thread_cfg) {
|
||||
auto parent = _log_cache_file.parent_path();
|
||||
@@ -158,7 +144,7 @@ void LoggingConfigPoller::init(
|
||||
google::protobuf::TextFormat::Print(cfg, &gs);
|
||||
|
||||
// initialize logging
|
||||
configure_logging(cfg, thread_cfg);
|
||||
configure_logging(cfg, thread_cfg, _log_file.string());
|
||||
_poller.start();
|
||||
}
|
||||
|
||||
@@ -175,7 +161,7 @@ void LoggingConfigPoller::poll_once() {
|
||||
if (google::protobuf::TextFormat::Parse(&gs, &cfg)) {
|
||||
LOG(Log, Info,
|
||||
"Detected logging configuration change. Reconfiguring logging.");
|
||||
configure_logging(cfg);
|
||||
configure_logging(cfg, ThreadGroupCfg{}, _log_file.string());
|
||||
} else
|
||||
LOG(Log, Error, "Detected invalid logging configuration. Ignoring.");
|
||||
}
|
||||
|
||||
@@ -87,8 +87,8 @@ extern thread_local LoggerHandle _thread_logger;
|
||||
void configure_logging(LoggingConfig const &cfg);
|
||||
void configure_logging(
|
||||
const logging::protobufs::Logging &cfg,
|
||||
const system::protobufs::ThreadGroupCfg &thread_cfg);
|
||||
void configure_logging(const bts::logging::protobufs::Logging &cfg);
|
||||
const system::protobufs::ThreadGroupCfg &thread_cfg,
|
||||
const std::string &log_file);
|
||||
void start_default_logging_thread();
|
||||
|
||||
class LoggingConfigPoller {
|
||||
@@ -96,7 +96,9 @@ class LoggingConfigPoller {
|
||||
LoggingConfigPoller();
|
||||
LoggingConfigPoller(
|
||||
const logging::protobufs::Logging &cfg,
|
||||
const system::protobufs::ThreadGroupCfg &thread_cfg);
|
||||
const system::protobufs::ThreadGroupCfg &thread_cfg,
|
||||
const std::string &log_file,
|
||||
const std::string &log_cache_file);
|
||||
~LoggingConfigPoller();
|
||||
|
||||
private:
|
||||
@@ -106,12 +108,11 @@ class LoggingConfigPoller {
|
||||
void poll_once();
|
||||
utils::DispatcherTimer _poller;
|
||||
std::time_t _last_ts;
|
||||
boost::filesystem::path _log_file;
|
||||
boost::filesystem::path _log_cache_file;
|
||||
std::atomic<bool> _done;
|
||||
};
|
||||
|
||||
boost::filesystem::path log_path();
|
||||
|
||||
#define LOG(subsystem, level, ...) \
|
||||
bts::logging::_thread_logger.logger->is_loggable( \
|
||||
bts::logging::subsystem, level) && \
|
||||
|
||||
@@ -96,16 +96,14 @@ std::ifstream::pos_type filesize(std::string const &filename) {
|
||||
}
|
||||
}
|
||||
|
||||
Sink file_sink(LoggingConfig const &cfg) {
|
||||
Sink file_sink(LoggingConfig const &cfg, const std::string &file) {
|
||||
static std::ofstream stream;
|
||||
static uint64_t bytes_written = 0;
|
||||
const auto max_sz = 120 * 1024 * 1024;
|
||||
|
||||
auto f = formatter(cfg);
|
||||
auto path = log_path();
|
||||
path /= "log";
|
||||
|
||||
auto parent = path.parent_path();
|
||||
boost::filesystem::path path{file};
|
||||
auto parent = path.parent_path();
|
||||
|
||||
if (!boost::filesystem::is_directory(parent))
|
||||
bts_assert(boost::filesystem::create_directories(parent));
|
||||
|
||||
@@ -31,7 +31,7 @@ GCC_DIAG_RESTORE()
|
||||
using Sink = std::function<void(LogMsg const &)>;
|
||||
Sink console_sink(LoggingConfig const &cfg);
|
||||
Sink syslog_sink(LoggingConfig const &cfg);
|
||||
Sink file_sink(LoggingConfig const &cfg);
|
||||
Sink file_sink(LoggingConfig const &cfg, const std::string &file);
|
||||
Sink null_sink();
|
||||
|
||||
} // namespace bts::logging
|
||||
@@ -1,6 +1,8 @@
|
||||
#include "system/bts_chrono.h"
|
||||
#include "system/logging/logging.h"
|
||||
#include "system/logging/protobufs/logging.h"
|
||||
#include "system/os.h"
|
||||
#include "system/protobufs/config/system_config.pb.h"
|
||||
#include "system/protobufs/logging.pb.h"
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
@@ -10,8 +12,9 @@
|
||||
namespace bts {
|
||||
int logging_tests;
|
||||
|
||||
TEST_CASE("Logging Tests", "[logging] [benchmark]") {
|
||||
using boost::filesystem::path;
|
||||
|
||||
TEST_CASE("Logging Tests", "[logging] [benchmark]") {
|
||||
auto sz = 1000000;
|
||||
logging::Filter f;
|
||||
for (auto &l : f)
|
||||
@@ -73,9 +76,12 @@ TEST_CASE("Logging Tests", "[logging] [benchmark]") {
|
||||
LOG(Testing, Info, "Logging: ns/msg for log skipping: {} tt: {} ",
|
||||
bts::ns_diff(stop, start) / sz, bts::ns_diff(stop, start));
|
||||
}
|
||||
const path log_file = os::app_data_path() / "data" /
|
||||
fmt::format("logging.{}", os::get_application_name()) /
|
||||
"log";
|
||||
auto cfg2 = logging::protobufs::default_logging_config();
|
||||
cfg2.mutable_console()->set_default_level("Fatal");
|
||||
logging::configure_logging(cfg2);
|
||||
logging::configure_logging(cfg2, {}, log_file.string());
|
||||
}
|
||||
|
||||
} // namespace bts
|
||||
@@ -0,0 +1,28 @@
|
||||
#pragma once
|
||||
|
||||
#include <ostream>
|
||||
|
||||
namespace bts::system {
|
||||
|
||||
struct OutputSwitch {
|
||||
std::ostream *out = nullptr;
|
||||
bool quiet = false;
|
||||
|
||||
template <class... Args>
|
||||
OutputSwitch &operator<<(Args &&...args) {
|
||||
if (!quiet) {
|
||||
(*out << ... << std::forward<Args>(args));
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
// Specialization for manipulators like std::endl
|
||||
OutputSwitch &operator<<(std::ostream &(*manip)(std::ostream &)) {
|
||||
if (!quiet) {
|
||||
manip(*out);
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace bts::system
|
||||
Reference in new issue
Block a user