From dd136afad723fdb66cd8d4f587bbcdaa76f8239f Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 15 Sep 2025 18:46:56 -0400 Subject: [PATCH] chore: Copy/paste some log utils from py interop dir to tools/py --- tools/py/core/logging.py | 161 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 160 insertions(+), 1 deletion(-) diff --git a/tools/py/core/logging.py b/tools/py/core/logging.py index 9f8f466c1..aa61b6379 100644 --- a/tools/py/core/logging.py +++ b/tools/py/core/logging.py @@ -1,4 +1,14 @@ -from typing import Callable +import json +import logging +import os +import string +import sys +import threading +import time + +from functools import reduce +from pathlib import Path +from typing import Callable, Optional def exit_msg(code=0, msg=None): @@ -31,3 +41,152 @@ def exec_and_log_error(exec: Callable, name=None, exit_on_err = True, reraise=Fa print(msg) if reraise: raise e + + +class JsonElement: + def __init__(self, value): + self.value = value + + def __str__(self): + try: + to_json = getattr(self.value, "to_json", None) + return json.dumps(to_json(self.value) if callable(to_json) else self.value) + except Exception as e: + print(f"JsonElement formatting error: {e}") + return str(self.value) + + +class StructuredLog: + def __init__(self, **kwargs): + self.kwargs = kwargs + if "ts" not in kwargs: + kwargs["ts"] = time.time_ns() + + def __str__(self): + try: + return json.dumps(self.kwargs) + except Exception as e: + print(f"StructuredLog formatting error: {e}") + return str(self.kwargs) + + +def _log_level_name(level: int): + match level: + case logging.DEBUG: + return "Diag" + case logging.INFO: + return "Info" + case logging.WARNING: + return "Warn" + case logging.ERROR: + return "Error" + case logging.CRITICAL: + return "Fatal" + case _: + return str(level) + + +class FileLogFormatter(logging.Formatter): + def __init__(self, subsystem: str): + self.subsystem = subsystem + + def formatTime(self, record, datefmt=None): + ct = self.converter(record.created) + if datefmt: + return time.strftime(datefmt, ct) + else: + t = time.strftime("%Y-%m-%d %H:%M:%S", ct) + return f"{t}.{int(record.msecs * 1000):06d}" + + def format(self, record): + env = "dev" + thread_id = threading.get_ident() + filename_line = f"{record.filename}:{record.lineno}" + timestamp = self.formatTime(record) + timestamp_ns = int(record.created * 1e9) + return f"{timestamp}|{timestamp_ns}|{self.subsystem:<8}|{_log_level_name(record.levelno):<5}|{os.getpid():<3}|{record.threadName:<20}|{thread_id}|{filename_line:<35}|{env}| {record.getMessage()}" + + +class ConsoleLogFormatter(logging.Formatter): + def __init__(self, subsystem: str): + self.subsystem = subsystem + + def formatTime(self, record, datefmt=None): + ct = self.converter(record.created) + if datefmt: + return time.strftime(datefmt, ct) + else: + t = time.strftime("%Y-%m-%d %H:%M:%S", ct) + return f"{t}.{int(record.msecs * 1000):06d}" + + def format(self, record): + filename_line = f"{record.filename}:{record.lineno}" + timestamp = self.formatTime(record) + return f"{timestamp}|{self.subsystem:<8}|{_log_level_name(record.levelno):<5}|{filename_line:<35}| {record.getMessage()}" + + +def _app_data_dir(): + if sys.platform == "win32": + return Path(string.Template("${APPDATA}").substitute(os.environ)).joinpath("bts") + else: + return Path(f"/home/bts/sparkdata/") + + +def _load_config_json(file_name: str, expand_vars: bool = True) -> Optional[dict]: + if not file_name.endswith(".json"): + file_name = file_name + ".json" + s = "" + try: + with open(_app_data_dir().joinpath("conf", file_name)) as f: + s = f.read() + except FileNotFoundError: + return None + if expand_vars: + try: + s = string.Template(s).substitute(os.environ) + except KeyError: + return None + return json.loads(s) + + +def load_config(names: list[str], expand_vars: bool = True): + def apply_overrides(cfg: dict, overrides: dict): + cfg.update(overrides) + return cfg + + return reduce( + apply_overrides, [_load_config_json(n, expand_vars) or {} for n in names] + ) + + +def configure_logger( + logger: logging.Logger, + subsystem: str, + **kwargs, +): + logger.propagate = False + # Console + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(kwargs.get("console_level", logging.WARNING)) + stdout_handler.formatter = ConsoleLogFormatter(subsystem=subsystem) + # File + log_file = _app_data_dir() + if sys.platform == "win32": + log_file = log_file.joinpath("data", "logging") + else: + log_file = log_file.joinpath("data", "logging.sparkd") + log_file = log_file.joinpath(f"{logger.name or 'py'}_log") + if not log_file.parent.exists(): + os.makedirs(str(log_file.parent), exist_ok=True) + file_handler = logging.FileHandler(str(log_file)) + file_handler.setLevel(kwargs.get("file_level", logging.INFO)) + file_handler.formatter = FileLogFormatter(subsystem=subsystem) + logger.handlers.clear() + logger.addHandler(stdout_handler) + logger.addHandler(file_handler) + logger.setLevel(min(map(lambda h: h.level, logger.handlers))) + return logger + + +def configure_default_logger(**kwargs): + configure_logger(logging.getLogger(), "Gen", **kwargs)