import json import logging import os import string import sys import threading import time from functools import reduce from pathlib import Path from typing import Callable, Optional def exit_msg(code=0, msg=None): if msg is not None: print(msg) exit(code) def exec_and_log_error(exec: Callable, name=None, exit_on_err = True, reraise=False): ''' Executes a callable within a 'try/except' block; if an exception occurs, logs it and optionally exits the program, and/or re-raises the exception ''' def fmt_name(name): if name is None: return 'anonymous fn' try: return str(name) except: return 'fn-name-caused-error' try: exec() except Exception as e: msg = f"Error executing '{fmt_name(name)}': {e}" if exit_on_err: exit_msg(1, msg=msg) else: print(msg) if reraise: raise e class JsonElement: def __init__(self, value): self.value = value def __str__(self): try: to_json = getattr(self.value, "to_json", None) return json.dumps(to_json(self.value) if callable(to_json) else self.value) except Exception as e: print(f"JsonElement formatting error: {e}") return str(self.value) class StructuredLog: def __init__(self, **kwargs): self.kwargs = kwargs if "ts" not in kwargs: kwargs["ts"] = time.time_ns() def __str__(self): try: return json.dumps(self.kwargs) except Exception as e: print(f"StructuredLog formatting error: {e}") return str(self.kwargs) def _log_level_name(level: int): match level: case logging.DEBUG: return "Diag" case logging.INFO: return "Info" case logging.WARNING: return "Warn" case logging.ERROR: return "Error" case logging.CRITICAL: return "Fatal" case _: return str(level) class FileLogFormatter(logging.Formatter): def __init__(self, subsystem: str): self.subsystem = subsystem def formatTime(self, record, datefmt=None): ct = self.converter(record.created) if datefmt: return time.strftime(datefmt, ct) else: t = time.strftime("%Y-%m-%d %H:%M:%S", ct) return f"{t}.{int(record.msecs * 1000):06d}" def format(self, record): env = "dev" thread_id = threading.get_ident() filename_line = f"{record.filename}:{record.lineno}" timestamp = self.formatTime(record) timestamp_ns = int(record.created * 1e9) return f"{timestamp}|{timestamp_ns}|{self.subsystem:<8}|{_log_level_name(record.levelno):<5}|{os.getpid():<3}|{record.threadName:<20}|{thread_id}|{filename_line:<35}|{env}| {record.getMessage()}" class ConsoleLogFormatter(logging.Formatter): def __init__(self, subsystem: str): self.subsystem = subsystem def formatTime(self, record, datefmt=None): ct = self.converter(record.created) if datefmt: return time.strftime(datefmt, ct) else: t = time.strftime("%Y-%m-%d %H:%M:%S", ct) return f"{t}.{int(record.msecs * 1000):06d}" def format(self, record): filename_line = f"{record.filename}:{record.lineno}" timestamp = self.formatTime(record) return f"{timestamp}|{self.subsystem:<8}|{_log_level_name(record.levelno):<5}|{filename_line:<35}| {record.getMessage()}" def _app_data_dir(): if sys.platform == "win32": return Path(string.Template("${APPDATA}").substitute(os.environ)).joinpath("bts") else: return Path(f"/home/bts/sparkdata/") def _load_config_json(file_name: str, expand_vars: bool = True) -> Optional[dict]: if not file_name.endswith(".json"): file_name = file_name + ".json" s = "" try: with open(_app_data_dir().joinpath("conf", file_name)) as f: s = f.read() except FileNotFoundError: return None if expand_vars: try: s = string.Template(s).substitute(os.environ) except KeyError: return None return json.loads(s) def load_config(names: list[str], expand_vars: bool = True): def apply_overrides(cfg: dict, overrides: dict): cfg.update(overrides) return cfg return reduce( apply_overrides, [_load_config_json(n, expand_vars) or {} for n in names] ) def configure_logger( logger: logging.Logger, subsystem: str, **kwargs, ): logger.propagate = False # Console stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setLevel(kwargs.get("console_level", logging.WARNING)) stdout_handler.formatter = ConsoleLogFormatter(subsystem=subsystem) # File log_file = _app_data_dir() if sys.platform == "win32": log_file = log_file.joinpath("data", "logging") else: log_file = log_file.joinpath("data", "logging.sparkd") log_file = log_file.joinpath(f"{logger.name or 'py'}_log") if not log_file.parent.exists(): os.makedirs(str(log_file.parent), exist_ok=True) file_handler = logging.FileHandler(str(log_file)) file_handler.setLevel(kwargs.get("file_level", logging.INFO)) file_handler.formatter = FileLogFormatter(subsystem=subsystem) logger.handlers.clear() logger.addHandler(stdout_handler) logger.addHandler(file_handler) logger.setLevel(min(map(lambda h: h.level, logger.handlers))) return logger def configure_default_logger(**kwargs): configure_logger(logging.getLogger(), "Gen", **kwargs)