Files
spark/tools/py/core/logging.py

193 lines
5.6 KiB
Python

import json
import logging
import os
import string
import sys
import threading
import time
from functools import reduce
from pathlib import Path
from typing import Callable, Optional
def exit_msg(code=0, msg=None):
if msg is not None:
print(msg)
exit(code)
def exec_and_log_error(exec: Callable, name=None, exit_on_err = True, reraise=False):
'''
Executes a callable within a 'try/except' block; if an exception occurs, logs it and optionally
exits the program, and/or re-raises the exception
'''
def fmt_name(name):
if name is None:
return 'anonymous fn'
try:
return str(name)
except:
return 'fn-name-caused-error'
try:
exec()
except Exception as e:
msg = f"Error executing '{fmt_name(name)}': {e}"
if exit_on_err:
exit_msg(1, msg=msg)
else:
print(msg)
if reraise:
raise e
class JsonElement:
def __init__(self, value):
self.value = value
def __str__(self):
try:
to_json = getattr(self.value, "to_json", None)
return json.dumps(to_json(self.value) if callable(to_json) else self.value)
except Exception as e:
print(f"JsonElement formatting error: {e}")
return str(self.value)
class StructuredLog:
def __init__(self, **kwargs):
self.kwargs = kwargs
if "ts" not in kwargs:
kwargs["ts"] = time.time_ns()
def __str__(self):
try:
return json.dumps(self.kwargs)
except Exception as e:
print(f"StructuredLog formatting error: {e}")
return str(self.kwargs)
def _log_level_name(level: int):
match level:
case logging.DEBUG:
return "Diag"
case logging.INFO:
return "Info"
case logging.WARNING:
return "Warn"
case logging.ERROR:
return "Error"
case logging.CRITICAL:
return "Fatal"
case _:
return str(level)
class FileLogFormatter(logging.Formatter):
def __init__(self, subsystem: str):
self.subsystem = subsystem
def formatTime(self, record, datefmt=None):
ct = self.converter(record.created)
if datefmt:
return time.strftime(datefmt, ct)
else:
t = time.strftime("%Y-%m-%d %H:%M:%S", ct)
return f"{t}.{int(record.msecs * 1000):06d}"
def format(self, record):
env = "dev"
thread_id = threading.get_ident()
filename_line = f"{record.filename}:{record.lineno}"
timestamp = self.formatTime(record)
timestamp_ns = int(record.created * 1e9)
return f"{timestamp}|{timestamp_ns}|{self.subsystem:<8}|{_log_level_name(record.levelno):<5}|{os.getpid():<3}|{record.threadName:<20}|{thread_id}|{filename_line:<35}|{env}| {record.getMessage()}"
class ConsoleLogFormatter(logging.Formatter):
def __init__(self, subsystem: str):
self.subsystem = subsystem
def formatTime(self, record, datefmt=None):
ct = self.converter(record.created)
if datefmt:
return time.strftime(datefmt, ct)
else:
t = time.strftime("%Y-%m-%d %H:%M:%S", ct)
return f"{t}.{int(record.msecs * 1000):06d}"
def format(self, record):
filename_line = f"{record.filename}:{record.lineno}"
timestamp = self.formatTime(record)
return f"{timestamp}|{self.subsystem:<8}|{_log_level_name(record.levelno):<5}|{filename_line:<35}| {record.getMessage()}"
def _app_data_dir():
if sys.platform == "win32":
return Path(string.Template("${APPDATA}").substitute(os.environ)).joinpath("bts")
else:
return Path(f"/home/bts/sparkdata/")
def _load_config_json(file_name: str, expand_vars: bool = True) -> Optional[dict]:
if not file_name.endswith(".json"):
file_name = file_name + ".json"
s = ""
try:
with open(_app_data_dir().joinpath("conf", file_name)) as f:
s = f.read()
except FileNotFoundError:
return None
if expand_vars:
try:
s = string.Template(s).substitute(os.environ)
except KeyError:
return None
return json.loads(s)
def load_config(names: list[str], expand_vars: bool = True):
def apply_overrides(cfg: dict, overrides: dict):
cfg.update(overrides)
return cfg
return reduce(
apply_overrides, [_load_config_json(n, expand_vars) or {} for n in names]
)
def configure_logger(
logger: logging.Logger,
subsystem: str,
**kwargs,
):
logger.propagate = False
# Console
stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.setLevel(kwargs.get("console_level", logging.WARNING))
stdout_handler.formatter = ConsoleLogFormatter(subsystem=subsystem)
# File
log_file = _app_data_dir()
if sys.platform == "win32":
log_file = log_file.joinpath("data", "logging")
else:
log_file = log_file.joinpath("data", "logging.sparkd")
log_file = log_file.joinpath(f"{logger.name or 'py'}_log")
if not log_file.parent.exists():
os.makedirs(str(log_file.parent), exist_ok=True)
file_handler = logging.FileHandler(str(log_file))
file_handler.setLevel(kwargs.get("file_level", logging.INFO))
file_handler.formatter = FileLogFormatter(subsystem=subsystem)
logger.handlers.clear()
logger.addHandler(stdout_handler)
logger.addHandler(file_handler)
logger.setLevel(min(map(lambda h: h.level, logger.handlers)))
return logger
def configure_default_logger(**kwargs):
configure_logger(logging.getLogger(), "Gen", **kwargs)