diff --git a/src/lib_python_interop/python_interop_plugins/spark_core.py b/src/lib_python_interop/python_interop_plugins/spark_core.py index 83bc87995..2b897f9d0 100644 --- a/src/lib_python_interop/python_interop_plugins/spark_core.py +++ b/src/lib_python_interop/python_interop_plugins/spark_core.py @@ -7,6 +7,7 @@ import sys import threading import time from functools import reduce +from pathlib import Path from typing import Any, Callable, Optional import websocket from python_interop_module import ( @@ -139,9 +140,10 @@ def configure_logger( stdout_handler.setLevel(kwargs.get("console_level", logging.WARNING)) stdout_handler.formatter = ConsoleLogFormatter(subsystem=subsystem) # File - file_handler = logging.FileHandler( - f"/home/bts/sparkdata/data/logging.sparkd/{logger.name or 'py'}_log" - ) + log_file=Path(f"/home/bts/sparkdata/data/logging.sparkd/{logger.name or 'py'}_log") + if not log_file.parent.exists(): + os.makedirs(str(log_file.parent), exist_ok=True) + file_handler = logging.FileHandler(str(log_file)) file_handler.setLevel(kwargs.get("file_level", logging.INFO)) file_handler.formatter = FileLogFormatter(subsystem=subsystem) logger.handlers.clear()