import argparse import os import subprocess import sys from shutil import copytree from typing import Optional from build_spark import SparkPaths from core.argparse_utils import add_toggle_opt from core.iter_utils import at_bounded def _dotnet_verbosity(level): names = ["quiet", "minimal", "normal", "detailed", "diagnostic"] return at_bounded(names, level - 1) def _cmake_verbosity(level): # Skip 'WARNING' and 'NOTICE' levels, since the default CMake verbosity, # 'STATUS', doesn't produce that much output. This also better normalizes # numeric representation of verbosity w/ dotnet, which only has 5 levels. names = [ "ERROR", # 'WARNING', # 'NOTICE', "STATUS", "VERBOSE", "DEBUG", "TRACE", ] return at_bounded(names, level - 1) def _cmake_preset(): match sys.platform: case "win32": return "spark-windows" case "linux": return "spark-linux" case _: raise ValueError(f"Unsupported platform {sys.platform}") def _dotnet_runtime(): match sys.platform: case "win32": return "win-x64" case "linux": return "linux-x64" case _: raise ValueError(f"Unsupported platform {sys.platform}") class PublishArgs: def __init__( self, configuration: str = "release", cmake_jobs: Optional[int] = None, clean: bool = False, cpp: bool = True, cs: bool = True, csproj_refs: bool = True, verbosity: int = 2, dry_run: bool = False, output: Optional[str] = None, framework="net8.0", runtime=_dotnet_runtime(), version: Optional[str] = None, ): self.configuration = configuration self.cmake_jobs = cmake_jobs self.clean = clean self.cpp = cpp self.cs = cs self.csproj_refs = csproj_refs self.verbosity = verbosity self.dry_run = dry_run self.dotnet_verbosity = at_bounded( ["quiet", "minimal", "normal", "detailed", "diagnostic"], verbosity ) # Leave out WARNING and NOTICE to normalize cmake log level and dotnet verbosity # (so both can be expressed as an int 1-5) self.cmake_log_level = at_bounded( [ "ERROR", # 'WARNING', # 'NOTICE', "STATUS", "VERBOSE", "DEBUG", "TRACE", ], verbosity, ) self.output = output self.framework = framework self.runtime = runtime self.version = version def _get_arg_parser(): parser = argparse.ArgumentParser( description="Publish a self-contined spark client application" ) parser.add_argument( "--configuration", metavar="debug|release", help="The build configuration", ) parser.add_argument( "--cmake-jobs", type=int, help="The number of cmake parallel jobs", ) parser.add_argument( "--clean", action="store_true", help="Perform a clean build (remove artifacts from previous build)", ) add_toggle_opt(parser, "cpp", help="Build c++ libs") add_toggle_opt(parser, "cs", help="Build c# libs") add_toggle_opt(parser, "csproj-refs", help="Use local csproj refs (instead of nugets)") parser.add_argument( "--spark-version", help="The version identifier to use (by default, uses git to generate semantic version number)" ) parser.add_argument( "-o", "--output", type=str, help="The output directory for published artifacts." ) parser.add_argument( "--verbosity", metavar="{0-5}", choices=range(1, 6), type=int, help="The output verbosity level", ) parser.add_argument( "--dry-run", action="store_true", help="Preview the build commands without running them", ) defaults = PublishArgs() parser.set_defaults( configuration=defaults.configuration, clean=defaults.clean, cpp=defaults.cpp, cs=defaults.cs, csproj_refs=defaults.csproj_refs, output=defaults.output, verbosity=defaults.verbosity, dry_run=defaults.dry_run, ) return parser def publish(args: PublishArgs): def run(cmd): print(cmd) if args.dry_run: return subprocess.run(cmd, shell=True).check_returncode() csproj = SparkPaths.root().joinpath("src", "client", "spark_api_server") cmake_preset = _cmake_preset() cmake_config = args.configuration.title() cmake_jobs = args.cmake_jobs cmake_verbosity = _cmake_verbosity(args.verbosity) dotnet_config = args.configuration.title() dotnet_verbosity = _dotnet_verbosity(args.verbosity) publish_dir = SparkPaths.default_build().joinpath( dotnet_config, "bin", args.framework, args.runtime, "publish", ) os.chdir(SparkPaths.root()) cmake_configure_cmd = f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}" cmake_build_cmd = f"cmake --build build/{cmake_preset} --config {cmake_config} --target install" if cmake_jobs: cmake_build_cmd += f" -j {cmake_jobs}" spark_version = args.version if not spark_version: from core.git_utils import git_head_version spark_version = git_head_version() if args.clean: if args.cpp: run(f"cmake --build build/{cmake_preset} --config {cmake_config} --target clean") if args.cs: run(f"dotnet clean {csproj} -c {dotnet_config} -v {dotnet_verbosity} -p:UseProjectRefs={str(args.csproj_refs).lower()} -p SparkVersion={spark_version}") if args.cpp: run(cmake_configure_cmd) run(cmake_build_cmd) if args.cs: run( f"dotnet publish {csproj} -c {dotnet_config} -v {dotnet_verbosity} -p:UseProjectRefs={str(args.csproj_refs).lower()} -p:SparkVersion={spark_version} -f {args.framework} -r {args.runtime} --sc" ) # Copy cpp bins print(f"[pseudo shell] cp -r install/{cmake_preset}/bin/* {publish_dir}") if not args.dry_run: copytree(f"install/{cmake_preset}/bin", publish_dir, dirs_exist_ok=True) if args.output: # Move to '--output' dir print(f"[pseudo shell] mv {publish_dir} {args.output}") if not args.dry_run: os.rename(publish_dir, args.output) def _main(): parser = _get_arg_parser() args = parser.parse_args() publish( PublishArgs( configuration=args.configuration, clean=args.clean, csproj_refs=args.csproj_refs, cmake_jobs=args.cmake_jobs, verbosity=args.verbosity, dry_run=args.dry_run, output=args.output, version=args.spark_version, ) ) if __name__ == "__main__": try: _main() except Exception as e: print(f"publish_api_server.py failed: {e}") exit(1)