spark/tools/py/publish_data_stream.py

import argparse
import os
import subprocess
import sys

from shutil import copytree
from typing import Optional

from build_spark import SparkPaths
from core.argparse_utils import add_toggle_opt
from core.iter_utils import at_bounded


def _dotnet_verbosity(level):
    names = ["quiet", "minimal", "normal", "detailed", "diagnostic"]
    return at_bounded(names, level - 1)


def _cmake_verbosity(level):
    # Skip 'WARNING' and 'NOTICE' levels, since the default CMake verbosity,
    # 'STATUS', doesn't produce that much output. This also better normalizes
    # numeric representation of verbosity w/ dotnet, which only has 5 levels.
    names = [
        "ERROR",
        # 'WARNING',
        # 'NOTICE',
        "STATUS",
        "VERBOSE",
        "DEBUG",
        "TRACE",
    ]
    return at_bounded(names, level - 1)


def _cmake_preset():
    match sys.platform:
        case "win32":
            return "data-stream-suite-windows"
        case "linux":
            return "data-stream-suite-linux"
        case _:
            raise ValueError(f"Unsupported platform {sys.platform}")


def _dotnet_runtime():
    match sys.platform:
        case "win32":
            return "win-x64"
        case "linux":
            return "x64"
        case _:
            raise ValueError(f"Unsupported platform {sys.platform}")


class PublishArgs:
    def __init__(
        self,
        configuration: str = "release",
        cmake_jobs: Optional[int] = None,
        clean: bool = False,
        cpp: bool = True,
        cs: bool = True,
        csproj_refs: bool = True,
        verbosity: int = 1,
        dry_run: bool = False,
        output: Optional[str] = None,
        framework="net8.0-windows",
        runtime=_dotnet_runtime(),
        version: Optional[str] = None,
    ):
        self.configuration = configuration
        self.cmake_jobs = cmake_jobs
        self.clean = clean
        self.cpp = cpp
        self.cs = cs
        self.csproj_refs = csproj_refs
        self.verbosity = verbosity
        self.dry_run = dry_run
        self.dotnet_verbosity = at_bounded(
            ["quiet", "minimal", "normal", "detailed", "diagnostic"], verbosity
        )
        # Leave out WARNING and NOTICE to normalize cmake log level and dotnet verbosity
        # (so both can be expressed as an int 1-5)
        self.cmake_log_level = at_bounded(
            [
                "ERROR",
                # 'WARNING',
                # 'NOTICE',
                "STATUS",
                "VERBOSE",
                "DEBUG",
                "TRACE",
            ],
            verbosity,
        )
        self.output = output
        self.framework = framework
        self.runtime = runtime
        self.version = version


def _get_arg_parser():
    parser = argparse.ArgumentParser(
        description="Publish self-contined data stream applications"
    )

    parser.add_argument(
        "--configuration",
        metavar="debug|release",
        help="The build configuration",
    )
    parser.add_argument(
        "--cmake-jobs",
        type=int,
        help="The number of cmake parallel jobs",
    )
    parser.add_argument(
        "--clean",
        action="store_true",
        help="Perform a clean build (remove artifacts from previous build)",
    )
    add_toggle_opt(parser, "cpp", help="Build c++ libs")
    add_toggle_opt(parser, "cs", help="Build c# libs")
    add_toggle_opt(parser, "csproj-refs", help="Use local csproj refs (instead of nugets)")
    parser.add_argument(
        "--spark-version",
        help="The version identifier to use (by default, uses git to generate semantic version number)"
    )
    parser.add_argument(
        "-o", "--output", type=str, help="The output directory for published artifacts."
    )
    parser.add_argument(
        "--verbosity",
        metavar="{0-5}",
        choices=range(1, 6),
        type=int,
        help="The output verbosity level",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Preview the build commands without running them",
    )

    defaults = PublishArgs()
    parser.set_defaults(
        configuration=defaults.configuration,
        clean=defaults.clean,
        cpp=defaults.cpp,
        cs=defaults.cs,
        csproj_refs=defaults.csproj_refs,
        output=defaults.output,
        verbosity=defaults.verbosity,
        dry_run=defaults.dry_run,
    )
    return parser


def publish(args: PublishArgs):
    def run(cmd):
        print(cmd)
        if args.dry_run:
            return
        subprocess.run(cmd, shell=True).check_returncode()

    csproj = SparkPaths.root().joinpath("src", "DataStreamSuite", "DataStreamQueryAnalyzer", "DataStreamQueryAnalyzer.csproj")
    cmake_preset = _cmake_preset()
    cmake_config = args.configuration.title()
    cmake_jobs = args.cmake_jobs
    cmake_verbosity = _cmake_verbosity(args.verbosity)
    dotnet_config = args.configuration.title()
    dotnet_verbosity = _dotnet_verbosity(args.verbosity)
    publish_dir = SparkPaths.default_build().joinpath(
        dotnet_config,
        "bin",
        args.framework,
        args.runtime,
        "publish",
    )
    os.chdir(SparkPaths.root())
    cmake_configure_cmd = f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}"
    cmake_build_cmd = f"cmake --build build/{cmake_preset} --config {cmake_config} --target install"
    if cmake_jobs:
        cmake_build_cmd += f" -j {cmake_jobs}"
    spark_version = args.version
    if not spark_version:
        from core.git_utils import git_head_version
        spark_version = git_head_version()
    if args.clean:
        if args.cpp:
            run(f"cmake --build build/{cmake_preset} --config {cmake_config} --target clean")
        if args.cs:
            run(f"dotnet clean {csproj} -c {dotnet_config} -v {dotnet_verbosity} -p:UseProjectRefs={str(args.csproj_refs).lower()} -p SparkVersion={spark_version}")
    if args.cpp:
        run(cmake_configure_cmd)
        run(cmake_build_cmd)
    if args.cs:
        run(
            f"dotnet publish {csproj} -c {dotnet_config} -v {dotnet_verbosity} -p:UseProjectRefs={str(args.csproj_refs).lower()} -p:SparkVersion={spark_version} -f {args.framework} -r {args.runtime} --sc"
        )
    # Copy cpp bins
    print(f"[pseudo shell] cp -r install/{cmake_preset}/bin/* {publish_dir}")
    if not args.dry_run:
        copytree(f"install/{cmake_preset}/bin", publish_dir, dirs_exist_ok=True)
    if args.output:
        # Move to '--output' dir
        print(f"[pseudo shell] mv {publish_dir} {args.output}")
        if not args.dry_run:
            os.rename(publish_dir, args.output)


def _main():
    parser = _get_arg_parser()
    args = parser.parse_args()
    publish(
        PublishArgs(
            configuration=args.configuration,
            clean=args.clean,
            cpp=args.cpp,
            cs=args.cs,
            csproj_refs=args.csproj_refs,
            verbosity=args.verbosity,
            dry_run=args.dry_run,
            output=args.output,
            version=args.spark_version,
        )
    )


if __name__ == "__main__":
    try:
        _main()
    except Exception as e:
        print(f"publish_data_stream.py failed: {e}")
        exit(1)