spark/tools/py/build_data_stream.py

import argparse
import os
import subprocess
import sys

from core.argparse_utils import add_toggle_opt
from core.file_utils import SparkPaths
from core.iter_utils import at_bounded


def _get_arg_parser():
    parser = argparse.ArgumentParser(description="Build DataStreamSuite.sln")
    parser.add_argument(
        "--configuration",
        metavar="debug|release",
        help="The build configuration",
    )
    parser.add_argument(
        "--cmake-jobs",
        type=int,
        help="The number of cmake parallel jobs",
    )
    parser.add_argument(
        "--clean",
        action="store_true",
        help="Perform a clean build (remove artifacts from previous build)",
    )
    add_toggle_opt(parser, "cpp", help="Build c++ libs")
    add_toggle_opt(parser, "cs", help="Build c# libs")
    add_toggle_opt(parser, "csproj-refs", help="Use local csproj refs (instead of nugets)")
    if sys.platform == "win32":
        add_toggle_opt(parser, "dsqa", help="Build DataStreamQueryAnalyzer")
    parser.add_argument(
        "--spark-version",
        help="The version identifier to use (by default, uses git to generate semantic version number)"
    )
    parser.add_argument(
        "--verbosity",
        metavar="{0-5}",
        choices=range(0, 6),
        type=int,
        help="The output verbosity level",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Preview the build commands without running them",
    )
    defaults = {
        "configuration": "debug",
        "clean": False,
        "cpp": True,
        "cs": True,
        "csproj_refs": True,
        "verbosity": 1,
    }
    if sys.platform == "win32":
        defaults["dsqa"] = True
    parser.set_defaults(**defaults)
    return parser


def _dotnet_verbosity(level):
    names = ["quiet", "minimal", "normal", "detailed", "diagnostic"]
    return at_bounded(names, level)


def _cmake_verbosity(level):
    # Skip 'WARNING' and 'NOTICE' levels, since the default CMake verbosity,
    # 'STATUS', doesn't produce that much output. This also better normalizes
    # numeric representation of verbosity w/ dotnet, which only has 5 levels.
    names = [
        "ERROR",
        # 'WARNING',
        # 'NOTICE',
        "STATUS",
        "VERBOSE",
        "DEBUG",
        "TRACE",
    ]
    return at_bounded(names, level)


def _cmake_preset():
    match sys.platform:
        case "win32":
            return "spark-windows"
        case "linux":
            return "spark-linux"
        case _:
            raise ValueError(f"Unsupported platform {sys.platform}")


def build(args):
    def run(cmd):
        print(cmd)
        if args.dry_run:
            return
        subprocess.run(cmd, shell=True).check_returncode()

    sln = SparkPaths.root().joinpath(
        "src",
        "DataStreamSuite",
        "DataStreamSuite.sln",
    )
    dsqa = SparkPaths.root().joinpath(
        "src", "DataStreamSuite", "DataStreamQueryAnalyzer",
        "DataStreamQueryAnalyzer.csproj"
    ) if sys.platform == "win32" and args.dsqa else None
    cmake_preset = _cmake_preset()
    cmake_config = args.configuration.title()
    cmake_jobs = args.cmake_jobs
    cmake_verbosity = _cmake_verbosity(args.verbosity)
    dotnet_config = args.configuration.title()
    dotnet_verbosity = _dotnet_verbosity(args.verbosity)
    os.chdir(SparkPaths.root())
    spark_version = args.spark_version
    if not spark_version:
        from core.git_utils import git_head_version
        spark_version = git_head_version()
    dotnet_build_fstr = f"dotnet build %s -c {dotnet_config} -v {dotnet_verbosity} -p:UseProjectRefs={str(args.csproj_refs).lower()} -p:SparkVersion={spark_version}"
    dotnet_clean_fstr = f"dotnet clean %s -c {dotnet_config} -v {dotnet_verbosity} -p:UseProjectRefs={str(args.csproj_refs).lower()} -p:SparkVersion={spark_version}"

    if args.clean:
        if args.cpp:
            run(f"cmake --build build/{cmake_preset} --target clean")
        if args.cs:
            run(dotnet_clean_fstr % sln)
            if dsqa:
                run(dotnet_clean_fstr % dsqa)
    if args.cpp:
        run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}")
        cmake_build_cmd = f"cmake --build build/{cmake_preset} --config {cmake_config}"
        if cmake_jobs:
            cmake_build_cmd += f" -j {cmake_jobs}"
        run(cmake_build_cmd)
    if args.cs:
        run(dotnet_build_fstr % sln)
        if dsqa:
            run(dotnet_build_fstr % dsqa)


def _main():
    parser = _get_arg_parser()
    args = parser.parse_args()
    build(args)


if __name__ == "__main__":
    try:
        _main()
    except Exception as e:
        print(f'build_data_stream.py failed: {e}')
        exit(1)