spark/tools/py/publish_data_stream.py

import argparse
import os
import subprocess
import sys

from typing import Optional

from build_spark import SparkPaths
from core.iter_utils import at_bounded


def _dotnet_verbosity(level):
    names = ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic']
    return at_bounded(names, level)


def _cmake_verbosity(level):
    # Skip 'WARNING' and 'NOTICE' levels, since the default CMake verbosity,
    # 'STATUS', doesn't produce that much output. This also better normalizes
    # numeric representation of verbosity w/ dotnet, which only has 5 levels.
    names = [
        'ERROR',
        # 'WARNING',
        # 'NOTICE',
        'STATUS',
        'VERBOSE',
        'DEBUG',
        'TRACE',
    ]
    return at_bounded(names, level)


def _cmake_preset():
    match sys.platform:
        case 'win32':
            return 'data-stream-suite-windows'
        case 'linux':
            return 'data-stream-suite-linux'
        case _:
            raise ValueError(f'Unsupported platform {sys.platform}')


def _cmake_build_preset(configuration):
    match sys.platform:
        case 'win32':
            return f'data-stream-suite-windows-{configuration}'
        case 'linux':
            return f'data-stream-suite-linux-{configuration}'
        case _:
            raise ValueError(f'Unsupported platform {sys.platform}')


def _dotnet_runtime():
    match sys.platform:
        case 'win32':
            return 'win-x64'
        case 'linux':
            return 'linux-x64'
        case _:
            raise ValueError(f'Unsupported platform {sys.platform}')


class PublishArgs:
    def __init__(
        self,
        configuration: str = 'release',
        clean: bool = False,
        verbosity: int = 1,
        dry_run: bool = False,
        output: Optional[str] = None,
        framework='net7.0',
        runtime=_dotnet_runtime(),
    ):
        self.configuration = configuration
        self.clean = clean
        self.verbosity = verbosity
        self.dry_run = dry_run
        self.dotnet_verbosity = at_bounded(
            ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'],
            verbosity)
        # Leave out WARNING and NOTICE to normalize cmake log level and dotnet verbosity
        # (so both can be expressed as an int 1-5)
        self.cmake_log_level = at_bounded(
            [
                'ERROR',
                # 'WARNING',
                # 'NOTICE',
                'STATUS',
                'VERBOSE',
                'DEBUG',
                'TRACE',
            ],
            verbosity)
        self.output = output
        self.framework = framework
        self.runtime = runtime


def _get_arg_parser():
    parser = argparse.ArgumentParser(
        description='Publish a self-contined spark client application')

    parser.add_argument(
        '--configuration',
        metavar='debug|release',
        help='The build configuration',
    )
    parser.add_argument(
        '--clean',
        action='store_true',
        help='Perform a clean build (remove artifacts from previous build)',
    )
    parser.add_argument(
        '-o',
        '--output',
        type=str,
        help='The output directory for published artifacts.')
    parser.add_argument(
        '--verbosity',
        metavar='{0-5}',
        choices=range(0, 6),
        type=int,
        help='The output verbosity level',
    )
    parser.add_argument(
        '--dry-run',
        action='store_true',
        help='Preview the build commands without running them',
    )

    defaults = PublishArgs()
    parser.set_defaults(
        configuration=defaults.configuration,
        clean=defaults.clean,
        output=defaults.output,
        verbosity=defaults.verbosity,
        dry_run=defaults.dry_run,
    )
    return parser


def publish(args: PublishArgs):
    def run(cmd):
        print(cmd)
        if args.dry_run:
            return
        subprocess.run(cmd, shell=True).check_returncode()

    def dotnet_cmd(action='build'):
        sln = SparkPaths.root().joinpath(
            'src',
            'DataStreamSuite',
            'DataStreamSuite.sln',
        )
        return ' '.join([
            f'dotnet {action} {sln}',
            f'--configuration {args.configuration.title()}',
            f'--verbosity {_dotnet_verbosity(args.verbosity)}',
        ])

    os.chdir(SparkPaths.root())
    publish_dir = SparkPaths.default_build().joinpath(
        args.configuration.title(),
        'bin',
        args.framework,
        args.runtime,
        'publish',
    )
    cmake_configure_cmd = ' '.join([
        f'cmake',
        f'--preset',
        _cmake_preset(),
        f'-DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}',
        f'-DDOTNET_BINARY_DIR={publish_dir}',
    ])
    cmake_build_cmd = ' '.join([
        f'cmake',
        f'--build',
        f'--preset {_cmake_build_preset(args.configuration.lower())}',
    ])
    if args.clean:
        run(f'{cmake_build_cmd} --target clean')
        run(dotnet_cmd('clean'))
    run(cmake_configure_cmd)
    run(cmake_build_cmd)
    publish_cmd = f'{dotnet_cmd("publish")} -f {args.framework} -r {args.runtime} --sc'
    run(publish_cmd)
    if args.output:
        run(f'mv {publish_dir} {args.output}')


def _main():
    parser = _get_arg_parser()
    args = parser.parse_args()
    publish(
        PublishArgs(
            configuration=args.configuration,
            clean=args.clean,
            verbosity=args.verbosity,
            dry_run=args.dry_run,
            output=args.output,
        ))


if __name__ == '__main__':
    try:
        _main()
    except Exception as e:
        print(f"publish_data_stream.py failed: {e}")
        exit(1)