From a14aa58a245ac45f84394c28795a33e185fa2913 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 4 Jul 2024 17:20:54 +0000 Subject: [PATCH] build: Add scripts to publish data stream apps --- tools/build_data_stream_web_app.sh | 5 + tools/jenkins_publish_data_stream.cmd | 13 ++ tools/jenkins_publish_data_stream.sh | 11 ++ tools/publish_data_stream.sh | 5 + tools/publish_data_stream_web_app.sh | 2 +- tools/py/core/file_utils.py | 16 +- tools/py/publish_data_stream.py | 214 ++++++++++++++++++++++++++ 7 files changed, 260 insertions(+), 6 deletions(-) create mode 100755 tools/build_data_stream_web_app.sh create mode 100644 tools/jenkins_publish_data_stream.cmd create mode 100755 tools/jenkins_publish_data_stream.sh create mode 100755 tools/publish_data_stream.sh create mode 100644 tools/py/publish_data_stream.py diff --git a/tools/build_data_stream_web_app.sh b/tools/build_data_stream_web_app.sh new file mode 100755 index 000000000..25bc3ac48 --- /dev/null +++ b/tools/build_data_stream_web_app.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +tools_path=$(dirname $(readlink -f $0)) +eval "$tools_path/ensure_venv.sh" 1> /dev/stdout 2> /dev/stderr +eval "$tools_path/venv_exec.sh python $tools_path/py/build_data_stream_web_app.py $@" 1> /dev/stdout 2> /dev/stderr diff --git a/tools/jenkins_publish_data_stream.cmd b/tools/jenkins_publish_data_stream.cmd new file mode 100644 index 000000000..466b06908 --- /dev/null +++ b/tools/jenkins_publish_data_stream.cmd @@ -0,0 +1,13 @@ +@echo off + +@REM Build script designed for our jenkins build job(s). It assumes the py +@REM venv already exists. + +setlocal +set tools_dir=%~dp0 +set venv_dir=%~dp0..\.venv\ +call "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\Tools\VsDevCmd.bat" +call "%venv_dir%Scripts\activate.bat" +call python "%tools_dir%py\publish_data_stream.py" %* + +EXIT /B %ERRORLEVEL% diff --git a/tools/jenkins_publish_data_stream.sh b/tools/jenkins_publish_data_stream.sh new file mode 100755 index 000000000..1adf45e1e --- /dev/null +++ b/tools/jenkins_publish_data_stream.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +tools_path=$(dirname $(readlink -f $0)) +if [ "$VIRTUAL_ENV" == "" ]; then + venv_activate="$(readlink -f "$tools_path/../.venv/bin/activate")" + echo "Activating Python virtual environment: $venv_activate" + source "$venv_activate" +else + echo "Python virtual environment already active: $VIRTUAL_ENV" +fi +python "$tools_path/py/publish_data_stream.py" $@ diff --git a/tools/publish_data_stream.sh b/tools/publish_data_stream.sh new file mode 100755 index 000000000..7aad3e398 --- /dev/null +++ b/tools/publish_data_stream.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +tools_path=$(dirname $(readlink -f $0)) +eval "$tools_path/ensure_venv.sh" 1> /dev/stdout 2> /dev/stderr +eval "$tools_path/venv_exec.sh python $tools_path/py/publish_data_stream.py $@" 1> /dev/stdout 2> /dev/stderr diff --git a/tools/publish_data_stream_web_app.sh b/tools/publish_data_stream_web_app.sh index 25bc3ac48..d11eb98a3 100644 --- a/tools/publish_data_stream_web_app.sh +++ b/tools/publish_data_stream_web_app.sh @@ -2,4 +2,4 @@ tools_path=$(dirname $(readlink -f $0)) eval "$tools_path/ensure_venv.sh" 1> /dev/stdout 2> /dev/stderr -eval "$tools_path/venv_exec.sh python $tools_path/py/build_data_stream_web_app.py $@" 1> /dev/stdout 2> /dev/stderr +eval "$tools_path/venv_exec.sh python $tools_path/py/publish_data_stream_web_app.py $@" 1> /dev/stdout 2> /dev/stderr diff --git a/tools/py/core/file_utils.py b/tools/py/core/file_utils.py index 03b5114e0..b219100f0 100644 --- a/tools/py/core/file_utils.py +++ b/tools/py/core/file_utils.py @@ -24,11 +24,17 @@ def _workspace_root(): def _default_build(): - if sys.platform == 'win32': - return _workspace_root().joinpath('build') - elif sys.platform == 'linux': - return _workspace_root().joinpath('build', 'gcc', 'release') - raise ValueError(f"Unknown platform '{sys.platform}'") + return _workspace_root().joinpath('build') + + +def _dotnet_runtime(): + match sys.platform: + case 'win32': + return 'win-x64' + case 'linux': + return 'linux-x64' + case _: + raise ValueError(f'Unsupported platform {sys.platform}') class SparkPaths: diff --git a/tools/py/publish_data_stream.py b/tools/py/publish_data_stream.py new file mode 100644 index 000000000..8b5907919 --- /dev/null +++ b/tools/py/publish_data_stream.py @@ -0,0 +1,214 @@ +import argparse +import os +import subprocess +import sys + +from typing import Optional + +from build_spark import SparkPaths +from core.iter_utils import at_bounded + + +def _dotnet_verbosity(level): + names = ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'] + return at_bounded(names, level) + + +def _cmake_verbosity(level): + # Skip 'WARNING' and 'NOTICE' levels, since the default CMake verbosity, + # 'STATUS', doesn't produce that much output. This also better normalizes + # numeric representation of verbosity w/ dotnet, which only has 5 levels. + names = [ + 'ERROR', + # 'WARNING', + # 'NOTICE', + 'STATUS', + 'VERBOSE', + 'DEBUG', + 'TRACE', + ] + return at_bounded(names, level) + + +def _cmake_preset(): + match sys.platform: + case 'win32': + return 'data-stream-suite-windows' + case 'linux': + return 'data-stream-suite-linux' + case _: + raise ValueError(f'Unsupported platform {sys.platform}') + + +def _cmake_build_preset(configuration): + match sys.platform: + case 'win32': + return f'data-stream-suite-windows-{configuration}' + case 'linux': + return f'data-stream-suite-linux-{configuration}' + case _: + raise ValueError(f'Unsupported platform {sys.platform}') + + +def _dotnet_runtime(): + match sys.platform: + case 'win32': + return 'win-x64' + case 'linux': + return 'linux-x64' + case _: + raise ValueError(f'Unsupported platform {sys.platform}') + + +class PublishArgs: + def __init__( + self, + configuration: str = 'release', + clean: bool = False, + verbosity: int = 1, + dry_run: bool = False, + output: Optional[str] = None, + framework='net7.0', + runtime=_dotnet_runtime(), + ): + self.configuration = configuration + self.clean = clean + self.verbosity = verbosity + self.dry_run = dry_run + self.dotnet_verbosity = at_bounded( + ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'], + verbosity) + # Leave out WARNING and NOTICE to normalize cmake log level and dotnet verbosity + # (so both can be expressed as an int 1-5) + self.cmake_log_level = at_bounded( + [ + 'ERROR', + # 'WARNING', + # 'NOTICE', + 'STATUS', + 'VERBOSE', + 'DEBUG', + 'TRACE', + ], + verbosity) + self.output = output + self.framework = framework + self.runtime = runtime + + +def _get_arg_parser(): + parser = argparse.ArgumentParser( + description='Publish a self-contined spark client application') + + parser.add_argument( + '--configuration', + metavar='debug|release', + help='The build configuration', + ) + parser.add_argument( + '--clean', + action='store_true', + help='Perform a clean build (remove artifacts from previous build)', + ) + parser.add_argument( + '-o', + '--output', + type=str, + help='The output directory for published artifacts.') + parser.add_argument( + '--verbosity', + metavar='{0-5}', + choices=range(0, 6), + type=int, + help='The output verbosity level', + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Preview the build commands without running them', + ) + + defaults = PublishArgs() + parser.set_defaults( + configuration=defaults.configuration, + clean=defaults.clean, + output=defaults.output, + verbosity=defaults.verbosity, + dry_run=defaults.dry_run, + ) + return parser + + +def publish(args: PublishArgs): + def run(cmd): + print(cmd) + if args.dry_run: + return + subprocess.run(cmd, shell=True).check_returncode() + + def dotnet_cmd(project, action='build'): + csproj = SparkPaths.root().joinpath( + 'src', + 'DataStreamSuite', + project, + f'{project}.csproj', + ) + return ' '.join([ + f'dotnet {action} {csproj}', + f'--configuration {args.configuration.title()}', + f'--verbosity {_dotnet_verbosity(args.verbosity)}', + ]) + + os.chdir(SparkPaths.root()) + publish_dir = SparkPaths.default_build().joinpath( + args.configuration.title(), + 'bin', + args.framework, + args.runtime, + 'publish', + ) + cmake_configure_cmd = ' '.join([ + f'cmake', + f'--preset', + _cmake_preset(), + f'-DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}', + f'-DDOTNET_BINARY_DIR={publish_dir}', + ]) + cmake_build_cmd = ' '.join([ + f'cmake', + f'--build', + f'--preset {_cmake_build_preset(args.configuration.lower())}', + ]) + dotnet_projects = ["PubSubServer", "DataStreamPublisher", "SharedStreams"] + if args.clean: + run(f'{cmake_build_cmd} --target clean') + for p in dotnet_projects: + run(dotnet_cmd(p, 'clean')) + run(cmake_configure_cmd) + run(cmake_build_cmd) + for p in dotnet_projects: + publish_cmd = f'{dotnet_cmd(p, "publish")} -f {args.framework} -r {args.runtime} --sc' + run(publish_cmd) + if args.output: + run(f'mv {publish_dir} {args.output}') + + +def _main(): + parser = _get_arg_parser() + args = parser.parse_args() + publish( + PublishArgs( + configuration=args.configuration, + clean=args.clean, + verbosity=args.verbosity, + dry_run=args.dry_run, + output=args.output, + )) + + +if __name__ == '__main__': + try: + _main() + except Exception as e: + print(f"publish_data_stream.py failed: {e}") + exit(1)