From 75907e2bd5b475de4656a3711eb45b42146fe34c Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 23 Oct 2024 03:34:18 +0000 Subject: [PATCH] Clean up build scripts --- tools/py/build_api_server.py | 107 +++++------ tools/py/build_data_stream.py | 120 ++++++------ tools/py/build_data_stream_query_analyzer.py | 29 ++- tools/py/build_data_stream_web_app.py | 37 ++-- tools/py/build_sparkd.py | 117 ++++++------ tools/py/publish_api_server.py | 53 +++--- tools/py/publish_data_stream.py | 188 +++++++++---------- tools/py/publish_spark.py | 179 +++++++++++------- tools/py/publish_sparkd.py | 157 ++++++++++++++++ 9 files changed, 584 insertions(+), 403 deletions(-) create mode 100644 tools/py/publish_sparkd.py diff --git a/tools/py/build_api_server.py b/tools/py/build_api_server.py index 2487f7e22..c8f4f8b56 100644 --- a/tools/py/build_api_server.py +++ b/tools/py/build_api_server.py @@ -8,31 +8,38 @@ from core.iter_utils import at_bounded def _get_arg_parser(): - parser = argparse.ArgumentParser(description='Build the Spark Api Server (websockets)') - parser.add_argument( - '--configuration', - metavar='debug|release', - help='The build configuration', + parser = argparse.ArgumentParser( + description="Build the Spark Api Server (websockets)" ) parser.add_argument( - '--clean', - action='store_true', - help='Perform a clean build (remove artifacts from previous build)', + "--configuration", + metavar="debug|release", + help="The build configuration", ) parser.add_argument( - '--verbosity', - metavar='{0-5}', + "--cmake-jobs", + type=int, + help="The number of cmake parallel jobs", + ) + parser.add_argument( + "--clean", + action="store_true", + help="Perform a clean build (remove artifacts from previous build)", + ) + parser.add_argument( + "--verbosity", + metavar="{0-5}", choices=range(0, 6), type=int, - help='The output verbosity level', + help="The output verbosity level", ) parser.add_argument( - '--dry-run', - action='store_true', - help='Preview the build commands without running them', + "--dry-run", + action="store_true", + help="Preview the build commands without running them", ) parser.set_defaults( - configuration='debug', + configuration="debug", clean=False, verbosity=1, ) @@ -40,7 +47,7 @@ def _get_arg_parser(): def _dotnet_verbosity(level): - names = ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'] + names = ["quiet", "minimal", "normal", "detailed", "diagnostic"] return at_bounded(names, level) @@ -49,33 +56,25 @@ def _cmake_verbosity(level): # 'STATUS', doesn't produce that much output. This also better normalizes # numeric representation of verbosity w/ dotnet, which only has 5 levels. names = [ - 'ERROR', + "ERROR", # 'WARNING', # 'NOTICE', - 'STATUS', - 'VERBOSE', - 'DEBUG', - 'TRACE', + "STATUS", + "VERBOSE", + "DEBUG", + "TRACE", ] return at_bounded(names, level) + def _cmake_preset(): match sys.platform: - case 'win32': - return 'spark-api-server-windows' - case 'linux': - return 'spark-api-server-linux' + case "win32": + return "spark-api-server-windows" + case "linux": + return "spark-api-server-linux" case _: - raise ValueError(f'Unsupported platform {sys.platform}') - -def _cmake_build_preset(configuration): - match sys.platform: - case 'win32': - return f'spark-api-server-windows-{configuration}' - case 'linux': - return f'spark-api-server-linux-{configuration}' - case _: - raise ValueError(f'Unsupported platform {sys.platform}') + raise ValueError(f"Unsupported platform {sys.platform}") def _main(): @@ -88,37 +87,27 @@ def _main(): return subprocess.run(cmd, shell=True).check_returncode() - def dotnet_cmd(action='build'): - sln = SparkPaths.root().joinpath('Spark.Api.Server.sln') - return ' '.join([ - f'dotnet {action} {sln}', - f'--configuration {args.configuration.title()}', - f'--verbosity {_dotnet_verbosity(args.verbosity)}', - ]) - + sln = SparkPaths.root().joinpath("Spark.Api.Server.sln") + cmake_preset = _cmake_preset() + cmake_jobs = args.cmake_jobs + cmake_verbosity = _cmake_verbosity(args.verbosity) + dotnet_config = args.configuration.title() + dotnet_verbosity = _dotnet_verbosity(args.verbosity) os.chdir(SparkPaths.root()) - cmake_configure_cmd = ' '.join([ - f'cmake', - f'--preset', - _cmake_preset(), - f'-DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}', - ]) - cmake_build_cmd = ' '.join([ - f'cmake', - f'--build', - f'--preset {_cmake_build_preset(args.configuration.lower())}', - ]) if args.clean: - run(f'{cmake_build_cmd} --target clean') - run(dotnet_cmd('clean')) - run(cmake_configure_cmd) + run(f"cmake --build build/{cmake_preset} --target clean") + run(f"dotnet clean {sln} -c {dotnet_config} -v {dotnet_verbosity}") + run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}") + cmake_build_cmd = f"cmake --build build/{cmake_preset}" + if cmake_jobs: + cmake_build_cmd += f" -j {cmake_jobs}" run(cmake_build_cmd) - run(dotnet_cmd('build')) + run(f"dotnet build {sln} -c {dotnet_config} -v {dotnet_verbosity}") if __name__ == "__main__": try: _main() except Exception as e: - print(f'build_api_server.py failed: {e}') + print(f"build_api_server.py failed: {e}") exit(1) diff --git a/tools/py/build_data_stream.py b/tools/py/build_data_stream.py index 64b3b09cf..853fd01ba 100644 --- a/tools/py/build_data_stream.py +++ b/tools/py/build_data_stream.py @@ -8,31 +8,36 @@ from core.iter_utils import at_bounded def _get_arg_parser(): - parser = argparse.ArgumentParser(description='Build DataStreamSuite.sln') + parser = argparse.ArgumentParser(description="Build DataStreamSuite.sln") parser.add_argument( - '--configuration', - metavar='debug|release', - help='The build configuration', + "--configuration", + metavar="debug|release", + help="The build configuration", ) parser.add_argument( - '--clean', - action='store_true', - help='Perform a clean build (remove artifacts from previous build)', + "--cmake-jobs", + type=int, + help="The number of cmake parallel jobs", ) parser.add_argument( - '--verbosity', - metavar='{0-5}', + "--clean", + action="store_true", + help="Perform a clean build (remove artifacts from previous build)", + ) + parser.add_argument( + "--verbosity", + metavar="{0-5}", choices=range(0, 6), type=int, - help='The output verbosity level', + help="The output verbosity level", ) parser.add_argument( - '--dry-run', - action='store_true', - help='Preview the build commands without running them', + "--dry-run", + action="store_true", + help="Preview the build commands without running them", ) parser.set_defaults( - configuration='debug', + configuration="debug", clean=False, verbosity=1, ) @@ -40,7 +45,7 @@ def _get_arg_parser(): def _dotnet_verbosity(level): - names = ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'] + names = ["quiet", "minimal", "normal", "detailed", "diagnostic"] return at_bounded(names, level) @@ -49,77 +54,60 @@ def _cmake_verbosity(level): # 'STATUS', doesn't produce that much output. This also better normalizes # numeric representation of verbosity w/ dotnet, which only has 5 levels. names = [ - 'ERROR', + "ERROR", # 'WARNING', # 'NOTICE', - 'STATUS', - 'VERBOSE', - 'DEBUG', - 'TRACE', + "STATUS", + "VERBOSE", + "DEBUG", + "TRACE", ] return at_bounded(names, level) def _cmake_preset(): match sys.platform: - case 'win32': - return 'data-stream-suite-windows' - case 'linux': - return 'data-stream-suite-linux' + case "win32": + return "data-stream-suite-windows" + case "linux": + return "data-stream-suite-linux" case _: - raise ValueError(f'Unsupported platform {sys.platform}') + raise ValueError(f"Unsupported platform {sys.platform}") -def _cmake_build_preset(configuration): - match sys.platform: - case 'win32': - return f'data-stream-suite-windows-{configuration}' - case 'linux': - return f'data-stream-suite-linux-{configuration}' - case _: - raise ValueError(f'Unsupported platform {sys.platform}') - - -def _main(): - parser = _get_arg_parser() - args = parser.parse_args() - +def build(args): def run(cmd): print(cmd) if args.dry_run: return subprocess.run(cmd, shell=True).check_returncode() - def dotnet_cmd(action='build'): - sln = SparkPaths.root().joinpath( - 'src', - 'DataStreamSuite', - 'DataStreamSuite.sln', - ) - return ' '.join([ - f'dotnet {action} {sln}', - f'--configuration {args.configuration.title()}', - f'--verbosity {_dotnet_verbosity(args.verbosity)}', - ]) - + sln = SparkPaths.root().joinpath( + "src", + "DataStreamSuite", + "DataStreamSuite.sln", + ) + cmake_preset = _cmake_preset() + cmake_jobs = args.cmake_jobs + cmake_verbosity = _cmake_verbosity(args.verbosity) + dotnet_config = args.configuration.title() + dotnet_verbosity = _dotnet_verbosity(args.verbosity) os.chdir(SparkPaths.root()) - cmake_configure_cmd = ' '.join([ - f'cmake', - f'--preset', - _cmake_preset(), - f'-DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}', - ]) - cmake_build_cmd = ' '.join([ - f'cmake', - f'--build', - f'--preset {_cmake_build_preset(args.configuration.lower())}', - ]) if args.clean: - run(f'{cmake_build_cmd} --target clean') - run(dotnet_cmd('clean')) - run(cmake_configure_cmd) + run(f"cmake --build build/{cmake_preset} --target clean") + run(f"dotnet clean {sln} -c {dotnet_config} -v {dotnet_verbosity}") + run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}") + cmake_build_cmd = f"cmake --build build/{cmake_preset}" + if cmake_jobs: + cmake_build_cmd += f" -j {cmake_jobs}" run(cmake_build_cmd) - run(dotnet_cmd('build')) + run(f"dotnet build {sln} -c {dotnet_config} -v {dotnet_verbosity}") + + +def _main(): + parser = _get_arg_parser() + args = parser.parse_args() + build(args) if __name__ == "__main__": diff --git a/tools/py/build_data_stream_query_analyzer.py b/tools/py/build_data_stream_query_analyzer.py index b4a4eac20..b7522d8f3 100644 --- a/tools/py/build_data_stream_query_analyzer.py +++ b/tools/py/build_data_stream_query_analyzer.py @@ -90,25 +90,22 @@ def _main(): return subprocess.run(cmd, shell=True).check_returncode() - def dotnet_cmd(action): - sln = SparkPaths.root().joinpath( - "src", "DataStreamSuite", "DataStreamQueryAnalyzer.sln" - ) - cfg = args.configuration.title() - verbosity = _dotnet_verbosity(args.verbosity) - return f"dotnet {action} {sln} --configuration {cfg} --verbosity {verbosity}" - os.chdir(SparkPaths.root()) - cmake_configure_cmd = f"cmake --preset {_cmake_preset()} -DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}" - cmake_build_cmd = ( - f"cmake --build --preset {_cmake_build_preset(args.configuration.lower())}" + sln = SparkPaths.root().joinpath( + "src", "DataStreamSuite", "DataStreamQueryAnalyzer.sln" ) + cmake_preset = _cmake_preset() + cmake_build_preset = _cmake_build_preset(args.configuration) + cmake_verbosity = _cmake_verbosity(args.verbosity) + dotnet_config = args.configuration.title() + dotnet_verbosity = _dotnet_verbosity(args.verbosity) + os.chdir(SparkPaths.root()) if args.clean: - run(f"{cmake_build_cmd} --target clean") - run(dotnet_cmd("clean")) - run(cmake_configure_cmd) - run(cmake_build_cmd) - run(dotnet_cmd("build")) + run(f"cmake --build --preset {cmake_build_preset} --target clean") + run(f"dotnet clean {sln} -c {dotnet_config} -v {dotnet_verbosity}") + run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}") + run(f"cmake --build --preset {cmake_build_preset}") + run(f"dotnet build {sln} -c {dotnet_config} -v {dotnet_verbosity}") if __name__ == "__main__": diff --git a/tools/py/build_data_stream_web_app.py b/tools/py/build_data_stream_web_app.py index 8aba3dfa1..a23b7f1b9 100644 --- a/tools/py/build_data_stream_web_app.py +++ b/tools/py/build_data_stream_web_app.py @@ -14,6 +14,11 @@ def _get_arg_parser(): metavar="debug|release", help="The build configuration", ) + parser.add_argument( + "--cmake-jobs", + type=int, + help="The number of cmake parallel jobs", + ) parser.add_argument( "--clean", action="store_true", @@ -90,23 +95,29 @@ def _main(): return subprocess.run(cmd, shell=True).check_returncode() - def dotnet_cmd(action="build"): - csproj = SparkPaths.root().joinpath( - "src", "DataStreamSuite", "Bts.DataStreamWebApp.Server", "Bts.DataStreamWebApp.Server.csproj" - ) - return f"dotnet {action} {csproj} --configuration {args.configuration.title()} --verbosity {_dotnet_verbosity(args.verbosity)}" - os.chdir(SparkPaths.root()) - cmake_configure_cmd = f"cmake --preset {_cmake_preset()} -DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}" - cmake_build_cmd = ( - f"cmake --build --preset {_cmake_build_preset(args.configuration.lower())}" + csproj = SparkPaths.root().joinpath( + "src", + "DataStreamSuite", + "Bts.DataStreamWebApp.Server", + "Bts.DataStreamWebApp.Server.csproj", ) + cmake_preset = _cmake_preset() + cmake_build_preset = _cmake_build_preset(args.configuration) + cmake_jobs = args.cmake_jobs + cmake_verbosity = _cmake_verbosity(args.verbosity) + dotnet_config = args.configuration.title() + dotnet_verbosity = _dotnet_verbosity(args.verbosity) + os.chdir(SparkPaths.root()) if args.clean: - run(f"{cmake_build_cmd} --target clean") - run(dotnet_cmd("clean")) - run(cmake_configure_cmd) + run(f"cmake --build --preset {cmake_build_preset} --target clean") + run(f"dotnet clean {csproj} -c {dotnet_config} -v {dotnet_verbosity}") + run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}") + cmake_build_cmd = f"cmake --build --preset {cmake_build_preset}" + if cmake_jobs: + cmake_build_cmd += f" -j {cmake_jobs}" run(cmake_build_cmd) - run(dotnet_cmd("build")) + run(f"dotnet build {csproj} -c {dotnet_config} -v {dotnet_verbosity}") if __name__ == "__main__": diff --git a/tools/py/build_sparkd.py b/tools/py/build_sparkd.py index 57229afac..e299d7792 100644 --- a/tools/py/build_sparkd.py +++ b/tools/py/build_sparkd.py @@ -9,49 +9,54 @@ from core.iter_utils import at_bounded def _get_arg_parser(): parser = argparse.ArgumentParser( - description='Builds the Spark server (A.K.A sparkd, the daemon)') - parser.add_argument( - '--build-type', - metavar='debug|release', - help='The build configuration', + description="Builds the Spark server (A.K.A sparkd, the daemon)" ) parser.add_argument( - '--compiler', - metavar='compiler', - choices=['gcc', 'clang'], - help='The compiler to use', + "--build-type", + metavar="debug|release", + help="The build configuration", ) parser.add_argument( - '--preset', - metavar= - 'sparkd-debug-clang|sparkd-debug-gcc|sparkd-release-clang|spard-release-gcc', + "--compiler", + metavar="compiler", + choices=["gcc", "clang"], + help="The compiler to use", + ) + parser.add_argument( + "--preset", + metavar="sparkd-debug-clang|sparkd-debug-gcc|sparkd-release-clang|spard-release-gcc", choices=[ - 'sparkd-debug-clang', - 'sparkd-debug-gcc', - 'sparkd-release-clang', - 'sparkd-release-gcc', + "sparkd-debug-clang", + "sparkd-debug-gcc", + "sparkd-release-clang", + "sparkd-release-gcc", ], - help='The CMake preset to use', + help="The CMake preset to use", ) parser.add_argument( - '--clean', - action='store_true', - help='Perform a clean build (remove artifacts from previous build)', + "--cmake-jobs", + type=int, + help="The number of cmake parallel jobs", ) parser.add_argument( - '--verbosity', - metavar='{0-5}', + "--clean", + action="store_true", + help="Perform a clean build (remove artifacts from previous build)", + ) + parser.add_argument( + "--verbosity", + metavar="{0-5}", choices=range(0, 6), type=int, - help='The output verbosity level', + help="The output verbosity level", ) parser.add_argument( - '--dry-run', - action='store_true', - help='Preview the build commands without running them', + "--dry-run", + action="store_true", + help="Preview the build commands without running them", ) parser.set_defaults( - build_type='debug', + build_type="debug", compiler=None, preset=None, clean=False, @@ -65,37 +70,31 @@ def _cmake_verbosity(level): # 'STATUS', doesn't produce that much output. This also better normalizes # numeric representation of verbosity w/ dotnet, which only has 5 levels. names = [ - 'ERROR', + "ERROR", # 'WARNING', # 'NOTICE', - 'STATUS', - 'VERBOSE', - 'DEBUG', - 'TRACE', + "STATUS", + "VERBOSE", + "DEBUG", + "TRACE", ] return at_bounded(names, level) def _cmake_compiler(build_type: str): - return 'clang' if build_type.lower() == 'debug' else 'gcc' + return "clang" if build_type.lower() == "debug" else "gcc" -def _cmake_preset(compiler: str, build_type: str): - return f'sparkd-{build_type}-{compiler}' - - -def _cmake_build_preset( - compiler: Optional[str] = None, - build_type: Optional[str] = None, - configure_preset: Optional[str] = None, +def _cmake_preset( + preset: Optional[str], compiler: Optional[str], build_type: Optional[str] ): - if configure_preset: - return configure_preset + if preset: + return preset if not build_type: - build_type = 'debug' + build_type = "debug" if not compiler: - compiler = 'clang' if build_type.lower() == 'debug' else 'gcc' - return f'sparkd-{build_type}-{compiler}' + compiler = _cmake_compiler(build_type) + return f"sparkd-{build_type}-{compiler}" def _main(): @@ -108,26 +107,22 @@ def _main(): return subprocess.run(cmd, shell=True).check_returncode() + cmake_preset = _cmake_preset(args.preset, args.compiler, args.build_type) + cmake_jobs: Optional[int] = args.cmake_jobs + cmake_verbosity = _cmake_verbosity(args.verbosity) os.chdir(SparkPaths.root()) - build_type: str = args.build_type - compiler: str = args.compiler or _cmake_compiler(build_type) - preset = args.preset or _cmake_preset(compiler.lower(), build_type.lower()) - build_preset = _cmake_build_preset( - compiler.lower(), - build_type.lower(), - args.preset, - ) - verbosity = _cmake_verbosity(args.verbosity) - configure_cmd = f'cmake --preset {preset} -DCMAKE_MESSAGE_LOG_LEVEL={verbosity}' - build_cmd = f'cmake --build --preset {build_preset}' if args.clean: - run(f'{build_cmd} --target clean') - run(configure_cmd) - run(build_cmd) + run(f"cmake --build build/{cmake_preset} --target clean") + run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}") + cmake_build_cmd = f"cmake --build build/{cmake_preset}" + if cmake_jobs: + cmake_build_cmd += f" -j {cmake_jobs}" + run(cmake_build_cmd) if __name__ == "__main__": try: _main() except Exception as e: - print(f'build_sparkd.py failed: {e}') + print(f"build_sparkd.py failed: {e}") + exit(1) diff --git a/tools/py/publish_api_server.py b/tools/py/publish_api_server.py index 96d972329..bdf9627d2 100644 --- a/tools/py/publish_api_server.py +++ b/tools/py/publish_api_server.py @@ -12,7 +12,7 @@ from core.iter_utils import at_bounded def _dotnet_verbosity(level): names = ["quiet", "minimal", "normal", "detailed", "diagnostic"] - return at_bounded(names, level) + return at_bounded(names, level - 1) def _cmake_verbosity(level): @@ -28,7 +28,7 @@ def _cmake_verbosity(level): "DEBUG", "TRACE", ] - return at_bounded(names, level) + return at_bounded(names, level - 1) def _cmake_preset(): @@ -41,16 +41,6 @@ def _cmake_preset(): raise ValueError(f"Unsupported platform {sys.platform}") -def _cmake_build_preset(configuration): - match sys.platform: - case "win32": - return f"spark-api-server-windows-{configuration}" - case "linux": - return f"spark-api-server-linux-{configuration}" - case _: - raise ValueError(f"Unsupported platform {sys.platform}") - - def _dotnet_runtime(): match sys.platform: case "win32": @@ -65,14 +55,16 @@ class PublishArgs: def __init__( self, configuration: str = "release", + cmake_jobs: Optional[int] = None, clean: bool = False, - verbosity: int = 1, + verbosity: int = 2, dry_run: bool = False, output: Optional[str] = None, framework="net7.0", runtime=_dotnet_runtime(), ): self.configuration = configuration + self.cmake_jobs = cmake_jobs self.clean = clean self.verbosity = verbosity self.dry_run = dry_run @@ -108,6 +100,11 @@ def _get_arg_parser(): metavar="debug|release", help="The build configuration", ) + parser.add_argument( + "--cmake-jobs", + type=int, + help="The number of cmake parallel jobs", + ) parser.add_argument( "--clean", action="store_true", @@ -119,7 +116,7 @@ def _get_arg_parser(): parser.add_argument( "--verbosity", metavar="{0-5}", - choices=range(0, 6), + choices=range(1, 6), type=int, help="The output verbosity level", ) @@ -148,32 +145,38 @@ def publish(args: PublishArgs): subprocess.run(cmd, shell=True).check_returncode() sln = SparkPaths.root().joinpath("Spark.Api.Server.sln") - dotnet_cfg = args.configuration.title() + cmake_preset = _cmake_preset() + cmake_config = args.configuration.title() + cmake_jobs = args.cmake_jobs + cmake_verbosity = _cmake_verbosity(args.verbosity) + dotnet_config = args.configuration.title() dotnet_verbosity = _dotnet_verbosity(args.verbosity) - os.chdir(SparkPaths.root()) publish_dir = SparkPaths.default_build().joinpath( - args.configuration.title(), + dotnet_config, "bin", args.framework, args.runtime, "publish", ) - cmake_preset = _cmake_preset() - cmake_build_preset = _cmake_build_preset(args.configuration.lower()) - cmake_verbosity = _cmake_verbosity(args.verbosity) + os.chdir(SparkPaths.root()) if args.clean: run( - f"cmake --build --preset {cmake_build_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity} --target clean" + f"cmake --build build/{cmake_preset} --config {cmake_config} --target clean" ) - run(f"dotnet clean {sln} -c {dotnet_cfg} -v {dotnet_verbosity}") + run(f"dotnet clean {sln} -c {dotnet_config} -v {dotnet_verbosity}") run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}") - run(f"cmake --build --preset {cmake_build_preset} --target install") + cmake_build_cmd = ( + f"cmake --build build/{cmake_preset} --config {cmake_config} --target install" + ) + if cmake_jobs: + cmake_build_cmd += f" -j {cmake_jobs}" + run(cmake_build_cmd) run( - f"dotnet publish {sln} -c {dotnet_cfg} -v {dotnet_verbosity} -f {args.framework} -r {args.runtime} --sc" + f"dotnet publish {sln} -c {dotnet_config} -v {dotnet_verbosity} -f {args.framework} -r {args.runtime} --sc" ) copytree(f"install/{cmake_preset}/lib", publish_dir, dirs_exist_ok=True) if args.output: - run(f"mv {publish_dir} {args.output}") + os.rename(publish_dir, args.output) def _main(): diff --git a/tools/py/publish_data_stream.py b/tools/py/publish_data_stream.py index 00fa81e3a..56a5a3ede 100644 --- a/tools/py/publish_data_stream.py +++ b/tools/py/publish_data_stream.py @@ -3,6 +3,7 @@ import os import subprocess import sys +from shutil import copytree from typing import Optional from build_spark import SparkPaths @@ -10,8 +11,8 @@ from core.iter_utils import at_bounded def _dotnet_verbosity(level): - names = ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'] - return at_bounded(names, level) + names = ["quiet", "minimal", "normal", "detailed", "diagnostic"] + return at_bounded(names, level - 1) def _cmake_verbosity(level): @@ -19,78 +20,71 @@ def _cmake_verbosity(level): # 'STATUS', doesn't produce that much output. This also better normalizes # numeric representation of verbosity w/ dotnet, which only has 5 levels. names = [ - 'ERROR', + "ERROR", # 'WARNING', # 'NOTICE', - 'STATUS', - 'VERBOSE', - 'DEBUG', - 'TRACE', + "STATUS", + "VERBOSE", + "DEBUG", + "TRACE", ] - return at_bounded(names, level) + return at_bounded(names, level - 1) def _cmake_preset(): match sys.platform: - case 'win32': - return 'data-stream-suite-windows' - case 'linux': - return 'data-stream-suite-linux' + case "win32": + return "data-stream-suite-windows" + case "linux": + return "data-stream-suite-linux" case _: - raise ValueError(f'Unsupported platform {sys.platform}') - - -def _cmake_build_preset(configuration): - match sys.platform: - case 'win32': - return f'data-stream-suite-windows-{configuration}' - case 'linux': - return f'data-stream-suite-linux-{configuration}' - case _: - raise ValueError(f'Unsupported platform {sys.platform}') + raise ValueError(f"Unsupported platform {sys.platform}") def _dotnet_runtime(): match sys.platform: - case 'win32': - return 'win-x64' - case 'linux': - return 'linux-x64' + case "win32": + return "win-x64" + case "linux": + return "linux-x64" case _: - raise ValueError(f'Unsupported platform {sys.platform}') + raise ValueError(f"Unsupported platform {sys.platform}") class PublishArgs: def __init__( self, - configuration: str = 'release', + configuration: str = "release", + cmake_jobs: Optional[int] = None, clean: bool = False, - verbosity: int = 1, + verbosity: int = 2, dry_run: bool = False, output: Optional[str] = None, - framework='net7.0', + framework="net7.0", runtime=_dotnet_runtime(), ): self.configuration = configuration + self.cmake_jobs = cmake_jobs self.clean = clean self.verbosity = verbosity self.dry_run = dry_run self.dotnet_verbosity = at_bounded( - ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'], - verbosity) + ["quiet", "minimal", "normal", "detailed", "diagnostic"], verbosity + ) # Leave out WARNING and NOTICE to normalize cmake log level and dotnet verbosity # (so both can be expressed as an int 1-5) self.cmake_log_level = at_bounded( [ - 'ERROR', + "ERROR", # 'WARNING', # 'NOTICE', - 'STATUS', - 'VERBOSE', - 'DEBUG', - 'TRACE', + "STATUS", + "VERBOSE", + "DEBUG", + "TRACE", ], - verbosity) + verbosity, + ) self.output = output self.framework = framework self.runtime = runtime @@ -98,34 +92,38 @@ class PublishArgs: def _get_arg_parser(): parser = argparse.ArgumentParser( - description='Publish a self-contined spark client application') + description="Publish a self-contined spark client application" + ) parser.add_argument( - '--configuration', - metavar='debug|release', - help='The build configuration', + "--configuration", + metavar="debug|release", + help="The build configuration", ) parser.add_argument( - '--clean', - action='store_true', - help='Perform a clean build (remove artifacts from previous build)', - ) - parser.add_argument( - '-o', - '--output', - type=str, - help='The output directory for published artifacts.') - parser.add_argument( - '--verbosity', - metavar='{0-5}', - choices=range(0, 6), + "--cmake-jobs", type=int, - help='The output verbosity level', + help="The number of cmake parallel jobs", ) parser.add_argument( - '--dry-run', - action='store_true', - help='Preview the build commands without running them', + "--clean", + action="store_true", + help="Perform a clean build (remove artifacts from previous build)", + ) + parser.add_argument( + "-o", "--output", type=str, help="The output directory for published artifacts." + ) + parser.add_argument( + "--verbosity", + metavar="{1-5}", + choices=range(1, 6), + type=int, + help="The output verbosity level", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Preview the build commands without running them", ) defaults = PublishArgs() @@ -139,54 +137,51 @@ def _get_arg_parser(): return parser -def publish(args: PublishArgs): +def publish(args): def run(cmd): print(cmd) if args.dry_run: return subprocess.run(cmd, shell=True).check_returncode() - def dotnet_cmd(action='build'): - sln = SparkPaths.root().joinpath( - 'src', - 'DataStreamSuite', - 'DataStreamSuite.sln', - ) - return ' '.join([ - f'dotnet {action} {sln}', - f'--configuration {args.configuration.title()}', - f'--verbosity {_dotnet_verbosity(args.verbosity)}', - ]) - - os.chdir(SparkPaths.root()) + sln = SparkPaths.root().joinpath( + "src", + "DataStreamSuite", + "DataStreamSuite.sln", + ) + cmake_preset = _cmake_preset() + cmake_config = args.configuration.title() + cmake_jobs = args.cmake_jobs + cmake_verbosity = _cmake_verbosity(args.verbosity) + dotnet_config = args.configuration.title() + dotnet_verbosity = _dotnet_verbosity(args.verbosity) publish_dir = SparkPaths.default_build().joinpath( args.configuration.title(), - 'bin', + "bin", args.framework, args.runtime, - 'publish', + "publish", ) - cmake_configure_cmd = ' '.join([ - f'cmake', - f'--preset', - _cmake_preset(), - f'-DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}', - f'-DDOTNET_BINARY_DIR={publish_dir}', - ]) - cmake_build_cmd = ' '.join([ - f'cmake', - f'--build', - f'--preset {_cmake_build_preset(args.configuration.lower())}', - ]) + os.chdir(SparkPaths.root()) if args.clean: - run(f'{cmake_build_cmd} --target clean') - run(dotnet_cmd('clean')) - run(cmake_configure_cmd) + run( + f"cmake --build build/{cmake_preset} --config {cmake_config} --target clean" + ) + run(f"dotnet clean {sln} -c {dotnet_config} -v {dotnet_verbosity}") + run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}") + cmake_build_cmd = ( + f"cmake --build build/{cmake_preset} --config {cmake_config} --target install" + ) + if cmake_jobs: + cmake_build_cmd += f" -j {cmake_jobs}" run(cmake_build_cmd) - publish_cmd = f'{dotnet_cmd("publish")} -f {args.framework} -r {args.runtime} --sc' - run(publish_cmd) + run(f"dotnet build {sln} -c {dotnet_config} -v {dotnet_verbosity}") + run( + f"dotnet publish {sln} -c {dotnet_config} -v {dotnet_verbosity} -f {args.framework} -r {args.runtime} --sc" + ) + copytree(f"./install/{cmake_preset}/lib", publish_dir, dirs_exist_ok=True) if args.output: - run(f'mv {publish_dir} {args.output}') + os.rename(publish_dir, args.output) def _main(): @@ -199,10 +194,11 @@ def _main(): verbosity=args.verbosity, dry_run=args.dry_run, output=args.output, - )) + ) + ) -if __name__ == '__main__': +if __name__ == "__main__": try: _main() except Exception as e: diff --git a/tools/py/publish_spark.py b/tools/py/publish_spark.py index df9d68dfe..8fb30b850 100644 --- a/tools/py/publish_spark.py +++ b/tools/py/publish_spark.py @@ -1,44 +1,86 @@ import argparse import os import subprocess +import sys +from shutil import copytree from typing import Optional from build_spark import SparkPaths from core.iter_utils import at_bounded +def _dotnet_verbosity(level): + names = ["quiet", "minimal", "normal", "detailed", "diagnostic"] + return at_bounded(names, level - 1) + + +def _cmake_verbosity(level): + # Skip 'WARNING' and 'NOTICE' levels, since the default CMake verbosity, + # 'STATUS', doesn't produce that much output. This also better normalizes + # numeric representation of verbosity w/ dotnet, which only has 5 levels. + names = [ + "ERROR", + # 'WARNING', + # 'NOTICE', + "STATUS", + "VERBOSE", + "DEBUG", + "TRACE", + ] + return at_bounded(names, level - 1) + + +def _cmake_preset(): + match sys.platform: + case "win32": + return "spark" + case _: + raise ValueError(f"Unsupported platform {sys.platform}") + + +def _dotnet_runtime(): + match sys.platform: + case "win32": + return "win-x64" + case _: + raise ValueError(f"Unsupported platform {sys.platform}") + + class PublishArgs: def __init__( self, - configuration: str = 'release', + configuration: str = "release", + cmake_jobs: Optional[int] = None, clean: bool = False, verbosity: int = 1, dry_run: bool = False, output: Optional[str] = None, - framework='net7.0-windows', - runtime='win-x64', + framework="net7.0-windows", + runtime=_dotnet_runtime(), ): self.configuration = configuration + self.cmake_jobs = cmake_jobs self.clean = clean self.verbosity = verbosity self.dry_run = dry_run self.dotnet_verbosity = at_bounded( - ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'], - verbosity) + ["quiet", "minimal", "normal", "detailed", "diagnostic"], verbosity + ) # Leave out WARNING and NOTICE to normalize cmake log level and dotnet verbosity # (so both can be expressed as an int 1-5) self.cmake_log_level = at_bounded( [ - 'ERROR', + "ERROR", # 'WARNING', # 'NOTICE', - 'STATUS', - 'VERBOSE', - 'DEBUG', - 'TRACE', + "STATUS", + "VERBOSE", + "DEBUG", + "TRACE", ], - verbosity) + verbosity, + ) self.output = output self.framework = framework self.runtime = runtime @@ -46,34 +88,38 @@ class PublishArgs: def _get_arg_parser(): parser = argparse.ArgumentParser( - description='Publish a self-contined spark client application') + description="Publish a self-contined spark client application" + ) parser.add_argument( - '--configuration', - metavar='debug|release', - help='The build configuration', + "--configuration", + metavar="debug|release", + help="The build configuration", ) parser.add_argument( - '--clean', - action='store_true', - help='Perform a clean build (remove artifacts from previous build)', - ) - parser.add_argument( - '-o', - '--output', - type=str, - help='The output directory for published artifacts.') - parser.add_argument( - '--verbosity', - metavar='{0-5}', - choices=range(0, 6), + "--cmake-jobs", type=int, - help='The output verbosity level', + help="The number of cmake parallel jobs", ) parser.add_argument( - '--dry-run', - action='store_true', - help='Preview the build commands without running them', + "--clean", + action="store_true", + help="Perform a clean build (remove artifacts from previous build)", + ) + parser.add_argument( + "-o", "--output", type=str, help="The output directory for published artifacts." + ) + parser.add_argument( + "--verbosity", + metavar="{0-5}", + choices=range(1, 6), + type=int, + help="The output verbosity level", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Preview the build commands without running them", ) defaults = PublishArgs() @@ -94,45 +140,39 @@ def publish(args: PublishArgs): return subprocess.run(cmd, shell=True).check_returncode() - def dotnet_cmd(action): - src_dir = SparkPaths.root().joinpath('src', 'client', 'spark') - return ' '.join( - [ - f'dotnet {action} {src_dir}', - f'--configuration {args.configuration.title()}', - f'--verbosity {args.dotnet_verbosity}', - ]) - - os.chdir(SparkPaths.root()) + csproj = SparkPaths.root().joinpath("src", "client", "spark", "Spark.csproj") + cmake_preset = _cmake_preset() + cmake_config = args.configuration.title() + cmake_jobs = args.cmake_jobs + cmake_verbosity = _cmake_verbosity(args.verbosity) + dotnet_config = args.configuration.title() + dotnet_verbosity = _dotnet_verbosity(args.verbosity) publish_dir = SparkPaths.default_build().joinpath( - args.configuration.title(), - 'bin', + dotnet_config, + "bin", args.framework, args.runtime, - 'publish', + "publish", ) - cmake_configure_cmd = ' '.join( - [ - f'cmake', - f'--preset spark', - f'-DCMAKE_MESSAGE_LOG_LEVEL={args.cmake_log_level}', - f'-DDOTNET_WINDOWS_BINARY_DIR={publish_dir}', - ]) - cmake_build_cmd = ' '.join( - [ - f'cmake', - f'--build', - f'--preset spark-{args.configuration.lower()}', - ]) + os.chdir(SparkPaths.root()) if args.clean: - run(f'{cmake_build_cmd} --target clean') - run(dotnet_cmd('clean')) - run(cmake_configure_cmd) + run( + f"cmake --build build/{cmake_preset} --config {cmake_config} --target clean" + ) + run(f"dotnet clean {csproj} -c {dotnet_config} -v {dotnet_verbosity}") + run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}") + cmake_build_cmd = ( + f"cmake --build build/{cmake_preset} --config {cmake_config} --target install" + ) + if cmake_jobs: + cmake_build_cmd += f" -j {cmake_jobs}" run(cmake_build_cmd) - publish_cmd = f'{dotnet_cmd("publish")} -f {args.framework} -r {args.runtime} --sc' - run(publish_cmd) + run( + f"dotnet publish {csproj} -c {dotnet_config} -v {dotnet_verbosity} -f {args.framework} -r {args.runtime} --sc" + ) + copytree(f"install/{cmake_preset}/lib", publish_dir, dirs_exist_ok=True) if args.output: - run(f'mv {publish_dir} {args.output}') + os.rename(publish_dir, args.output) def _main(): @@ -145,8 +185,13 @@ def _main(): verbosity=args.verbosity, dry_run=args.dry_run, output=args.output, - )) + ) + ) -if __name__ == '__main__': - _main() +if __name__ == "__main__": + try: + _main() + except Exception as e: + print(f"publish_api_server.py failed: {e}") + exit(1) diff --git a/tools/py/publish_sparkd.py b/tools/py/publish_sparkd.py new file mode 100644 index 000000000..4f136558f --- /dev/null +++ b/tools/py/publish_sparkd.py @@ -0,0 +1,157 @@ +import argparse +import os +import subprocess + +from pathlib import Path +from typing import Optional + +from build_spark import SparkPaths +from core.iter_utils import at_bounded + + +def _cmake_verbosity(level): + # Skip 'WARNING' and 'NOTICE' levels, since the default CMake verbosity, + # 'STATUS', doesn't produce that much output. This also better normalizes + # numeric representation of verbosity w/ dotnet, which only has 5 levels. + names = [ + "ERROR", + # 'WARNING', + # 'NOTICE', + "STATUS", + "VERBOSE", + "DEBUG", + "TRACE", + ] + return at_bounded(names, level - 1) + + +def _cmake_preset(configuration: str): + return ( + "sparkd-debug-clang" + if configuration.lower() == "debug" + else "sparkd-release-gcc" + ) + + +class PublishArgs: + def __init__( + self, + configuration: str = "release", + cmake_jobs: Optional[int] = None, + clean: bool = False, + verbosity: int = 2, + dry_run: bool = False, + output: Optional[str] = None, + ): + self.configuration = configuration + self.cmake_jobs = cmake_jobs + self.clean = clean + self.verbosity = verbosity + self.dry_run = dry_run + # Leave out WARNING and NOTICE to normalize cmake log level and dotnet verbosity + # (so both can be expressed as an int 1-5) + self.cmake_log_level = at_bounded( + [ + "ERROR", + # 'WARNING', + # 'NOTICE', + "STATUS", + "VERBOSE", + "DEBUG", + "TRACE", + ], + verbosity, + ) + self.output = output + + +def _get_arg_parser(): + parser = argparse.ArgumentParser( + description="Publish a self-contined spark client application" + ) + + parser.add_argument( + "--configuration", + metavar="debug|release", + help="The build configuration", + ) + parser.add_argument( + "--cmake-jobs", + type=int, + help="The number of cmake parallel jobs", + ) + parser.add_argument( + "--clean", + action="store_true", + help="Perform a clean build (remove artifacts from previous build)", + ) + parser.add_argument( + "-o", "--output", type=str, help="The output directory for published artifacts." + ) + parser.add_argument( + "--verbosity", + metavar="{0-5}", + choices=range(1, 6), + type=int, + help="The output verbosity level", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Preview the build commands without running them", + ) + + defaults = PublishArgs() + parser.set_defaults( + configuration=defaults.configuration, + clean=defaults.clean, + output=defaults.output, + verbosity=defaults.verbosity, + dry_run=defaults.dry_run, + ) + return parser + + +def publish(args): + def run(cmd): + print(cmd) + if args.dry_run: + return + subprocess.run(cmd, shell=True).check_returncode() + + cmake_preset = _cmake_preset(args.configuration) + cmake_jobs = args.cmake_jobs + cmake_verbosity = _cmake_verbosity(args.verbosity) + os.chdir(SparkPaths.root()) + if args.clean: + run(f"cmake --build build/{cmake_preset} --target clean") + run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}") + cmake_build_cmd = f"cmake --build build/{cmake_preset} --target install" + if cmake_jobs: + cmake_build_cmd += f" -j {cmake_jobs}" + run(cmake_build_cmd) + if args.output: + os.rename(Path("install").joinpath(cmake_preset), args.output) + + +def _main(): + parser = _get_arg_parser() + args = parser.parse_args() + publish( + PublishArgs( + configuration=args.configuration, + cmake_jobs=args.cmake_jobs, + clean=args.clean, + verbosity=args.verbosity, + dry_run=args.dry_run, + output=args.output, + ) + ) + + +if __name__ == "__main__": + try: + _main() + except Exception as e: + print(f"publish_sparkd.py failed: {e}") + exit(1)