Clean up build scripts

This commit is contained in:
John Colvin committed 2024-11-26 18:53:32 +00:00
1 parent 73d670c0dc
commit 75907e2bd5
9 files changed
+584 -403

No files matched your search

+48 -59
View File
@@ -8,31 +8,38 @@ from core.iter_utils import at_bounded
def _get_arg_parser():
parser = argparse.ArgumentParser(description='Build the Spark Api Server (websockets)')
parser.add_argument(
'--configuration',
metavar='debug|release',
help='The build configuration',
parser = argparse.ArgumentParser(
description="Build the Spark Api Server (websockets)"
)
parser.add_argument(
'--clean',
action='store_true',
help='Perform a clean build (remove artifacts from previous build)',
"--configuration",
metavar="debug|release",
help="The build configuration",
)
parser.add_argument(
'--verbosity',
metavar='{0-5}',
"--cmake-jobs",
type=int,
help="The number of cmake parallel jobs",
)
parser.add_argument(
"--clean",
action="store_true",
help="Perform a clean build (remove artifacts from previous build)",
)
parser.add_argument(
"--verbosity",
metavar="{0-5}",
choices=range(0, 6),
type=int,
help='The output verbosity level',
help="The output verbosity level",
)
parser.add_argument(
'--dry-run',
action='store_true',
help='Preview the build commands without running them',
"--dry-run",
action="store_true",
help="Preview the build commands without running them",
)
parser.set_defaults(
configuration='debug',
configuration="debug",
clean=False,
verbosity=1,
)
@@ -40,7 +47,7 @@ def _get_arg_parser():
def _dotnet_verbosity(level):
names = ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic']
names = ["quiet", "minimal", "normal", "detailed", "diagnostic"]
return at_bounded(names, level)
@@ -49,33 +56,25 @@ def _cmake_verbosity(level):
# 'STATUS', doesn't produce that much output. This also better normalizes
# numeric representation of verbosity w/ dotnet, which only has 5 levels.
names = [
'ERROR',
"ERROR",
# 'WARNING',
# 'NOTICE',
'STATUS',
'VERBOSE',
'DEBUG',
'TRACE',
"STATUS",
"VERBOSE",
"DEBUG",
"TRACE",
]
return at_bounded(names, level)
def _cmake_preset():
match sys.platform:
case 'win32':
return 'spark-api-server-windows'
case 'linux':
return 'spark-api-server-linux'
case "win32":
return "spark-api-server-windows"
case "linux":
return "spark-api-server-linux"
case _:
raise ValueError(f'Unsupported platform {sys.platform}')
def _cmake_build_preset(configuration):
match sys.platform:
case 'win32':
return f'spark-api-server-windows-{configuration}'
case 'linux':
return f'spark-api-server-linux-{configuration}'
case _:
raise ValueError(f'Unsupported platform {sys.platform}')
raise ValueError(f"Unsupported platform {sys.platform}")
def _main():
@@ -88,37 +87,27 @@ def _main():
return
subprocess.run(cmd, shell=True).check_returncode()
def dotnet_cmd(action='build'):
sln = SparkPaths.root().joinpath('Spark.Api.Server.sln')
return ' '.join([
f'dotnet {action} {sln}',
f'--configuration {args.configuration.title()}',
f'--verbosity {_dotnet_verbosity(args.verbosity)}',
])
sln = SparkPaths.root().joinpath("Spark.Api.Server.sln")
cmake_preset = _cmake_preset()
cmake_jobs = args.cmake_jobs
cmake_verbosity = _cmake_verbosity(args.verbosity)
dotnet_config = args.configuration.title()
dotnet_verbosity = _dotnet_verbosity(args.verbosity)
os.chdir(SparkPaths.root())
cmake_configure_cmd = ' '.join([
f'cmake',
f'--preset',
_cmake_preset(),
f'-DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}',
])
cmake_build_cmd = ' '.join([
f'cmake',
f'--build',
f'--preset {_cmake_build_preset(args.configuration.lower())}',
])
if args.clean:
run(f'{cmake_build_cmd} --target clean')
run(dotnet_cmd('clean'))
run(cmake_configure_cmd)
run(f"cmake --build build/{cmake_preset} --target clean")
run(f"dotnet clean {sln} -c {dotnet_config} -v {dotnet_verbosity}")
run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}")
cmake_build_cmd = f"cmake --build build/{cmake_preset}"
if cmake_jobs:
cmake_build_cmd += f" -j {cmake_jobs}"
run(cmake_build_cmd)
run(dotnet_cmd('build'))
run(f"dotnet build {sln} -c {dotnet_config} -v {dotnet_verbosity}")
if __name__ == "__main__":
try:
_main()
except Exception as e:
print(f'build_api_server.py failed: {e}')
print(f"build_api_server.py failed: {e}")
exit(1)
+54 -66
View File
@@ -8,31 +8,36 @@ from core.iter_utils import at_bounded
def _get_arg_parser():
parser = argparse.ArgumentParser(description='Build DataStreamSuite.sln')
parser = argparse.ArgumentParser(description="Build DataStreamSuite.sln")
parser.add_argument(
'--configuration',
metavar='debug|release',
help='The build configuration',
"--configuration",
metavar="debug|release",
help="The build configuration",
)
parser.add_argument(
'--clean',
action='store_true',
help='Perform a clean build (remove artifacts from previous build)',
"--cmake-jobs",
type=int,
help="The number of cmake parallel jobs",
)
parser.add_argument(
'--verbosity',
metavar='{0-5}',
"--clean",
action="store_true",
help="Perform a clean build (remove artifacts from previous build)",
)
parser.add_argument(
"--verbosity",
metavar="{0-5}",
choices=range(0, 6),
type=int,
help='The output verbosity level',
help="The output verbosity level",
)
parser.add_argument(
'--dry-run',
action='store_true',
help='Preview the build commands without running them',
"--dry-run",
action="store_true",
help="Preview the build commands without running them",
)
parser.set_defaults(
configuration='debug',
configuration="debug",
clean=False,
verbosity=1,
)
@@ -40,7 +45,7 @@ def _get_arg_parser():
def _dotnet_verbosity(level):
names = ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic']
names = ["quiet", "minimal", "normal", "detailed", "diagnostic"]
return at_bounded(names, level)
@@ -49,77 +54,60 @@ def _cmake_verbosity(level):
# 'STATUS', doesn't produce that much output. This also better normalizes
# numeric representation of verbosity w/ dotnet, which only has 5 levels.
names = [
'ERROR',
"ERROR",
# 'WARNING',
# 'NOTICE',
'STATUS',
'VERBOSE',
'DEBUG',
'TRACE',
"STATUS",
"VERBOSE",
"DEBUG",
"TRACE",
]
return at_bounded(names, level)
def _cmake_preset():
match sys.platform:
case 'win32':
return 'data-stream-suite-windows'
case 'linux':
return 'data-stream-suite-linux'
case "win32":
return "data-stream-suite-windows"
case "linux":
return "data-stream-suite-linux"
case _:
raise ValueError(f'Unsupported platform {sys.platform}')
raise ValueError(f"Unsupported platform {sys.platform}")
def _cmake_build_preset(configuration):
match sys.platform:
case 'win32':
return f'data-stream-suite-windows-{configuration}'
case 'linux':
return f'data-stream-suite-linux-{configuration}'
case _:
raise ValueError(f'Unsupported platform {sys.platform}')
def _main():
parser = _get_arg_parser()
args = parser.parse_args()
def build(args):
def run(cmd):
print(cmd)
if args.dry_run:
return
subprocess.run(cmd, shell=True).check_returncode()
def dotnet_cmd(action='build'):
sln = SparkPaths.root().joinpath(
'src',
'DataStreamSuite',
'DataStreamSuite.sln',
)
return ' '.join([
f'dotnet {action} {sln}',
f'--configuration {args.configuration.title()}',
f'--verbosity {_dotnet_verbosity(args.verbosity)}',
])
sln = SparkPaths.root().joinpath(
"src",
"DataStreamSuite",
"DataStreamSuite.sln",
)
cmake_preset = _cmake_preset()
cmake_jobs = args.cmake_jobs
cmake_verbosity = _cmake_verbosity(args.verbosity)
dotnet_config = args.configuration.title()
dotnet_verbosity = _dotnet_verbosity(args.verbosity)
os.chdir(SparkPaths.root())
cmake_configure_cmd = ' '.join([
f'cmake',
f'--preset',
_cmake_preset(),
f'-DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}',
])
cmake_build_cmd = ' '.join([
f'cmake',
f'--build',
f'--preset {_cmake_build_preset(args.configuration.lower())}',
])
if args.clean:
run(f'{cmake_build_cmd} --target clean')
run(dotnet_cmd('clean'))
run(cmake_configure_cmd)
run(f"cmake --build build/{cmake_preset} --target clean")
run(f"dotnet clean {sln} -c {dotnet_config} -v {dotnet_verbosity}")
run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}")
cmake_build_cmd = f"cmake --build build/{cmake_preset}"
if cmake_jobs:
cmake_build_cmd += f" -j {cmake_jobs}"
run(cmake_build_cmd)
run(dotnet_cmd('build'))
run(f"dotnet build {sln} -c {dotnet_config} -v {dotnet_verbosity}")
def _main():
parser = _get_arg_parser()
args = parser.parse_args()
build(args)
if __name__ == "__main__":
+13 -16
View File
@@ -90,25 +90,22 @@ def _main():
return
subprocess.run(cmd, shell=True).check_returncode()
def dotnet_cmd(action):
sln = SparkPaths.root().joinpath(
"src", "DataStreamSuite", "DataStreamQueryAnalyzer.sln"
)
cfg = args.configuration.title()
verbosity = _dotnet_verbosity(args.verbosity)
return f"dotnet {action} {sln} --configuration {cfg} --verbosity {verbosity}"
os.chdir(SparkPaths.root())
cmake_configure_cmd = f"cmake --preset {_cmake_preset()} -DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}"
cmake_build_cmd = (
f"cmake --build --preset {_cmake_build_preset(args.configuration.lower())}"
sln = SparkPaths.root().joinpath(
"src", "DataStreamSuite", "DataStreamQueryAnalyzer.sln"
)
cmake_preset = _cmake_preset()
cmake_build_preset = _cmake_build_preset(args.configuration)
cmake_verbosity = _cmake_verbosity(args.verbosity)
dotnet_config = args.configuration.title()
dotnet_verbosity = _dotnet_verbosity(args.verbosity)
os.chdir(SparkPaths.root())
if args.clean:
run(f"{cmake_build_cmd} --target clean")
run(dotnet_cmd("clean"))
run(cmake_configure_cmd)
run(cmake_build_cmd)
run(dotnet_cmd("build"))
run(f"cmake --build --preset {cmake_build_preset} --target clean")
run(f"dotnet clean {sln} -c {dotnet_config} -v {dotnet_verbosity}")
run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}")
run(f"cmake --build --preset {cmake_build_preset}")
run(f"dotnet build {sln} -c {dotnet_config} -v {dotnet_verbosity}")
if __name__ == "__main__":
+24 -13
View File
@@ -14,6 +14,11 @@ def _get_arg_parser():
metavar="debug|release",
help="The build configuration",
)
parser.add_argument(
"--cmake-jobs",
type=int,
help="The number of cmake parallel jobs",
)
parser.add_argument(
"--clean",
action="store_true",
@@ -90,23 +95,29 @@ def _main():
return
subprocess.run(cmd, shell=True).check_returncode()
def dotnet_cmd(action="build"):
csproj = SparkPaths.root().joinpath(
"src", "DataStreamSuite", "Bts.DataStreamWebApp.Server", "Bts.DataStreamWebApp.Server.csproj"
)
return f"dotnet {action} {csproj} --configuration {args.configuration.title()} --verbosity {_dotnet_verbosity(args.verbosity)}"
os.chdir(SparkPaths.root())
cmake_configure_cmd = f"cmake --preset {_cmake_preset()} -DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}"
cmake_build_cmd = (
f"cmake --build --preset {_cmake_build_preset(args.configuration.lower())}"
csproj = SparkPaths.root().joinpath(
"src",
"DataStreamSuite",
"Bts.DataStreamWebApp.Server",
"Bts.DataStreamWebApp.Server.csproj",
)
cmake_preset = _cmake_preset()
cmake_build_preset = _cmake_build_preset(args.configuration)
cmake_jobs = args.cmake_jobs
cmake_verbosity = _cmake_verbosity(args.verbosity)
dotnet_config = args.configuration.title()
dotnet_verbosity = _dotnet_verbosity(args.verbosity)
os.chdir(SparkPaths.root())
if args.clean:
run(f"{cmake_build_cmd} --target clean")
run(dotnet_cmd("clean"))
run(cmake_configure_cmd)
run(f"cmake --build --preset {cmake_build_preset} --target clean")
run(f"dotnet clean {csproj} -c {dotnet_config} -v {dotnet_verbosity}")
run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}")
cmake_build_cmd = f"cmake --build --preset {cmake_build_preset}"
if cmake_jobs:
cmake_build_cmd += f" -j {cmake_jobs}"
run(cmake_build_cmd)
run(dotnet_cmd("build"))
run(f"dotnet build {csproj} -c {dotnet_config} -v {dotnet_verbosity}")
if __name__ == "__main__":
+56 -61
View File
@@ -9,49 +9,54 @@ from core.iter_utils import at_bounded
def _get_arg_parser():
parser = argparse.ArgumentParser(
description='Builds the Spark server (A.K.A sparkd, the daemon)')
parser.add_argument(
'--build-type',
metavar='debug|release',
help='The build configuration',
description="Builds the Spark server (A.K.A sparkd, the daemon)"
)
parser.add_argument(
'--compiler',
metavar='compiler',
choices=['gcc', 'clang'],
help='The compiler to use',
"--build-type",
metavar="debug|release",
help="The build configuration",
)
parser.add_argument(
'--preset',
metavar=
'sparkd-debug-clang|sparkd-debug-gcc|sparkd-release-clang|spard-release-gcc',
"--compiler",
metavar="compiler",
choices=["gcc", "clang"],
help="The compiler to use",
)
parser.add_argument(
"--preset",
metavar="sparkd-debug-clang|sparkd-debug-gcc|sparkd-release-clang|spard-release-gcc",
choices=[
'sparkd-debug-clang',
'sparkd-debug-gcc',
'sparkd-release-clang',
'sparkd-release-gcc',
"sparkd-debug-clang",
"sparkd-debug-gcc",
"sparkd-release-clang",
"sparkd-release-gcc",
],
help='The CMake preset to use',
help="The CMake preset to use",
)
parser.add_argument(
'--clean',
action='store_true',
help='Perform a clean build (remove artifacts from previous build)',
"--cmake-jobs",
type=int,
help="The number of cmake parallel jobs",
)
parser.add_argument(
'--verbosity',
metavar='{0-5}',
"--clean",
action="store_true",
help="Perform a clean build (remove artifacts from previous build)",
)
parser.add_argument(
"--verbosity",
metavar="{0-5}",
choices=range(0, 6),
type=int,
help='The output verbosity level',
help="The output verbosity level",
)
parser.add_argument(
'--dry-run',
action='store_true',
help='Preview the build commands without running them',
"--dry-run",
action="store_true",
help="Preview the build commands without running them",
)
parser.set_defaults(
build_type='debug',
build_type="debug",
compiler=None,
preset=None,
clean=False,
@@ -65,37 +70,31 @@ def _cmake_verbosity(level):
# 'STATUS', doesn't produce that much output. This also better normalizes
# numeric representation of verbosity w/ dotnet, which only has 5 levels.
names = [
'ERROR',
"ERROR",
# 'WARNING',
# 'NOTICE',
'STATUS',
'VERBOSE',
'DEBUG',
'TRACE',
"STATUS",
"VERBOSE",
"DEBUG",
"TRACE",
]
return at_bounded(names, level)
def _cmake_compiler(build_type: str):
return 'clang' if build_type.lower() == 'debug' else 'gcc'
return "clang" if build_type.lower() == "debug" else "gcc"
def _cmake_preset(compiler: str, build_type: str):
return f'sparkd-{build_type}-{compiler}'
def _cmake_build_preset(
compiler: Optional[str] = None,
build_type: Optional[str] = None,
configure_preset: Optional[str] = None,
def _cmake_preset(
preset: Optional[str], compiler: Optional[str], build_type: Optional[str]
):
if configure_preset:
return configure_preset
if preset:
return preset
if not build_type:
build_type = 'debug'
build_type = "debug"
if not compiler:
compiler = 'clang' if build_type.lower() == 'debug' else 'gcc'
return f'sparkd-{build_type}-{compiler}'
compiler = _cmake_compiler(build_type)
return f"sparkd-{build_type}-{compiler}"
def _main():
@@ -108,26 +107,22 @@ def _main():
return
subprocess.run(cmd, shell=True).check_returncode()
cmake_preset = _cmake_preset(args.preset, args.compiler, args.build_type)
cmake_jobs: Optional[int] = args.cmake_jobs
cmake_verbosity = _cmake_verbosity(args.verbosity)
os.chdir(SparkPaths.root())
build_type: str = args.build_type
compiler: str = args.compiler or _cmake_compiler(build_type)
preset = args.preset or _cmake_preset(compiler.lower(), build_type.lower())
build_preset = _cmake_build_preset(
compiler.lower(),
build_type.lower(),
args.preset,
)
verbosity = _cmake_verbosity(args.verbosity)
configure_cmd = f'cmake --preset {preset} -DCMAKE_MESSAGE_LOG_LEVEL={verbosity}'
build_cmd = f'cmake --build --preset {build_preset}'
if args.clean:
run(f'{build_cmd} --target clean')
run(configure_cmd)
run(build_cmd)
run(f"cmake --build build/{cmake_preset} --target clean")
run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}")
cmake_build_cmd = f"cmake --build build/{cmake_preset}"
if cmake_jobs:
cmake_build_cmd += f" -j {cmake_jobs}"
run(cmake_build_cmd)
if __name__ == "__main__":
try:
_main()
except Exception as e:
print(f'build_sparkd.py failed: {e}')
print(f"build_sparkd.py failed: {e}")
exit(1)
+28 -25
View File
@@ -12,7 +12,7 @@ from core.iter_utils import at_bounded
def _dotnet_verbosity(level):
names = ["quiet", "minimal", "normal", "detailed", "diagnostic"]
return at_bounded(names, level)
return at_bounded(names, level - 1)
def _cmake_verbosity(level):
@@ -28,7 +28,7 @@ def _cmake_verbosity(level):
"DEBUG",
"TRACE",
]
return at_bounded(names, level)
return at_bounded(names, level - 1)
def _cmake_preset():
@@ -41,16 +41,6 @@ def _cmake_preset():
raise ValueError(f"Unsupported platform {sys.platform}")
def _cmake_build_preset(configuration):
match sys.platform:
case "win32":
return f"spark-api-server-windows-{configuration}"
case "linux":
return f"spark-api-server-linux-{configuration}"
case _:
raise ValueError(f"Unsupported platform {sys.platform}")
def _dotnet_runtime():
match sys.platform:
case "win32":
@@ -65,14 +55,16 @@ class PublishArgs:
def __init__(
self,
configuration: str = "release",
cmake_jobs: Optional[int] = None,
clean: bool = False,
verbosity: int = 1,
verbosity: int = 2,
dry_run: bool = False,
output: Optional[str] = None,
framework="net7.0",
runtime=_dotnet_runtime(),
):
self.configuration = configuration
self.cmake_jobs = cmake_jobs
self.clean = clean
self.verbosity = verbosity
self.dry_run = dry_run
@@ -108,6 +100,11 @@ def _get_arg_parser():
metavar="debug|release",
help="The build configuration",
)
parser.add_argument(
"--cmake-jobs",
type=int,
help="The number of cmake parallel jobs",
)
parser.add_argument(
"--clean",
action="store_true",
@@ -119,7 +116,7 @@ def _get_arg_parser():
parser.add_argument(
"--verbosity",
metavar="{0-5}",
choices=range(0, 6),
choices=range(1, 6),
type=int,
help="The output verbosity level",
)
@@ -148,32 +145,38 @@ def publish(args: PublishArgs):
subprocess.run(cmd, shell=True).check_returncode()
sln = SparkPaths.root().joinpath("Spark.Api.Server.sln")
dotnet_cfg = args.configuration.title()
cmake_preset = _cmake_preset()
cmake_config = args.configuration.title()
cmake_jobs = args.cmake_jobs
cmake_verbosity = _cmake_verbosity(args.verbosity)
dotnet_config = args.configuration.title()
dotnet_verbosity = _dotnet_verbosity(args.verbosity)
os.chdir(SparkPaths.root())
publish_dir = SparkPaths.default_build().joinpath(
args.configuration.title(),
dotnet_config,
"bin",
args.framework,
args.runtime,
"publish",
)
cmake_preset = _cmake_preset()
cmake_build_preset = _cmake_build_preset(args.configuration.lower())
cmake_verbosity = _cmake_verbosity(args.verbosity)
os.chdir(SparkPaths.root())
if args.clean:
run(
f"cmake --build --preset {cmake_build_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity} --target clean"
f"cmake --build build/{cmake_preset} --config {cmake_config} --target clean"
)
run(f"dotnet clean {sln} -c {dotnet_cfg} -v {dotnet_verbosity}")
run(f"dotnet clean {sln} -c {dotnet_config} -v {dotnet_verbosity}")
run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}")
run(f"cmake --build --preset {cmake_build_preset} --target install")
cmake_build_cmd = (
f"cmake --build build/{cmake_preset} --config {cmake_config} --target install"
)
if cmake_jobs:
cmake_build_cmd += f" -j {cmake_jobs}"
run(cmake_build_cmd)
run(
f"dotnet publish {sln} -c {dotnet_cfg} -v {dotnet_verbosity} -f {args.framework} -r {args.runtime} --sc"
f"dotnet publish {sln} -c {dotnet_config} -v {dotnet_verbosity} -f {args.framework} -r {args.runtime} --sc"
)
copytree(f"install/{cmake_preset}/lib", publish_dir, dirs_exist_ok=True)
if args.output:
run(f"mv {publish_dir} {args.output}")
os.rename(publish_dir, args.output)
def _main():
+92 -96
View File
@@ -3,6 +3,7 @@ import os
import subprocess
import sys
from shutil import copytree
from typing import Optional
from build_spark import SparkPaths
@@ -10,8 +11,8 @@ from core.iter_utils import at_bounded
def _dotnet_verbosity(level):
names = ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic']
return at_bounded(names, level)
names = ["quiet", "minimal", "normal", "detailed", "diagnostic"]
return at_bounded(names, level - 1)
def _cmake_verbosity(level):
@@ -19,78 +20,71 @@ def _cmake_verbosity(level):
# 'STATUS', doesn't produce that much output. This also better normalizes
# numeric representation of verbosity w/ dotnet, which only has 5 levels.
names = [
'ERROR',
"ERROR",
# 'WARNING',
# 'NOTICE',
'STATUS',
'VERBOSE',
'DEBUG',
'TRACE',
"STATUS",
"VERBOSE",
"DEBUG",
"TRACE",
]
return at_bounded(names, level)
return at_bounded(names, level - 1)
def _cmake_preset():
match sys.platform:
case 'win32':
return 'data-stream-suite-windows'
case 'linux':
return 'data-stream-suite-linux'
case "win32":
return "data-stream-suite-windows"
case "linux":
return "data-stream-suite-linux"
case _:
raise ValueError(f'Unsupported platform {sys.platform}')
def _cmake_build_preset(configuration):
match sys.platform:
case 'win32':
return f'data-stream-suite-windows-{configuration}'
case 'linux':
return f'data-stream-suite-linux-{configuration}'
case _:
raise ValueError(f'Unsupported platform {sys.platform}')
raise ValueError(f"Unsupported platform {sys.platform}")
def _dotnet_runtime():
match sys.platform:
case 'win32':
return 'win-x64'
case 'linux':
return 'linux-x64'
case "win32":
return "win-x64"
case "linux":
return "linux-x64"
case _:
raise ValueError(f'Unsupported platform {sys.platform}')
raise ValueError(f"Unsupported platform {sys.platform}")
class PublishArgs:
def __init__(
self,
configuration: str = 'release',
configuration: str = "release",
cmake_jobs: Optional[int] = None,
clean: bool = False,
verbosity: int = 1,
verbosity: int = 2,
dry_run: bool = False,
output: Optional[str] = None,
framework='net7.0',
framework="net7.0",
runtime=_dotnet_runtime(),
):
self.configuration = configuration
self.cmake_jobs = cmake_jobs
self.clean = clean
self.verbosity = verbosity
self.dry_run = dry_run
self.dotnet_verbosity = at_bounded(
['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'],
verbosity)
["quiet", "minimal", "normal", "detailed", "diagnostic"], verbosity
)
# Leave out WARNING and NOTICE to normalize cmake log level and dotnet verbosity
# (so both can be expressed as an int 1-5)
self.cmake_log_level = at_bounded(
[
'ERROR',
"ERROR",
# 'WARNING',
# 'NOTICE',
'STATUS',
'VERBOSE',
'DEBUG',
'TRACE',
"STATUS",
"VERBOSE",
"DEBUG",
"TRACE",
],
verbosity)
verbosity,
)
self.output = output
self.framework = framework
self.runtime = runtime
@@ -98,34 +92,38 @@ class PublishArgs:
def _get_arg_parser():
parser = argparse.ArgumentParser(
description='Publish a self-contined spark client application')
description="Publish a self-contined spark client application"
)
parser.add_argument(
'--configuration',
metavar='debug|release',
help='The build configuration',
"--configuration",
metavar="debug|release",
help="The build configuration",
)
parser.add_argument(
'--clean',
action='store_true',
help='Perform a clean build (remove artifacts from previous build)',
)
parser.add_argument(
'-o',
'--output',
type=str,
help='The output directory for published artifacts.')
parser.add_argument(
'--verbosity',
metavar='{0-5}',
choices=range(0, 6),
"--cmake-jobs",
type=int,
help='The output verbosity level',
help="The number of cmake parallel jobs",
)
parser.add_argument(
'--dry-run',
action='store_true',
help='Preview the build commands without running them',
"--clean",
action="store_true",
help="Perform a clean build (remove artifacts from previous build)",
)
parser.add_argument(
"-o", "--output", type=str, help="The output directory for published artifacts."
)
parser.add_argument(
"--verbosity",
metavar="{1-5}",
choices=range(1, 6),
type=int,
help="The output verbosity level",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Preview the build commands without running them",
)
defaults = PublishArgs()
@@ -139,54 +137,51 @@ def _get_arg_parser():
return parser
def publish(args: PublishArgs):
def publish(args):
def run(cmd):
print(cmd)
if args.dry_run:
return
subprocess.run(cmd, shell=True).check_returncode()
def dotnet_cmd(action='build'):
sln = SparkPaths.root().joinpath(
'src',
'DataStreamSuite',
'DataStreamSuite.sln',
)
return ' '.join([
f'dotnet {action} {sln}',
f'--configuration {args.configuration.title()}',
f'--verbosity {_dotnet_verbosity(args.verbosity)}',
])
os.chdir(SparkPaths.root())
sln = SparkPaths.root().joinpath(
"src",
"DataStreamSuite",
"DataStreamSuite.sln",
)
cmake_preset = _cmake_preset()
cmake_config = args.configuration.title()
cmake_jobs = args.cmake_jobs
cmake_verbosity = _cmake_verbosity(args.verbosity)
dotnet_config = args.configuration.title()
dotnet_verbosity = _dotnet_verbosity(args.verbosity)
publish_dir = SparkPaths.default_build().joinpath(
args.configuration.title(),
'bin',
"bin",
args.framework,
args.runtime,
'publish',
"publish",
)
cmake_configure_cmd = ' '.join([
f'cmake',
f'--preset',
_cmake_preset(),
f'-DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}',
f'-DDOTNET_BINARY_DIR={publish_dir}',
])
cmake_build_cmd = ' '.join([
f'cmake',
f'--build',
f'--preset {_cmake_build_preset(args.configuration.lower())}',
])
os.chdir(SparkPaths.root())
if args.clean:
run(f'{cmake_build_cmd} --target clean')
run(dotnet_cmd('clean'))
run(cmake_configure_cmd)
run(
f"cmake --build build/{cmake_preset} --config {cmake_config} --target clean"
)
run(f"dotnet clean {sln} -c {dotnet_config} -v {dotnet_verbosity}")
run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}")
cmake_build_cmd = (
f"cmake --build build/{cmake_preset} --config {cmake_config} --target install"
)
if cmake_jobs:
cmake_build_cmd += f" -j {cmake_jobs}"
run(cmake_build_cmd)
publish_cmd = f'{dotnet_cmd("publish")} -f {args.framework} -r {args.runtime} --sc'
run(publish_cmd)
run(f"dotnet build {sln} -c {dotnet_config} -v {dotnet_verbosity}")
run(
f"dotnet publish {sln} -c {dotnet_config} -v {dotnet_verbosity} -f {args.framework} -r {args.runtime} --sc"
)
copytree(f"./install/{cmake_preset}/lib", publish_dir, dirs_exist_ok=True)
if args.output:
run(f'mv {publish_dir} {args.output}')
os.rename(publish_dir, args.output)
def _main():
@@ -199,10 +194,11 @@ def _main():
verbosity=args.verbosity,
dry_run=args.dry_run,
output=args.output,
))
)
)
if __name__ == '__main__':
if __name__ == "__main__":
try:
_main()
except Exception as e:
+112 -67
View File
@@ -1,44 +1,86 @@
import argparse
import os
import subprocess
import sys
from shutil import copytree
from typing import Optional
from build_spark import SparkPaths
from core.iter_utils import at_bounded
def _dotnet_verbosity(level):
names = ["quiet", "minimal", "normal", "detailed", "diagnostic"]
return at_bounded(names, level - 1)
def _cmake_verbosity(level):
# Skip 'WARNING' and 'NOTICE' levels, since the default CMake verbosity,
# 'STATUS', doesn't produce that much output. This also better normalizes
# numeric representation of verbosity w/ dotnet, which only has 5 levels.
names = [
"ERROR",
# 'WARNING',
# 'NOTICE',
"STATUS",
"VERBOSE",
"DEBUG",
"TRACE",
]
return at_bounded(names, level - 1)
def _cmake_preset():
match sys.platform:
case "win32":
return "spark"
case _:
raise ValueError(f"Unsupported platform {sys.platform}")
def _dotnet_runtime():
match sys.platform:
case "win32":
return "win-x64"
case _:
raise ValueError(f"Unsupported platform {sys.platform}")
class PublishArgs:
def __init__(
self,
configuration: str = 'release',
configuration: str = "release",
cmake_jobs: Optional[int] = None,
clean: bool = False,
verbosity: int = 1,
dry_run: bool = False,
output: Optional[str] = None,
framework='net7.0-windows',
runtime='win-x64',
framework="net7.0-windows",
runtime=_dotnet_runtime(),
):
self.configuration = configuration
self.cmake_jobs = cmake_jobs
self.clean = clean
self.verbosity = verbosity
self.dry_run = dry_run
self.dotnet_verbosity = at_bounded(
['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'],
verbosity)
["quiet", "minimal", "normal", "detailed", "diagnostic"], verbosity
)
# Leave out WARNING and NOTICE to normalize cmake log level and dotnet verbosity
# (so both can be expressed as an int 1-5)
self.cmake_log_level = at_bounded(
[
'ERROR',
"ERROR",
# 'WARNING',
# 'NOTICE',
'STATUS',
'VERBOSE',
'DEBUG',
'TRACE',
"STATUS",
"VERBOSE",
"DEBUG",
"TRACE",
],
verbosity)
verbosity,
)
self.output = output
self.framework = framework
self.runtime = runtime
@@ -46,34 +88,38 @@ class PublishArgs:
def _get_arg_parser():
parser = argparse.ArgumentParser(
description='Publish a self-contined spark client application')
description="Publish a self-contined spark client application"
)
parser.add_argument(
'--configuration',
metavar='debug|release',
help='The build configuration',
"--configuration",
metavar="debug|release",
help="The build configuration",
)
parser.add_argument(
'--clean',
action='store_true',
help='Perform a clean build (remove artifacts from previous build)',
)
parser.add_argument(
'-o',
'--output',
type=str,
help='The output directory for published artifacts.')
parser.add_argument(
'--verbosity',
metavar='{0-5}',
choices=range(0, 6),
"--cmake-jobs",
type=int,
help='The output verbosity level',
help="The number of cmake parallel jobs",
)
parser.add_argument(
'--dry-run',
action='store_true',
help='Preview the build commands without running them',
"--clean",
action="store_true",
help="Perform a clean build (remove artifacts from previous build)",
)
parser.add_argument(
"-o", "--output", type=str, help="The output directory for published artifacts."
)
parser.add_argument(
"--verbosity",
metavar="{0-5}",
choices=range(1, 6),
type=int,
help="The output verbosity level",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Preview the build commands without running them",
)
defaults = PublishArgs()
@@ -94,45 +140,39 @@ def publish(args: PublishArgs):
return
subprocess.run(cmd, shell=True).check_returncode()
def dotnet_cmd(action):
src_dir = SparkPaths.root().joinpath('src', 'client', 'spark')
return ' '.join(
[
f'dotnet {action} {src_dir}',
f'--configuration {args.configuration.title()}',
f'--verbosity {args.dotnet_verbosity}',
])
os.chdir(SparkPaths.root())
csproj = SparkPaths.root().joinpath("src", "client", "spark", "Spark.csproj")
cmake_preset = _cmake_preset()
cmake_config = args.configuration.title()
cmake_jobs = args.cmake_jobs
cmake_verbosity = _cmake_verbosity(args.verbosity)
dotnet_config = args.configuration.title()
dotnet_verbosity = _dotnet_verbosity(args.verbosity)
publish_dir = SparkPaths.default_build().joinpath(
args.configuration.title(),
'bin',
dotnet_config,
"bin",
args.framework,
args.runtime,
'publish',
"publish",
)
cmake_configure_cmd = ' '.join(
[
f'cmake',
f'--preset spark',
f'-DCMAKE_MESSAGE_LOG_LEVEL={args.cmake_log_level}',
f'-DDOTNET_WINDOWS_BINARY_DIR={publish_dir}',
])
cmake_build_cmd = ' '.join(
[
f'cmake',
f'--build',
f'--preset spark-{args.configuration.lower()}',
])
os.chdir(SparkPaths.root())
if args.clean:
run(f'{cmake_build_cmd} --target clean')
run(dotnet_cmd('clean'))
run(cmake_configure_cmd)
run(
f"cmake --build build/{cmake_preset} --config {cmake_config} --target clean"
)
run(f"dotnet clean {csproj} -c {dotnet_config} -v {dotnet_verbosity}")
run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}")
cmake_build_cmd = (
f"cmake --build build/{cmake_preset} --config {cmake_config} --target install"
)
if cmake_jobs:
cmake_build_cmd += f" -j {cmake_jobs}"
run(cmake_build_cmd)
publish_cmd = f'{dotnet_cmd("publish")} -f {args.framework} -r {args.runtime} --sc'
run(publish_cmd)
run(
f"dotnet publish {csproj} -c {dotnet_config} -v {dotnet_verbosity} -f {args.framework} -r {args.runtime} --sc"
)
copytree(f"install/{cmake_preset}/lib", publish_dir, dirs_exist_ok=True)
if args.output:
run(f'mv {publish_dir} {args.output}')
os.rename(publish_dir, args.output)
def _main():
@@ -145,8 +185,13 @@ def _main():
verbosity=args.verbosity,
dry_run=args.dry_run,
output=args.output,
))
)
)
if __name__ == '__main__':
_main()
if __name__ == "__main__":
try:
_main()
except Exception as e:
print(f"publish_api_server.py failed: {e}")
exit(1)
+157
View File
@@ -0,0 +1,157 @@
import argparse
import os
import subprocess
from pathlib import Path
from typing import Optional
from build_spark import SparkPaths
from core.iter_utils import at_bounded
def _cmake_verbosity(level):
# Skip 'WARNING' and 'NOTICE' levels, since the default CMake verbosity,
# 'STATUS', doesn't produce that much output. This also better normalizes
# numeric representation of verbosity w/ dotnet, which only has 5 levels.
names = [
"ERROR",
# 'WARNING',
# 'NOTICE',
"STATUS",
"VERBOSE",
"DEBUG",
"TRACE",
]
return at_bounded(names, level - 1)
def _cmake_preset(configuration: str):
return (
"sparkd-debug-clang"
if configuration.lower() == "debug"
else "sparkd-release-gcc"
)
class PublishArgs:
def __init__(
self,
configuration: str = "release",
cmake_jobs: Optional[int] = None,
clean: bool = False,
verbosity: int = 2,
dry_run: bool = False,
output: Optional[str] = None,
):
self.configuration = configuration
self.cmake_jobs = cmake_jobs
self.clean = clean
self.verbosity = verbosity
self.dry_run = dry_run
# Leave out WARNING and NOTICE to normalize cmake log level and dotnet verbosity
# (so both can be expressed as an int 1-5)
self.cmake_log_level = at_bounded(
[
"ERROR",
# 'WARNING',
# 'NOTICE',
"STATUS",
"VERBOSE",
"DEBUG",
"TRACE",
],
verbosity,
)
self.output = output
def _get_arg_parser():
parser = argparse.ArgumentParser(
description="Publish a self-contined spark client application"
)
parser.add_argument(
"--configuration",
metavar="debug|release",
help="The build configuration",
)
parser.add_argument(
"--cmake-jobs",
type=int,
help="The number of cmake parallel jobs",
)
parser.add_argument(
"--clean",
action="store_true",
help="Perform a clean build (remove artifacts from previous build)",
)
parser.add_argument(
"-o", "--output", type=str, help="The output directory for published artifacts."
)
parser.add_argument(
"--verbosity",
metavar="{0-5}",
choices=range(1, 6),
type=int,
help="The output verbosity level",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Preview the build commands without running them",
)
defaults = PublishArgs()
parser.set_defaults(
configuration=defaults.configuration,
clean=defaults.clean,
output=defaults.output,
verbosity=defaults.verbosity,
dry_run=defaults.dry_run,
)
return parser
def publish(args):
def run(cmd):
print(cmd)
if args.dry_run:
return
subprocess.run(cmd, shell=True).check_returncode()
cmake_preset = _cmake_preset(args.configuration)
cmake_jobs = args.cmake_jobs
cmake_verbosity = _cmake_verbosity(args.verbosity)
os.chdir(SparkPaths.root())
if args.clean:
run(f"cmake --build build/{cmake_preset} --target clean")
run(f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity}")
cmake_build_cmd = f"cmake --build build/{cmake_preset} --target install"
if cmake_jobs:
cmake_build_cmd += f" -j {cmake_jobs}"
run(cmake_build_cmd)
if args.output:
os.rename(Path("install").joinpath(cmake_preset), args.output)
def _main():
parser = _get_arg_parser()
args = parser.parse_args()
publish(
PublishArgs(
configuration=args.configuration,
cmake_jobs=args.cmake_jobs,
clean=args.clean,
verbosity=args.verbosity,
dry_run=args.dry_run,
output=args.output,
)
)
if __name__ == "__main__":
try:
_main()
except Exception as e:
print(f"publish_sparkd.py failed: {e}")
exit(1)