Files
spark/tools/py/publish_data_stream.py

200 lines
5.6 KiB
Python

import argparse
import os
import subprocess
import sys
from pathlib import Path
from typing import Optional
from build_spark import SparkPaths
from core.iter_utils import at_bounded
def _dotnet_verbosity(level):
names = ["quiet", "minimal", "normal", "detailed", "diagnostic"]
return at_bounded(names, level)
def _cmake_verbosity(level):
# Skip 'WARNING' and 'NOTICE' levels, since the default CMake verbosity,
# 'STATUS', doesn't produce that much output. This also better normalizes
# numeric representation of verbosity w/ dotnet, which only has 5 levels.
names = [
"ERROR",
# 'WARNING',
# 'NOTICE',
"STATUS",
"VERBOSE",
"DEBUG",
"TRACE",
]
return at_bounded(names, level)
def _cmake_preset():
match sys.platform:
case "win32":
return "data-stream-suite-windows"
case "linux":
return "data-stream-suite-linux"
case _:
raise ValueError(f"Unsupported platform {sys.platform}")
def _dotnet_runtime():
match sys.platform:
case "win32":
return "win-x64"
case "linux":
return "linux-x64"
case _:
raise ValueError(f"Unsupported platform {sys.platform}")
class PublishArgs:
def __init__(
self,
configuration: str = "release",
cmake_jobs: Optional[int] = None,
clean: bool = False,
verbosity: int = 1,
dry_run: bool = False,
output: Optional[str] = None,
framework="net8.0",
runtime=_dotnet_runtime(),
):
self.configuration = configuration
self.cmake_jobs = cmake_jobs
self.clean = clean
self.verbosity = verbosity
self.dry_run = dry_run
self.dotnet_verbosity = at_bounded(
["quiet", "minimal", "normal", "detailed", "diagnostic"], verbosity
)
# Leave out WARNING and NOTICE to normalize cmake log level and dotnet verbosity
# (so both can be expressed as an int 1-5)
self.cmake_log_level = at_bounded(
[
"ERROR",
# 'WARNING',
# 'NOTICE',
"STATUS",
"VERBOSE",
"DEBUG",
"TRACE",
],
verbosity,
)
self.output = output
self.framework = framework
self.runtime = runtime
def _get_arg_parser():
parser = argparse.ArgumentParser(
description="Publish a self-contined spark client application"
)
parser.add_argument(
"--configuration",
metavar="debug|release",
help="The build configuration",
)
parser.add_argument(
"--clean",
action="store_true",
help="Perform a clean build (remove artifacts from previous build)",
)
parser.add_argument(
"-o", "--output", type=str, help="The output directory for published artifacts."
)
parser.add_argument(
"--verbosity",
metavar="{0-5}",
choices=range(0, 6),
type=int,
help="The output verbosity level",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Preview the build commands without running them",
)
defaults = PublishArgs()
parser.set_defaults(
configuration=defaults.configuration,
clean=defaults.clean,
output=defaults.output,
verbosity=defaults.verbosity,
dry_run=defaults.dry_run,
)
return parser
def publish(args: PublishArgs):
def run(cmd):
print(cmd)
if args.dry_run:
return
subprocess.run(cmd, shell=True).check_returncode()
def dotnet_cmd(action="build"):
sln = SparkPaths.root().joinpath(
"src",
"DataStreamSuite",
"DataStreamSuite.sln",
)
return f"dotnet {action} {sln} --configuration {args.configuration.title()} --verbosity {_dotnet_verbosity(args.verbosity)}"
os.chdir(SparkPaths.root())
cmake_preset = _cmake_preset()
cmake_jobs = args.cmake_jobs
cmake_verbosity = _cmake_verbosity(args.verbosity)
publish_dir = SparkPaths.default_build().joinpath(
args.configuration.title(),
"bin",
args.framework,
args.runtime,
"publish",
)
cmake_configure_cmd = f"cmake --preset {cmake_preset} -DCMAKE_MESSAGE_LOG_LEVEL={cmake_verbosity} -DDOTNET_BINARY_DIR={publish_dir}"
cmake_build_dir = Path(f"build/{cmake_preset}/")
if args.clean:
if not cmake_build_dir.exists():
print("Skipping clean, build dir does not exist")
else:
run(f"cmake --build {cmake_build_dir} --config {args.configuration.title()} --target clean")
run(dotnet_cmd("clean"))
cmake_build_cmd = f"cmake --build {cmake_build_dir} --config {args.configuration.title()} --target install"
if cmake_jobs:
cmake_build_cmd += f" -j {cmake_jobs}"
run(cmake_configure_cmd)
run(cmake_build_cmd)
publish_cmd = f"{dotnet_cmd('publish')} -f {args.framework} -r {args.runtime} --sc"
run(publish_cmd)
if args.output:
os.rename(publish_dir, args.output)
def _main():
parser = _get_arg_parser()
args = parser.parse_args()
publish(
PublishArgs(
configuration=args.configuration,
clean=args.clean,
verbosity=args.verbosity,
dry_run=args.dry_run,
output=args.output,
)
)
if __name__ == "__main__":
try:
_main()
except Exception as e:
print(f"publish_data_stream.py failed: {e}")
exit(1)