build: Add scripts to publish data stream apps

This commit is contained in:
John Colvin committed 2024-09-18 14:02:35 +00:00
1 parent b9ef3ddf91
commit 09a41775bb
7 files changed
+260 -6

No files matched your search

+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
tools_path=$(dirname $(readlink -f $0))
eval "$tools_path/ensure_venv.sh" 1> /dev/stdout 2> /dev/stderr
eval "$tools_path/venv_exec.sh python $tools_path/py/build_data_stream_web_app.py $@" 1> /dev/stdout 2> /dev/stderr
+13
View File
@@ -0,0 +1,13 @@
@echo off
@REM Build script designed for our jenkins build job(s). It assumes the py
@REM venv already exists.
setlocal
set tools_dir=%~dp0
set venv_dir=%~dp0..\.venv\
call "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\Tools\VsDevCmd.bat"
call "%venv_dir%Scripts\activate.bat"
call python "%tools_dir%py\publish_data_stream.py" %*
EXIT /B %ERRORLEVEL%
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
tools_path=$(dirname $(readlink -f $0))
if [ "$VIRTUAL_ENV" == "" ]; then
venv_activate="$(readlink -f "$tools_path/../.venv/bin/activate")"
echo "Activating Python virtual environment: $venv_activate"
source "$venv_activate"
else
echo "Python virtual environment already active: $VIRTUAL_ENV"
fi
python "$tools_path/py/publish_data_stream.py" $@
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
tools_path=$(dirname $(readlink -f $0))
eval "$tools_path/ensure_venv.sh" 1> /dev/stdout 2> /dev/stderr
eval "$tools_path/venv_exec.sh python $tools_path/py/publish_data_stream.py $@" 1> /dev/stdout 2> /dev/stderr
+1 -1
View File
@@ -2,4 +2,4 @@
tools_path=$(dirname $(readlink -f $0))
eval "$tools_path/ensure_venv.sh" 1> /dev/stdout 2> /dev/stderr
eval "$tools_path/venv_exec.sh python $tools_path/py/build_data_stream_web_app.py $@" 1> /dev/stdout 2> /dev/stderr
eval "$tools_path/venv_exec.sh python $tools_path/py/publish_data_stream_web_app.py $@" 1> /dev/stdout 2> /dev/stderr
+11 -5
View File
@@ -24,11 +24,17 @@ def _workspace_root():
def _default_build():
if sys.platform == 'win32':
return _workspace_root().joinpath('build')
elif sys.platform == 'linux':
return _workspace_root().joinpath('build', 'gcc', 'release')
raise ValueError(f"Unknown platform '{sys.platform}'")
return _workspace_root().joinpath('build')
def _dotnet_runtime():
match sys.platform:
case 'win32':
return 'win-x64'
case 'linux':
return 'linux-x64'
case _:
raise ValueError(f'Unsupported platform {sys.platform}')
class SparkPaths:
+214
View File
@@ -0,0 +1,214 @@
import argparse
import os
import subprocess
import sys
from typing import Optional
from build_spark import SparkPaths
from core.iter_utils import at_bounded
def _dotnet_verbosity(level):
names = ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic']
return at_bounded(names, level)
def _cmake_verbosity(level):
# Skip 'WARNING' and 'NOTICE' levels, since the default CMake verbosity,
# 'STATUS', doesn't produce that much output. This also better normalizes
# numeric representation of verbosity w/ dotnet, which only has 5 levels.
names = [
'ERROR',
# 'WARNING',
# 'NOTICE',
'STATUS',
'VERBOSE',
'DEBUG',
'TRACE',
]
return at_bounded(names, level)
def _cmake_preset():
match sys.platform:
case 'win32':
return 'data-stream-suite-windows'
case 'linux':
return 'data-stream-suite-linux'
case _:
raise ValueError(f'Unsupported platform {sys.platform}')
def _cmake_build_preset(configuration):
match sys.platform:
case 'win32':
return f'data-stream-suite-windows-{configuration}'
case 'linux':
return f'data-stream-suite-linux-{configuration}'
case _:
raise ValueError(f'Unsupported platform {sys.platform}')
def _dotnet_runtime():
match sys.platform:
case 'win32':
return 'win-x64'
case 'linux':
return 'linux-x64'
case _:
raise ValueError(f'Unsupported platform {sys.platform}')
class PublishArgs:
def __init__(
self,
configuration: str = 'release',
clean: bool = False,
verbosity: int = 1,
dry_run: bool = False,
output: Optional[str] = None,
framework='net7.0',
runtime=_dotnet_runtime(),
):
self.configuration = configuration
self.clean = clean
self.verbosity = verbosity
self.dry_run = dry_run
self.dotnet_verbosity = at_bounded(
['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'],
verbosity)
# Leave out WARNING and NOTICE to normalize cmake log level and dotnet verbosity
# (so both can be expressed as an int 1-5)
self.cmake_log_level = at_bounded(
[
'ERROR',
# 'WARNING',
# 'NOTICE',
'STATUS',
'VERBOSE',
'DEBUG',
'TRACE',
],
verbosity)
self.output = output
self.framework = framework
self.runtime = runtime
def _get_arg_parser():
parser = argparse.ArgumentParser(
description='Publish a self-contined spark client application')
parser.add_argument(
'--configuration',
metavar='debug|release',
help='The build configuration',
)
parser.add_argument(
'--clean',
action='store_true',
help='Perform a clean build (remove artifacts from previous build)',
)
parser.add_argument(
'-o',
'--output',
type=str,
help='The output directory for published artifacts.')
parser.add_argument(
'--verbosity',
metavar='{0-5}',
choices=range(0, 6),
type=int,
help='The output verbosity level',
)
parser.add_argument(
'--dry-run',
action='store_true',
help='Preview the build commands without running them',
)
defaults = PublishArgs()
parser.set_defaults(
configuration=defaults.configuration,
clean=defaults.clean,
output=defaults.output,
verbosity=defaults.verbosity,
dry_run=defaults.dry_run,
)
return parser
def publish(args: PublishArgs):
def run(cmd):
print(cmd)
if args.dry_run:
return
subprocess.run(cmd, shell=True).check_returncode()
def dotnet_cmd(project, action='build'):
csproj = SparkPaths.root().joinpath(
'src',
'DataStreamSuite',
project,
f'{project}.csproj',
)
return ' '.join([
f'dotnet {action} {csproj}',
f'--configuration {args.configuration.title()}',
f'--verbosity {_dotnet_verbosity(args.verbosity)}',
])
os.chdir(SparkPaths.root())
publish_dir = SparkPaths.default_build().joinpath(
args.configuration.title(),
'bin',
args.framework,
args.runtime,
'publish',
)
cmake_configure_cmd = ' '.join([
f'cmake',
f'--preset',
_cmake_preset(),
f'-DCMAKE_MESSAGE_LOG_LEVEL={_cmake_verbosity(args.verbosity)}',
f'-DDOTNET_BINARY_DIR={publish_dir}',
])
cmake_build_cmd = ' '.join([
f'cmake',
f'--build',
f'--preset {_cmake_build_preset(args.configuration.lower())}',
])
dotnet_projects = ["PubSubServer", "DataStreamPublisher", "SharedStreams"]
if args.clean:
run(f'{cmake_build_cmd} --target clean')
for p in dotnet_projects:
run(dotnet_cmd(p, 'clean'))
run(cmake_configure_cmd)
run(cmake_build_cmd)
for p in dotnet_projects:
publish_cmd = f'{dotnet_cmd(p, "publish")} -f {args.framework} -r {args.runtime} --sc'
run(publish_cmd)
if args.output:
run(f'mv {publish_dir} {args.output}')
def _main():
parser = _get_arg_parser()
args = parser.parse_args()
publish(
PublishArgs(
configuration=args.configuration,
clean=args.clean,
verbosity=args.verbosity,
dry_run=args.dry_run,
output=args.output,
))
if __name__ == '__main__':
try:
_main()
except Exception as e:
print(f"publish_data_stream.py failed: {e}")
exit(1)