Files
spark/tools/py/build_spark.py
T
John Colvin 365394f28b build: Fix build_spark.py (broken since CMakePresets introduced)
Pass additional cmake opts through cli. Specifically, those that were
moved from CMakeLists.txt to CMakePrests.json
2023-01-02 18:42:39 -05:00

425 lines
14 KiB
Python

import argparse
import os
from pathlib import Path
import subprocess
import shutil
from os import PathLike
from core.argparse_utils import add_toggle_opt, get_arg
from core.cmake_utils import CMakeConfigureCommand
from core.file_utils import CWD, SparkPaths, create_dir, link_dir, unlink
from core.logging import exit_msg
from core.str_utils import wrap_dquotes
from typing import Optional, Tuple
def _get_cmake_opts():
cxx_flags = [
'/MP',
'/WX',
'/wd4250',
'/wd4251',
'/wd4244',
'/wd4267',
'/wd4355',
'/wd4275',
'/wd4345',
'/bigobj',
'/EHsc',
'/DCURL_STATICLIB',
'/D_WINSOCK_DEPRECATED_NO_WARNINGS',
'/D__TBB_NO_IMPLICIT_LINKAGE',
'/DBOOST_ALL_NO_LIB',
'/std:c++17',
'/D_SCL_SECURE_NO_WARNINGS',
'/D_CRT_SECURE_NO_WARNINGS',
'/D_WIN32_WINNT=0x0600',
]
linker_flags = [
'/DEBUG',
'/OPT:REF',
'/OPT:ICF',
'/LTCG',
]
cxx_flags_debug = cxx_flags + [
'/MTd',
]
cxx_flags_release = cxx_flags + [
'/MT',
]
opts = {
'CMAKE_CXX_FLAGS': wrap_dquotes(' '.join(cxx_flags)),
'CMAKE_CXX_FLAGS_DEBUG': wrap_dquotes(' '.join(cxx_flags_debug)),
'CMAKE_CXX_FLAGS_RELEASE': wrap_dquotes(' '.join(cxx_flags_release)),
'CMAKE_SHARED_LINKER_FLAGS_RELEASE': wrap_dquotes(' '.join(linker_flags))
}
return opts
class VisualStudioContext:
def __init__(self, version, year, prompt_path, msbuild_path, msbuild_arg_prefix):
self.version = version
self.year = year
self.prompt_path = prompt_path
self.msbuild_path = msbuild_path
self._msbuild_arg_prefix = msbuild_arg_prefix
def prompt_cmd(self):
return f'{self.prompt_path} -arch=x64 -host_arch=x64'
def configure_libspark(
self,
build_dir: PathLike,
source_dir: Optional[PathLike] = None,
) -> subprocess.CompletedProcess:
source_dir = source_dir or SparkPaths.root()
cmd = CMakeConfigureCommand(
source_dir=source_dir,
build_dir=build_dir,
generator=f'"Visual Studio {self.version} {self.year}"')
opts = _get_cmake_opts()
cmd.add_opts(opts)
print(f'configure_cmd: {cmd}')
with CWD(build_dir):
res = subprocess.run(str(cmd))
shutil.copy(
SparkPaths.tools().joinpath('cpp.build.props'),
'Directory.Build.Props')
return res
def msbuild_flag(self, flag):
return f'{self._msbuild_arg_prefix}{flag}'
def msbuild_clp(self, key, value=None):
rv = f'{self._msbuild_arg_prefix}clp:{key}'
if value is not None:
rv = f'{rv}={value}'
return rv
def msbuild_target(self, target):
return f'{self._msbuild_arg_prefix}t:{target}'
def msbuild_prop(self, key, value):
return f'{self._msbuild_arg_prefix}p:{key}={value}'
def _deduce_visual_studio_context():
# Wrap paths in quotes so spaces don't need to be escaped
# promp_path values are tuples with fields 'version, year, prompt_path', respectively
vs_contexts = [
VisualStudioContext(
17, 2022,
r'C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\Tools\VsDevCmd.bat',
r'"C:\Program Files\Microsoft Visual Studio\2022\Community\Msbuild\Current\Bin\amd64\MSBuild.exe"',
'-'),
VisualStudioContext(
16, 2019,
r'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\Common7\Tools\VsDevCmd.bat',
r'"C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe"',
'/')
]
for vs_context in vs_contexts:
if os.path.exists(f'{vs_context.prompt_path}'):
return vs_context
raise ValueError('Failed to deduce visual studio context')
class BuildTarget:
def __init__(self, name, path, cmd=None):
self.name = name
self.path = path
self.cmd = cmd
class BuildContext:
def __init__(self, workspace_root=None, build_dir=None):
workspace_root = workspace_root or SparkPaths.root()
build_dir = build_dir or SparkPaths.default_build()
self.workspace_root = workspace_root
self.build_dir = build_dir
def get_csharp_targets() -> dict:
targets = {}
csproj_root = workspace_root.joinpath('src', 'client')
def add_target(target: BuildTarget):
targets[target.name] = target
sln_tgt = BuildTarget(
'spark_client', workspace_root.joinpath('spark_client.sln'))
add_target(sln_tgt)
def add_csproj_target(name: str):
add_target(
BuildTarget(
name, csproj_root.joinpath(name, f'{name}.csproj')))
# TODO: rename 'Spark.csproj' -> 'spark.csproj' to be consistent
# with all other csproj filenames, which are snake_case
add_target(
BuildTarget(
'spark', csproj_root.joinpath('spark', 'Spark.csproj')))
add_csproj_target('spark_api')
add_csproj_target('spark_core')
add_csproj_target('spark_sdk')
add_csproj_target('spark_test')
add_csproj_target('sparkcmd')
return targets
self.csharp_targets = get_csharp_targets()
self.csharp_aliases = {
'all': ['spark_client'],
'api': ['spark_api'],
'core': ['spark_core'],
'cmd': ['sparkcmd'],
'test': ['test'],
'sdk': ['spark_sdk']
}
def get_cpp_targets() -> dict:
targets = {}
def add_target(target: BuildTarget):
targets[target.name] = target
add_target(
BuildTarget(
'libspark', build_dir.joinpath('ALL_BUILD.vcxproj')))
return targets
self.cpp_targets = get_cpp_targets()
self.cpp_aliases = {}
publish_spark = BuildTarget(
'publish',
Path(workspace_root, 'src', 'client', 'spark', 'spark.csproj'))
self.publish_targets = {publish_spark.name: publish_spark}
self.publish_aliases = {}
def parse_targets(self, targets) -> Tuple[list, list, list]:
'''Parses target input (from cli) and returns a tuple whose items are a
list of csharp targets, and cpp targets, respectively'''
csharp_ts = []
cpp_ts = []
publish_ts = []
for t in targets:
# Always try to parse `t` as both cpp and csharp targets, since it
# could be an alias for a combination of csharp and cpp targets.
csharp_ts.extend(
self._parse_target(
t, self.csharp_targets, self.csharp_aliases))
cpp_ts.extend(
self._parse_target(t, self.cpp_targets, self.cpp_aliases))
publish_ts.extend(
self._parse_target(
t, self.publish_targets, self.publish_aliases))
return (csharp_ts, cpp_ts, publish_ts)
def _parse_target(self, t, targets, aliases) -> list[BuildTarget]:
if t in targets:
return [targets[t]]
elif t in aliases:
alias_ts = aliases[t]
for a in alias_ts:
if a not in targets:
msg = f"Failed to find target for '{t}' (alias '{a}'). known targets: {targets}"
exit_msg(1, msg)
return [targets[alias_t] for alias_t in aliases[t]]
return []
def _get_arg_parser():
parser = argparse.ArgumentParser(description='Build spark_client.sln')
parser.add_argument(
'--configuration',
metavar='debug|release',
type=str,
nargs='?',
help='The build configuration',
default='Debug')
parser.add_argument(
'-o',
'--output-dir',
metavar='path',
type=str,
nargs='?',
help='The directory in which to build')
parser.add_argument(
'-l',
'--link-output-dir',
metavar='link_path',
type=str,
nargs='?',
help='The symbolink link to --output-dir')
parser.add_argument(
'-t',
'--target',
metavar='targets',
type=str,
action='append',
nargs='+',
help='The target(s) to build')
add_toggle_opt(parser, 'cpp', help='Whether or not to build cpp')
add_toggle_opt(parser, 'csharp', help='Whether or not to build csharp')
add_toggle_opt(
parser, 'restore', help='Whether or not to run msbuild /t:restore')
add_toggle_opt(
parser,
'clean',
help=
'Whether or not to delete and overwrite an existing build directory')
parser.add_argument(
'--verbosity',
metavar='[q(uiet)|m(inimal)|n(ormal)|d(etailed)|diag(nostic)]',
choices=[
'q', 'quiet', 'm', 'minimal', 'n', 'normal', 'd', 'detailed',
'diag', 'diagnostic'
],
type=str,
help='The output verbosity level')
parser.set_defaults(
restore=True, cpp=True, csharp=True, verbosity='minimal')
return parser
def build(
configuration: str | None = None,
restore: bool = True,
clean: bool = False,
build_dir: Path | None = None,
link_build_dir: Path | None = None,
targets: list[str] | None = None,
verbosity: str | None = None) -> list[subprocess.CompletedProcess]:
'''Builds the spark client'''
# check if explicit no-op
if targets is not None and len(targets) == 0:
return []
proc_results = []
# init unspecified args w/ defaults
if targets is None:
targets = ['all']
if configuration is None:
configuration = 'release' if 'publish' in targets else 'debug'
if build_dir is None:
build_dir = SparkPaths.default_build()
if verbosity is None:
verbosity = 'minimal'
vs_context = _deduce_visual_studio_context()
try:
build_dir = build_dir.absolute()
if link_build_dir is not None:
unlink(link_build_dir)
create_dir(build_dir, clean)
cfg_res = vs_context.configure_libspark(
build_dir=build_dir, source_dir=SparkPaths.root())
proc_results.append(cfg_res)
except Exception as e:
exit_msg(1, e)
build_ctx = BuildContext(SparkPaths.root(), build_dir)
csharp_tgts, cpp_tgts, publish_tgts = build_ctx.parse_targets(targets)
build_cmd_steps = [vs_context.prompt_cmd()]
base_output_path_arg = vs_context.msbuild_prop(
'BaseOutputPath', f'{build_dir}\\{configuration}')
# restore build (downloads nuget package deps)
if restore or clean:
restore_cmd_prefix = ' '.join(
[
f"{vs_context.msbuild_path}",
f"{vs_context.msbuild_flag('m')}",
f"{vs_context.msbuild_prop('platform', 'x64')}",
f"{vs_context.msbuild_prop('configuration', configuration)}",
f"{vs_context.msbuild_target('restore')}",
f"{vs_context.msbuild_clp('verbosity', verbosity)}",
f"{vs_context.msbuild_clp('summary')}"
])
for t in cpp_tgts:
build_cmd_steps.append(f'{restore_cmd_prefix} {t.path}')
for t in csharp_tgts:
build_cmd_steps.append(
f'{restore_cmd_prefix} {base_output_path_arg} {t.path}')
# main build
debugtype = 'portable' if configuration.lower() == 'debug' else 'none'
build_cmd_prefix = ' '.join(
[
f"{vs_context.msbuild_path}", f"{vs_context.msbuild_flag('m')}",
f"{vs_context.msbuild_prop('platform', 'x64')}",
f"{vs_context.msbuild_prop('GenerateFullPaths', 'true')}",
f"{vs_context.msbuild_prop('configuration', configuration)}",
f"{vs_context.msbuild_prop('debugtype', debugtype)}",
f"{vs_context.msbuild_clp('verbosity', verbosity)}",
f"{vs_context.msbuild_clp('summary')}"
])
for t in cpp_tgts:
build_cmd_steps.append(f'{build_cmd_prefix} {t.path}')
for t in csharp_tgts:
build_cmd_steps.append(
f'{build_cmd_prefix} {base_output_path_arg} {t.path}')
bins = SparkPaths.client_bins(build_dir, configuration=configuration)
for t in publish_tgts:
build_cmd_steps.append(f'cd {Path(t.path).parent}')
build_cmd_steps.append('dotnet publish -c Release -r win-x64 --sc')
build_cmd_steps.append(
f'copy {bins.joinpath("libspark.dll")} {bins.joinpath("publish")}')
build_cmd = ' && '.join(build_cmd_steps)
proc_results.append(subprocess.run(build_cmd))
if link_build_dir is not None:
link_dir(build_dir, link_build_dir)
return proc_results
def _main():
parser = _get_arg_parser()
args = parser.parse_args()
def arg_to_path(arg):
return None if arg is None else Path(arg)
def flatten_arglist(arglist):
return None if arglist is None else [a for grp in arglist for a in grp]
args_dict = vars(args)
build_dir = arg_to_path(get_arg(args_dict, 'output_dir'))
link_build_dir = arg_to_path(get_arg(args_dict, 'link_output_dir'))
targets = flatten_arglist(get_arg(args_dict, 'target'))
verbosity = get_arg(args_dict, 'verbosity') or None
build(
configuration=args.configuration,
clean=args.clean,
restore=args.restore,
build_dir=build_dir,
link_build_dir=link_build_dir,
targets=targets,
verbosity=verbosity)
if __name__ == "__main__":
_main()