From 5180d13fb02bc11446d5dfbafbf61760ee445327 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 16 Aug 2023 11:32:47 -0400 Subject: [PATCH] build (refactor): Organize py classes a bit --- tools/py/build_spark.py | 243 +-------------------------- tools/py/build_sparkd.py | 3 +- tools/py/cmake/cmake_utils.py | 57 ------- tools/py/core/build_target.py | 35 ++++ tools/py/core/cmake_utils.py | 85 ++++++++++ tools/py/core/dotnet_utils.py | 78 +++++++++ tools/py/core/visual_studio_utils.py | 72 ++++++++ 7 files changed, 278 insertions(+), 295 deletions(-) delete mode 100644 tools/py/cmake/cmake_utils.py create mode 100644 tools/py/core/build_target.py create mode 100644 tools/py/core/dotnet_utils.py create mode 100644 tools/py/core/visual_studio_utils.py diff --git a/tools/py/build_spark.py b/tools/py/build_spark.py index 7d79976aa..4bc8c6ef6 100644 --- a/tools/py/build_spark.py +++ b/tools/py/build_spark.py @@ -1,247 +1,18 @@ -from abc import ABC, abstractmethod import argparse from pathlib import Path import subprocess from core.argparse_utils import add_toggle_opt, get_arg -from core.file_utils import SparkPaths, copy, create_dir -from core.table_formatter import fmt_table +from core.build_target import BuildTarget +from core.cmake_utils import CMakeTarget +from core.dotnet_utils import CsprojTarget +from core.file_utils import SparkPaths, create_dir from core.logging import exit_msg -from core.iter_utils import at_bounded, join_t +from core.iter_utils import at_bounded +from core.visual_studio_utils import get_preferred_vs_context, VisualStudioContext from typing import Optional -class VisualStudioContext: - def __init__( - self, - version, - year, - prompt_path: Path, - msbuild_path: Path, - msbuild_arg_prefix, - ): - self.version = version - self.year = year - self.prompt_path = prompt_path - self.msbuild_path = msbuild_path - self._msbuild_arg_prefix = msbuild_arg_prefix - - def prompt_cmd(self): - return f'{self.prompt_path} -arch=x64 -host_arch=x64' - - def msbuild_flag(self, flag): - return f'{self._msbuild_arg_prefix}{flag}' - - def msbuild_clp(self, key, value=None): - rv = f'{self._msbuild_arg_prefix}clp:{key}' - if value is not None: - rv = f'{rv}={value}' - return rv - - def msbuild_target(self, target): - return f'{self._msbuild_arg_prefix}t:{target}' - - def msbuild_prop(self, key, value): - return f'{self._msbuild_arg_prefix}p:{key}={value}' - - -def _get_supported_vs_contexts(): - '''Gets a list of supported Visual Studio contexts (versions)''' - # vs 17 - vs17_path = Path('C:/Program Files/Microsoft Visual Studio/2022/Community') - vs17 = VisualStudioContext( - 17, - 2022, - vs17_path.joinpath('Common7/Tools/VsDevCmd.bat'), - vs17_path.joinpath('Msbuild/Current/Bin/amd64/MSBuild.exe'), - '-', - ) - - # vs 16 - vs16_path = Path( - 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Community') - vs16 = VisualStudioContext( - 16, - 2019, - vs16_path.joinpath('Common7/Tools/VsDevCmd.bat'), - vs16_path.joinpath('MSBuild/Current/Bin/MSBuild.exe'), - '/', - ) - return [vs17, vs16] - - -def _get_preferred_vs_context(): - '''Gets the most preferred Visual Studio context (version) on this computer''' - vs_ctxs = _get_supported_vs_contexts() - for vs_context in vs_ctxs: - if vs_context.prompt_path.exists(): - return vs_context - supported_versions = ', '.join([str(v.version) for v in vs_ctxs]) - raise ValueError( - f'Failed to find supported Visual Studio version (works with {supported_versions})' - ) - - -class BuildTarget(ABC): - def __init__( - self, - name, - source_path: Optional[Path], - out_path: Optional[Path], - aliases=[]): - self.name = name - self.source_path = source_path - self.out_path = out_path - self.aliases = aliases - - @abstractmethod - def configure(self): - pass - - @abstractmethod - def build(self): - pass - - def is_match(self, targets): - return any( - t.lower() == self.name.lower() or t in self.aliases - for t in targets) - - -class CMakeTarget(BuildTarget): - def __init__( - self, - name, - preset='spark', - build_preset='spark-debug', - source_path=SparkPaths.root(), - out_path=SparkPaths.default_build(), - aliases=[], - verbosity: Optional[str] = None, - ): - if out_path is None: - raise ValueError('out_path is None') - if source_path is None: - raise ValueError('source_path is None') - BuildTarget.__init__(self, name, source_path, out_path, aliases) - self.preset = preset - self.build_preset = build_preset - self.verbosity = verbosity - - def _get_configure_cmd(self): - cmd_parts = ['cmake'] - if self.preset is not None: - cmd_parts.append(f'--preset {self.preset}') - if self.verbosity is not None: - cmd_parts.append(f'-DCMAKE_MESSAGE_LOG_LEVEL={self.verbosity}') - if self.source_path is not None: - cmd_parts.append(f'-S {self.source_path}') - if self.out_path is not None: - cmd_parts.append(f'-B {self.out_path}') - return ' '.join(cmd_parts) - - def _get_build_cmd(self): - cmd_parts = ['cmake --build'] - # In CMake, the build dir arg (-B) can't be combined w/ --preset - if self.build_preset is not None: - cmd_parts.append(f'--preset {self.build_preset}') - elif self.out_path is not None: - cmd_parts.append(f'{self.out_path}') - return ' '.join(cmd_parts) - - def configure(self): - cmd = self._get_configure_cmd() - print(f'configure_cmd: {cmd}') - subprocess.run(cmd).check_returncode() - if self.out_path is not None: - copy( - SparkPaths.tools().joinpath('cpp.build.props'), - self.out_path.joinpath('Directory.Build.Props')) - - def build(self): - cmd = self._get_build_cmd() - print(f'build_cmd: {cmd}') - subprocess.run(cmd).check_returncode() - - def __str__(self): - return fmt_table( - { - 'name': self.name, - 'configure_cmd': self._get_configure_cmd(), - 'build_cmd': self._get_build_cmd(), - }) - - -class CsprojTarget(BuildTarget): - def __init__( - self, - name: str, - source_path: Path, - out_path: Path, - aliases=[], - configuration: str = 'debug', - verbosity: str = 'minimal', - vs_context: Optional[VisualStudioContext] = None, - restore=True, - ): - BuildTarget.__init__(self, name, source_path, out_path, aliases) - self.configuration = configuration - self.verbosity = verbosity - self.restore = restore - self.vs_context = vs_context or _get_preferred_vs_context() - - def _get_build_cmd(self): - cfg = self.configuration - build_cmd_steps = [self.vs_context.prompt_cmd()] - base_output_path_arg = self.vs_context.msbuild_prop( - 'BaseOutputPath', self.out_path.joinpath(cfg.title())) - if self.restore: - # Restore build (download nugets) - restore_cmd_args = [ - f'"{self.vs_context.msbuild_path}"', - f"{self.vs_context.msbuild_flag('m')}", - f"{self.vs_context.msbuild_prop('platform', 'x64')}", - f"{self.vs_context.msbuild_prop('configuration', cfg.title())}", - f"{self.vs_context.msbuild_target('restore')}", - f"{self.vs_context.msbuild_clp('verbosity', self.verbosity)}", - f"{self.vs_context.msbuild_clp('summary')}", - base_output_path_arg, - self.source_path, - ] - restore_cmd = join_t(' ', restore_cmd_args) - build_cmd_steps.append(restore_cmd) - - # Main build - dbg_type = 'portable' if cfg.lower() == 'debug' else 'none' - build_cmd_prefix = ' '.join( - [ - f'"{self.vs_context.msbuild_path}"', - f"{self.vs_context.msbuild_flag('m')}", - f"{self.vs_context.msbuild_prop('platform', 'x64')}", - f"{self.vs_context.msbuild_prop('GenerateFullPaths', 'true')}", - f"{self.vs_context.msbuild_prop('configuration', cfg.title())}", - f"{self.vs_context.msbuild_prop('debugtype', dbg_type)}", - f"{self.vs_context.msbuild_clp('verbosity', self.verbosity)}", - f"{self.vs_context.msbuild_clp('summary')}" - ]) - build_cmd_steps.append( - f'{build_cmd_prefix} {base_output_path_arg} {self.source_path}') - return ' && '.join(build_cmd_steps) - - def configure(self): - pass - - def build(self): - subprocess.run(self._get_build_cmd()).check_returncode() - - def __str__(self): - return fmt_table( - { - 'name': self.name, - 'build_cmd': self._get_build_cmd() - }) - - def _get_arg_parser(): parser = argparse.ArgumentParser(description='Build spark_client.sln') parser.add_argument( @@ -397,7 +168,7 @@ def _main(): if tgt_names is None or len(tgt_names) == 0: tgt_names = ['libspark', 'spark'] - vs_context = _get_preferred_vs_context() + vs_context = get_preferred_vs_context() tgts = _get_targets( build_type=args.configuration, build_dir=build_dir, diff --git a/tools/py/build_sparkd.py b/tools/py/build_sparkd.py index 71f1c9906..fcc0af2f0 100644 --- a/tools/py/build_sparkd.py +++ b/tools/py/build_sparkd.py @@ -2,9 +2,8 @@ import argparse from typing import cast import subprocess -from cmake.cmake_utils import CMakeCompilerPair, CMakeConfigureCommand -from core.argparse_utils import add_toggle_opt, get_arg from core.cmake_utils import CMakeCompilerPair, CMakeConfigureCommand +from core.argparse_utils import add_toggle_opt, get_arg from core.file_utils import CWD, SparkPaths, as_path, create_dir from core.iter_utils import distinct, flatten from core.logging import exit_msg diff --git a/tools/py/cmake/cmake_utils.py b/tools/py/cmake/cmake_utils.py deleted file mode 100644 index 266357467..000000000 --- a/tools/py/cmake/cmake_utils.py +++ /dev/null @@ -1,57 +0,0 @@ -from typing import Optional - - -class CMakeCompilerPair: - '''Represents a pair of c, cxx compilers''' - def __init__(self, name, c, cxx): - self.name = name - self.c = c - self.cxx = cxx - - def to_opts(self): - opts = {} - if self.c: - opts['C_COMPILER'] = self.c - if self.cxx: - opts['CXX_COMPILER'] = self.cxx - return opts - - def __str__(self): - return f'{self.name} ({self.c}, {self.cxx})' - - -class CMakeConfigureCommand: - def __init__( - self, - src_dir, - build_dir, - generator: Optional[str] = None, - build_type: Optional[str] = None, - compiler: Optional[CMakeCompilerPair] = None): - if src_dir is None: - raise ValueError('Source dir cannot be None') - self.cmd = 'cmake' - self.add_opt('-S', src_dir) - self.add_opt('-B', build_dir) - if generator is not None: - self.add_opt('-G', generator) - if build_type is not None: - self.add_opt('CMAKE_BUILD_TYPE', build_type) - if compiler is not None: - self.add_opt('CMAKE_C_COMPILER', compiler.c) - self.add_opt('CMAKE_CXX_COMPILER', compiler.cxx) - - def add_opt(self, name: str, value): - if name is None: - raise ValueError('opt name cannot be None') - if value is None: - return self - if not name.startswith('-'): - name = f'-D{name}' - self.cmd = f'{self.cmd} {name}={value}' - else: - self.cmd = f'{self.cmd} {name} {value}' - return self - - def __str__(self): - return self.cmd diff --git a/tools/py/core/build_target.py b/tools/py/core/build_target.py new file mode 100644 index 000000000..d8c2e5d88 --- /dev/null +++ b/tools/py/core/build_target.py @@ -0,0 +1,35 @@ +from abc import ABC, abstractmethod +from pathlib import Path +import subprocess + +from core.table_formatter import fmt_table +from core.iter_utils import join_t +from core.visual_studio_utils import get_preferred_vs_context, VisualStudioContext +from typing import Optional + + +class BuildTarget(ABC): + def __init__( + self, + name, + source_path: Optional[Path], + out_path: Optional[Path], + aliases=[]): + self.name = name + self.source_path = source_path + self.out_path = out_path + self.aliases = aliases + + @abstractmethod + def configure(self): + pass + + @abstractmethod + def build(self): + pass + + def is_match(self, targets): + return any( + t.lower() == self.name.lower() or t in self.aliases + for t in targets) + diff --git a/tools/py/core/cmake_utils.py b/tools/py/core/cmake_utils.py index 5cc35ffda..6755f5493 100644 --- a/tools/py/core/cmake_utils.py +++ b/tools/py/core/cmake_utils.py @@ -1,4 +1,9 @@ +import subprocess + +from build_target import BuildTarget from os import PathLike +from file_utils import copy, SparkPaths +from table_formatter import fmt_table from typing import Optional @@ -63,3 +68,83 @@ class CMakeConfigureCommand: def __str__(self): return self.cmd + + +class CMakeTarget(BuildTarget): + def __init__( + self, + name, + preset='spark', + build_preset='spark-debug', + source_path=SparkPaths.root(), + out_path=SparkPaths.default_build(), + aliases=[], + verbosity: Optional[str] = None, + ): + if out_path is None: + raise ValueError('out_path is None') + if source_path is None: + raise ValueError('source_path is None') + BuildTarget.__init__(self, name, source_path, out_path, aliases) + self.preset = preset + self.build_preset = build_preset + self.verbosity = verbosity + + def _get_configure_cmd(self): + cmd_parts = ['cmake'] + if self.preset is not None: + cmd_parts.append(f'--preset {self.preset}') + if self.verbosity is not None: + cmd_parts.append(f'-DCMAKE_MESSAGE_LOG_LEVEL={self.verbosity}') + if self.source_path is not None: + cmd_parts.append(f'-S {self.source_path}') + if self.out_path is not None: + cmd_parts.append(f'-B {self.out_path}') + return ' '.join(cmd_parts) + + def _get_build_cmd(self): + cmd_parts = ['cmake --build'] + # In CMake, the build dir arg (-B) can't be combined w/ --preset + if self.build_preset is not None: + cmd_parts.append(f'--preset {self.build_preset}') + elif self.out_path is not None: + cmd_parts.append(f'{self.out_path}') + return ' '.join(cmd_parts) + + def configure(self): + cmd = self._get_configure_cmd() + print(f'configure_cmd: {cmd}') + subprocess.run(cmd).check_returncode() + if self.out_path is not None: + copy( + SparkPaths.tools().joinpath('cpp.build.props'), + self.out_path.joinpath('Directory.Build.Props')) + + def build(self): + cmd = self._get_build_cmd() + print(f'build_cmd: {cmd}') + subprocess.run(cmd).check_returncode() + + def __str__(self): + return fmt_table( + { + 'name': self.name, + 'configure_cmd': self._get_configure_cmd(), + 'build_cmd': self._get_build_cmd(), + }) + + +def get_cmake_build_type(build_type: str): + ''' + Converts a case-insensitive build-type str (from cli) to a case-sensitive CMAKE_BUILD_TYPE var + ''' + + # The 'build_type' arg is case-sensitive because 'CMAKE_BUILD_TYPE' is substitued for a folder + # in the externals-libs path. In order to match a real path, it must be 'Debug' or 'Release'. + # Keep the orig_build_type around for potential use in err msg. + orig_build_type = build_type + build_type = build_type or 'debug' + build_type = build_type.title() + if build_type not in ['Debug', 'Release']: + raise ValueError(f"Unexpected build_type '{orig_build_type}'") + return build_type diff --git a/tools/py/core/dotnet_utils.py b/tools/py/core/dotnet_utils.py new file mode 100644 index 000000000..c718650c4 --- /dev/null +++ b/tools/py/core/dotnet_utils.py @@ -0,0 +1,78 @@ +from pathlib import Path +import subprocess + +from core.table_formatter import fmt_table +from core.build_target import BuildTarget +from core.iter_utils import join_t +from core.visual_studio_utils import get_preferred_vs_context, VisualStudioContext +from typing import Optional + + +class CsprojTarget(BuildTarget): + def __init__( + self, + name: str, + source_path: Path, + out_path: Path, + aliases=[], + configuration: str = 'debug', + verbosity: str = 'minimal', + vs_context: Optional[VisualStudioContext] = None, + restore=True, + ): + BuildTarget.__init__(self, name, source_path, out_path, aliases) + self.configuration = configuration + self.verbosity = verbosity + self.restore = restore + self.vs_context = vs_context or get_preferred_vs_context() + + def _get_build_cmd(self): + cfg = self.configuration + build_cmd_steps = [self.vs_context.prompt_cmd()] + base_output_path_arg = self.vs_context.msbuild_prop( + 'BaseOutputPath', self.out_path.joinpath(cfg.title())) + if self.restore: + # Restore build (download nugets) + restore_cmd_args = [ + f'"{self.vs_context.msbuild_path}"', + f"{self.vs_context.msbuild_flag('m')}", + f"{self.vs_context.msbuild_prop('platform', 'x64')}", + f"{self.vs_context.msbuild_prop('configuration', cfg.title())}", + f"{self.vs_context.msbuild_target('restore')}", + f"{self.vs_context.msbuild_clp('verbosity', self.verbosity)}", + f"{self.vs_context.msbuild_clp('summary')}", + base_output_path_arg, + self.source_path, + ] + restore_cmd = join_t(' ', restore_cmd_args) + build_cmd_steps.append(restore_cmd) + + # Main build + dbg_type = 'portable' if cfg.lower() == 'debug' else 'none' + build_cmd_prefix = ' '.join( + [ + f'"{self.vs_context.msbuild_path}"', + f"{self.vs_context.msbuild_flag('m')}", + f"{self.vs_context.msbuild_prop('platform', 'x64')}", + f"{self.vs_context.msbuild_prop('GenerateFullPaths', 'true')}", + f"{self.vs_context.msbuild_prop('configuration', cfg.title())}", + f"{self.vs_context.msbuild_prop('debugtype', dbg_type)}", + f"{self.vs_context.msbuild_clp('verbosity', self.verbosity)}", + f"{self.vs_context.msbuild_clp('summary')}" + ]) + build_cmd_steps.append( + f'{build_cmd_prefix} {base_output_path_arg} {self.source_path}') + return ' && '.join(build_cmd_steps) + + def configure(self): + pass + + def build(self): + subprocess.run(self._get_build_cmd()).check_returncode() + + def __str__(self): + return fmt_table( + { + 'name': self.name, + 'build_cmd': self._get_build_cmd() + }) diff --git a/tools/py/core/visual_studio_utils.py b/tools/py/core/visual_studio_utils.py new file mode 100644 index 000000000..795786760 --- /dev/null +++ b/tools/py/core/visual_studio_utils.py @@ -0,0 +1,72 @@ +from pathlib import Path + + +class VisualStudioContext: + def __init__( + self, + version, + year, + prompt_path: Path, + msbuild_path: Path, + msbuild_arg_prefix, + ): + self.version = version + self.year = year + self.prompt_path = prompt_path + self.msbuild_path = msbuild_path + self._msbuild_arg_prefix = msbuild_arg_prefix + + def prompt_cmd(self): + return f'{self.prompt_path} -arch=x64 -host_arch=x64' + + def msbuild_flag(self, flag): + return f'{self._msbuild_arg_prefix}{flag}' + + def msbuild_clp(self, key, value=None): + rv = f'{self._msbuild_arg_prefix}clp:{key}' + if value is not None: + rv = f'{rv}={value}' + return rv + + def msbuild_target(self, target): + return f'{self._msbuild_arg_prefix}t:{target}' + + def msbuild_prop(self, key, value): + return f'{self._msbuild_arg_prefix}p:{key}={value}' + + +def get_supported_vs_contexts(): + '''Gets a list of supported Visual Studio contexts (versions)''' + # vs 17 + vs17_path = Path('C:/Program Files/Microsoft Visual Studio/2022/Community') + vs17 = VisualStudioContext( + 17, + 2022, + vs17_path.joinpath('Common7/Tools/VsDevCmd.bat'), + vs17_path.joinpath('Msbuild/Current/Bin/amd64/MSBuild.exe'), + '-', + ) + + # vs 16 + vs16_path = Path( + 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Community') + vs16 = VisualStudioContext( + 16, + 2019, + vs16_path.joinpath('Common7/Tools/VsDevCmd.bat'), + vs16_path.joinpath('MSBuild/Current/Bin/MSBuild.exe'), + '/', + ) + return [vs17, vs16] + + +def get_preferred_vs_context(): + '''Gets the most preferred Visual Studio context (version) on this computer''' + vs_ctxs = get_supported_vs_contexts() + for vs_context in vs_ctxs: + if vs_context.prompt_path.exists(): + return vs_context + supported_versions = ', '.join([str(v.version) for v in vs_ctxs]) + raise ValueError( + f'Failed to find supported Visual Studio version (works with {supported_versions})' + )