from shutil import copytree import glob from typing import Iterable import os from pathlib import Path from core.file_utils import CWD, SparkPaths, as_absolute_paths, link_dir, rmdir_or_unlink from core.build_step import BuildStep from core.logging import exec_and_log_error from core.table_formatter import fmt_table import subprocess import argparse class BuildArgs: def __init__(self, args): self.protos = args.protos self.mode = args.mode self.tsconfig = args.tsconfig self.root_path = SparkPaths.root() self.src_path = self.root_path.joinpath('src', 'Spark.Admin.WebApp') self.out_path = Path(args.output_dir).absolute() self.pbufs_abs = as_absolute_paths( self.protos, with_respect_to=self.src_path) or [] def __repr__(self): return fmt_table(self.__dict__) def _wrapped_run(*args, require_success=True, **kwargs): # by default, print header in green and flush buffer immediately to avoid # non-linear build output if 'color' not in kwargs: kwargs['color'] = 'green' if 'flush' not in kwargs: kwargs['flush'] = True cmd = ' '.join(args) print(f'BEGIN {cmd}', flush=True) res = subprocess.run(args, shell=True) if require_success and res.returncode != 0: raise ValueError( f'Command \'{cmd}\' exited with code {res.returncode}') print(f'END {cmd}', flush=True) def _get_arg_parser(): parser = argparse.ArgumentParser(description='Build spark web server') parser.add_argument( '--mode', metavar='development|production', type=str, nargs='?', help='The build configuration', default='development') parser.add_argument( '--tsconfig', metavar='path', type=str, help='The tsconfig file to use', required=True) parser.add_argument( '-o', '--output-dir', metavar='path', type=str, nargs='?', help='The directory in which to build') parser.add_argument( '--protos', type=str, nargs='+', help='The protocol buffers to transpile') parser.set_defaults(mode='development') return parser def _gen_fs_pbufs( root_path: Path, src_path: Path, out_path: Path, protos: Iterable[str], ): with CWD(src_path): protos_rel = [os.path.relpath(p, src_path) for p in protos] protoc_path = root_path.joinpath('externals', 'linux', 'bin', 'protoc') plugin_script_path = root_path.joinpath( 'src', 'lib', 'api', 'service_compiler', 'plugin.sh') cmd_parts = [ f'{protoc_path}', f'--plugin=protoc-gen-service={plugin_script_path}', f'--service_out=lang=fs,strip_out_path=1:{out_path.joinpath("fable")}', f'-I {os.path.relpath(root_path.joinpath("src"), os.curdir)}', f'-I {os.path.relpath(str(root_path.joinpath("externals", "linux", "include")), os.curdir)}', ' '.join(protos_rel) ] cmd = ' '.join(cmd_parts) with BuildStep('generate fs protos', cmd): return subprocess.run(cmd, shell=True) def _gen_js_pbufs(src_path: Path, out_path: Path, protos: Iterable): with CWD(src_path): protos_rel = [os.path.relpath(p, src_path) for p in protos] cmd_parts = [ f'npx --no-install pbjs', f'-t json-module', f'-w commonjs', f'-o {out_path.joinpath("proto.js")}', ' '.join(protos_rel) ] cmd = ' '.join(cmd_parts) with BuildStep('generate js protocol buffers', cmd): return subprocess.run(cmd, shell=True) def _gen_fsts_stubs(api_source_path: Path, api_output_path: Path): def gen_fs(ts): # The triple 'with_suffix' chain isn't a typo. It's a hack to remove # (up to) 2 suffixes to account for the compound ext '.d.ts'. fs_out = Path(ts).with_suffix('').with_suffix('').with_suffix('.fs') subprocess.run(f'npx --no-install ts2fable {ts} {fs_out}', shell=True) # Generate fs stubs from typescript that was just built typescripts = glob.glob(f'{api_output_path.joinpath("*.d.ts")}') with BuildStep('generate fs -> ts stubs', '\n'.join(typescripts)), CWD(api_source_path): return [gen_fs(ts) for ts in typescripts] def _tsc(src_path, tscfg_path): with BuildStep('tsc', str(tscfg_path)), CWD(src_path): return subprocess.run( f'npx --no-install tsc -p {tscfg_path}', shell=True) def _fable_webpack(src_path, out_path, mode): with BuildStep('fable webpack'): copytree(str(src_path.joinpath('public')), out_path.joinpath('public')) os.chdir(src_path.joinpath('src')) _wrapped_run('dotnet restore') _wrapped_run('dotnet tool restore') # Note: We used to convert fs -> js via webpack, using the node pkgs # fable-compiler & fable-loader. However, as of 02/24/2023, they are # incompatible w/ net7.0. Fortunately the dotnet tool 'fable' IS # compatible w/ net7.0. _wrapped_run(f'dotnet fable -o {out_path}') # Run webpack via 'npm run build' # (the build cmd is defined in package.json) _wrapped_run(f'npm run build -- --mode {mode}') def _main(): parser = _get_arg_parser() args = BuildArgs(parser.parse_args()) # needed bc 'Syntax Error: f-string expression part cannot include a backslash' newline = '\n' print( f'protos: {(newline.join([str(p) for p in args.protos]))}\n\n', flush=True) # setup dirs/env vars rmdir_or_unlink(args.out_path.joinpath('public')) rmdir_or_unlink(args.out_path.joinpath('fable')) os.makedirs(args.out_path.joinpath('fable')) os.putenv('SPARK_ADMIN_WEB_APP_OUTPUT_DIR', str(args.out_path)) # This build is part of the larger CMake 'sparkd' build, which means the # output dir depends on the CMake 'configure' step. Dotnet isn't privy to # CMake configuration, so the output dir is passed to this script as a # command line argument (per a custom target defined CMakeLists). Since the # output dir isn't defined in a dotnet proj file, intellisense won't know # where to look for generated source files. As a workaround, link the # actual output dir to a path defined in the fsproj file. out_proxy = SparkPaths.root().joinpath('build', 'Spark.Admin.WebApp') if args.out_path != out_proxy: rmdir_or_unlink(out_proxy) link_dir(args.out_path, out_proxy) os.chdir(args.src_path) # 1) Build js protobufs _gen_js_pbufs(args.src_path, args.out_path, args.pbufs_abs) # 2) Build fs protobufs _gen_fs_pbufs(args.root_path, args.src_path, args.out_path, args.pbufs_abs) # 3) Build typescript _tsc(args.src_path, args.tsconfig) # 4) Generate fs stubs from typescript that was just built _gen_fsts_stubs(args.src_path, args.out_path) # 5) Build fs fable (webpack to js) _fable_webpack(args.src_path, args.out_path, args.mode) if __name__ == '__main__': exec_and_log_error(_main)