Files
spark/tools/py/build_web.py
T

201 lines
6.9 KiB
Python

from shutil import copytree
import json
import glob
from time import sleep
from typing import Iterable, cast
import termcolor
import os
from pathlib import Path
from core.file_utils import SparkPaths, as_absolute_paths, rmdir_or_unlink
from core.build_step import BuildStep
from core.logging import exec_and_log_error
from core.table_formatter import fmt_table
from core.type_utils import notnone
import shutil
import subprocess
import argparse
class CWD:
def __init__(self, push_path, pop_path=os.getcwd()):
self.push_path = push_path
self.pop_path = pop_path
def __enter__(self):
if self.push_path is not None:
os.chdir(self.push_path)
def __exit__(self, exc_type, exc_val, exc_tb):
if self.pop_path is not None:
os.chdir(self.pop_path)
class BuildWebArgs:
def __init__(self, args):
self.protos = args.protos
self.webpack_mode = args.mode
self.tsconfig = args.tsconfig
self.build_type = 'debug' if self.webpack_mode == 'development' else 'release'
self.build_type = self.build_type.lower()
self.root_path = SparkPaths.root()
self.src_path = self.root_path.joinpath('src', 'web')
self.pbuf_src_path = self.src_path.joinpath('api')
self.out_path = Path(args.output_dir).absolute()
self.pbuf_out_path = self.out_path.joinpath('api')
self.ui_out_path = self.out_path.joinpath('ui')
self.pbufs_abs = as_absolute_paths(self.protos, with_respect_to=self.src_path) or []
def __repr__(self):
return fmt_table(self.__dict__)
def _wrapped_run(*args, require_success=True, **kwargs):
# by default, print header in green and flush buffer immediately to avoid
# non-linear build output
if 'color' not in kwargs:
kwargs['color'] = 'green'
if 'flush' not in kwargs:
kwargs['flush'] = True
cmd = ' '.join(args)
print(f'BEGIN {cmd}', flush=True)
res = subprocess.run(args, shell=True)
if require_success and res.returncode != 0:
raise ValueError(
f'Command \'{cmd}\' exited with code {res.returncode}')
print(f'END {cmd}', flush=True)
def _get_arg_parser():
parser = argparse.ArgumentParser(description='Build spark web server')
parser.add_argument(
'--mode',
metavar='development|production',
type=str,
nargs='?',
help='The build configuration',
default='development')
parser.add_argument(
'--tsconfig',
metavar='path',
type=str,
help='The tsconfig file to use',
required=True)
parser.add_argument(
'-o',
'--output-dir',
metavar='path',
type=str,
nargs='?',
help='The directory in which to build')
parser.add_argument(
'--protos',
type=str,
nargs='+',
help='The protocol buffers to transpile')
parser.set_defaults(mode='development')
return parser
def _gen_fs_pbufs(root_path: Path, src_path: Path, out_path: Path, protos: Iterable[str]):
with CWD(src_path):
protos_rel = [os.path.relpath(p, src_path) for p in protos]
protoc_path = root_path.joinpath('externals', 'linux', 'bin', 'protoc')
plugin_script_path = root_path.joinpath(
'src', 'lib', 'api', 'service_compiler', 'plugin.py')
cmd_parts = [
f'{protoc_path}', f'--plugin=protoc-gen-service={plugin_script_path}',
f'--service_out=lang=fs,strip_out_path=1:{out_path.joinpath("fable")}',
f'-I {os.path.relpath(root_path.joinpath("src"), os.curdir)}',
f'-I {os.path.relpath(str(root_path.joinpath("externals", "linux", "include")), os.curdir)}',
' '.join(protos_rel)
]
cmd = ' '.join(cmd_parts)
with BuildStep('generate fs protos', cmd):
return subprocess.run(cmd, shell=True)
def _gen_js_pbufs(src_path: Path, out_path: Path, protos: Iterable):
with CWD(src_path):
protos_rel = [os.path.relpath(p, src_path) for p in protos]
cmd_parts = [
f'npx --no-install pbjs', f'-t json-module', f'-w commonjs',
f'-o {out_path.joinpath("proto.js")}', ' '.join(protos_rel)
]
cmd = ' '.join(cmd_parts)
with BuildStep('generate js protocol buffers', cmd):
return subprocess.run(cmd, shell=True)
def _gen_fsts_stubs(api_source_path: Path, api_output_path: Path):
def gen_fs(ts):
# The triple 'with_suffix' chain isn't a typo. It's a hack to remove
# (up to) 2 suffixes to account for the compound ext '.d.ts'.
fs_out = Path(ts).with_suffix('').with_suffix('').with_suffix('.fs')
subprocess.run(f'npx --no-install ts2fable {ts} {fs_out}', shell=True)
# Generate fs stubs from typescript that was just built
typescripts = glob.glob(f'{api_output_path.joinpath("*.d.ts")}')
with BuildStep('generate fs -> ts stubs',
'\n'.join(typescripts)), CWD(api_source_path):
return [gen_fs(ts) for ts in typescripts]
def _tsc(api_src_path, tscfg_path):
with BuildStep('tsc', str(tscfg_path)), CWD(api_src_path):
return subprocess.run(f'npx --no-install tsc -p {tscfg_path}', shell=True)
def _fable_webpack(src_path, ui_out_path):
with BuildStep('fable webpack'):
copytree(str(src_path.joinpath('ui', 'public')), str(ui_out_path))
os.chdir(src_path.joinpath('ui', 'src'))
_wrapped_run('dotnet restore')
_wrapped_run('dotnet tool restore')
# Note: We used to convert fs -> js via webpack, using the node pkgs
# fable-compiler & fable-loader. However, as of 02/24/2023, they are
# incompatible w/ net7.0. Fortunately the dotnet tool 'fable' IS
# compatible w/ net7.0.
_wrapped_run(f'dotnet fable -o {ui_out_path}')
# Run webpack via 'npm run build'
# (the build cmd is defined in package.json)
_wrapped_run('npm run build')
def _main():
parser = _get_arg_parser()
args = BuildWebArgs(parser.parse_args())
# needed bc 'Syntax Error: f-string expression part cannot include a backslash'
newline = '\n'
print(f'protos: {(newline.join([str(p) for p in args.protos]))}\n\n', flush=True)
# setup dirs/env vars
rmdir_or_unlink(args.pbuf_out_path)
rmdir_or_unlink(args.ui_out_path)
os.makedirs(args.pbuf_out_path.joinpath('fable'))
os.putenv('WEB_BUILD_PATH', str(args.out_path))
os.chdir(args.pbuf_src_path)
# 1) Build js protobufs
_gen_js_pbufs(args.pbuf_src_path, args.pbuf_out_path, args.pbufs_abs)
# 2) Build fs protobufs
_gen_fs_pbufs(args.root_path, args.pbuf_src_path, args.pbuf_out_path, args.pbufs_abs)
# 3) Build typescript
_tsc(args.pbuf_src_path, args.tsconfig)
# 4) Generate fs stubs from typescript that was just built
_gen_fsts_stubs(args.pbuf_src_path, args.pbuf_out_path)
# 5) Build fs fable (webpack to js)
_fable_webpack(args.src_path, args.ui_out_path)
if __name__ == '__main__':
exec_and_log_error(_main)