diff --git a/cmake/ProtobufGenerateFs.cmake b/cmake/ProtobufGenerateFs.cmake new file mode 100644 index 000000000..5a44732d1 --- /dev/null +++ b/cmake/ProtobufGenerateFs.cmake @@ -0,0 +1,49 @@ +include_guard(DIRECTORY) + +function(protobuf_generate_fs) + set(_options) + set(_one_value_args TARGET_NAME OUT_DIR PROTOC) + set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES) + cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN}) + + if (NOT DEFINED _ARGS_TARGET_NAME) + set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos") + endif() + if (NOT DEFINED _ARGS_PROTOC) + set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc") + endif() + + set(_proto_include_cli_args) + foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS}) + list(APPEND _proto_include_cli_args -I ${_dir}) + endforeach() + + set(_outputs) + foreach(_proto_source ${_ARGS_PROTO_SOURCES}) + # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE) + get_filename_component(_proto_source_we ${_proto_source} NAME_WE) + # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY) + list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs) + endforeach() + + set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh) + set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py) + set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs) + add_custom_command( + OUTPUT ${_outputs} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND ${_venv_exec} + ${_ARGS_PROTOC} + --plugin=protoc-gen-service=${_plugin_py} + --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR} + ${_proto_include_cli_args} + ${_ARGS_PROTO_SOURCES} + DEPENDS + ${_venv_exec} + ${_plugin_py} + ${_template_fs} + ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json + ${_ARGS_PROTO_SOURCES} + ) + add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}") +endfunction() diff --git a/cmake/ProtobufGenerateJs.cmake b/cmake/ProtobufGenerateJs.cmake new file mode 100644 index 000000000..70f6c0387 --- /dev/null +++ b/cmake/ProtobufGenerateJs.cmake @@ -0,0 +1,33 @@ +include_guard(DIRECTORY) + +function(protobuf_generate_js) + set(_options) + set(_one_value_args TARGET_NAME OUTPUT) + set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES) + cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN}) + + if (NOT DEFINED _ARGS_TARGET_NAME) + set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos") + endif() + + set(_proto_include_cli_args) + foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS}) + list(APPEND _proto_include_cli_args -p ${_dir}) + endforeach() + + add_custom_command( + OUTPUT ${_ARGS_OUTPUT} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND npx + --no-install pbjs + -t json-module + -w commonjs + -o ${_ARGS_OUTPUT} + ${_proto_include_cli_args} + ${_ARGS_PROTO_SOURCES} + DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json + ${_ARGS_PROTO_SOURCES} + ) + add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT}) +endfunction() diff --git a/src/Spark.Admin.WebApp/CMakeLists.txt b/src/Spark.Admin.WebApp/CMakeLists.txt index 7c45a850b..f89cb8bda 100644 --- a/src/Spark.Admin.WebApp/CMakeLists.txt +++ b/src/Spark.Admin.WebApp/CMakeLists.txt @@ -1,57 +1,43 @@ cmake_minimum_required(VERSION 3.5) project(Spark.Admin.WebApp) -include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake) include(GNUInstallDirs) +include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateJs.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake) -# dependencies -set(PROTO_GLOBS - ../lib_domain/domain/*.proto +file( + GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + ../lib_domain/domain/protobufs/*.proto ../lib_spark/spark/api/*.proto ../lib_spark/spark/api/services/*.proto ../lib_spark/spark/domain_model/*.proto ../lib_spark/spark/sparkproto/*.proto ../lib_spark/spark/md/protos/*.proto ../../externals/linux/include/google/protobuf/descriptor.proto - ../../externals/linux/include/google/protobuf/timestamp.proto) + ../../externals/linux/include/google/protobuf/timestamp.proto +) -file( - GLOB_RECURSE _proto_rel_SOURCES - RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} - ${PROTO_GLOBS}) - -file( - GLOB_RECURSE _proto_abs_SOURCES - ${PROTO_GLOBS}) - -file( - GLOB_RECURSE _ts_SOURCES - RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} - src/*.ts) - -file( - GLOB_RECURSE _fs_SOURCES - RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} - src/*.fs) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json + COMMAND npm install --include=dev + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/package.json + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) set(_tools ${CMAKE_SOURCE_DIR}/tools) set(_quiet_exec ${_tools}/quiet_exec.sh) set(_venv_exec ${_tools}/venv_exec.sh) set(_tools_py ${CMAKE_SOURCE_DIR}/tools/py) -# The tsconfig file is generated as a function of the cmake build type, and -# binary dir. Depending on the build type, it extends either -# 'tsconfig.debug.json' or 'tsconfig.release.json'. set(_tsconfig_base "${CMAKE_CURRENT_SOURCE_DIR}/tsconfig.$,debug,release>.json") -set(WEBPACK_MODE $,"development","production">) set(_tsconfig "${CMAKE_CURRENT_BINARY_DIR}/tsconfig.json") set(_venv_python ${CMAKE_SOURCE_DIR}/.venv/bin/python) set(_gen_tsconfig_py ${_tools_py}/gen_tsconfig.py) set(_build_admin_web_app_py ${_tools_py}/build_admin_web_app.py) - -set(_gen_tsconfig_deps ${_venv_exec} ${_venv_python} ${_gen_tsconfig_py} ${_tsconfig_base}) - set(_build_admin_web_app_deps ${_quiet_exec} ${_venv_exec} @@ -61,49 +47,133 @@ set(_build_admin_web_app_deps ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json - ${_proto_rel_SOURCES} + ${${PROJECT_NAME}_PROTO_SOURCES} ${_api_ts_SOURCES} ${_ui_fs_SOURCES} ${_tsconfig}) update_deps_file("${_build_admin_web_app_deps}") -function(npm_install path) - add_custom_command( - OUTPUT ${path}/package-lock.json - COMMAND "npm" ARGS "install" "--include=dev" - DEPENDS ${path}/package.json - WORKING_DIRECTORY ${path} - COMMENT "Run 'npm install' in ${path}") -endfunction() - -npm_install(${CMAKE_CURRENT_SOURCE_DIR}) - add_custom_command( OUTPUT ${_tsconfig} COMMAND ${_venv_exec} ARGS - ${_venv_python} - ${_gen_tsconfig_py} - --config-base ${_tsconfig_base} - --config-out ${_tsconfig} - --build-out ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS ${_gen_tsconfig_deps}) + ${_venv_python} + ${_gen_tsconfig_py} + --config-base ${_tsconfig_base} + --config-out ${_tsconfig} + --build-out ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS + ${_venv_exec} + ${_venv_python} + ${_gen_tsconfig_py} + ${_tsconfig_base} + ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json +) + +protobuf_generate_js( + TARGET_NAME generate_js_protos + PROTO_INCLUDE_DIRS + ../lib_domain + ../lib_spark + ../../externals/linux/include + PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js +) + +protobuf_generate_fs( + TARGET_NAME generate_fs_protos + PROTO_INCLUDE_DIRS + ../lib_domain + ../lib_spark + ../../externals/linux/include + PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable +) + +# Each input file maps to multiple output files with different extensions +# (js, js.map, d.ts, fs). Transpile ts -> d.ts (and js), then d.ts -> fs. +# Who knew front-end development could be so complicated? `\_(=_=)_/` +file( + GLOB_RECURSE ${PROJECT_NAME}_TS_SOURCES + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + src/*.ts +) +set(${PROJECT_NAME}_JS_PROTOS) +set(${PROJECT_NAME}_TS_PROTOS) +set(${PROJECT_NAME}_FS_PROTOS) +foreach(_ts_source ${${PROJECT_NAME}_TS_SOURCES}) + get_filename_component(_ts_source_we ${_ts_source} NAME_WE) + set(_prefix ${CMAKE_CURRENT_BINARY_DIR}/${_ts_source_we}) + list(APPEND ${PROJECT_NAME}_JS_PROTOS ${_prefix}.js ${_prefix}.js.map) + list(APPEND ${PROJECT_NAME}_TS_PROTOS ${_prefix}.d.ts ${_prefix}.d.ts.map) + list(APPEND ${PROJECT_NAME}_FS_PROTOS ${_prefix}.fs) + add_custom_command( + OUTPUT ${_prefix}.fs + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND npx --no-install ts2fable ${_prefix}.d.ts ${_prefix}.fs + DEPENDS ${_prefix}.d.ts + ) +endforeach() add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bundle.js - COMMAND ${_quiet_exec} - ARGS - ${_venv_exec} - ${_venv_python} - ${_build_admin_web_app_py} - --protos ${_proto_abs_SOURCES} - --tsconfig ${_tsconfig} - --mode ${WEBPACK_MODE} - --output-dir ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Building Spark.Admin.WebApp" - DEPENDS ${_build_admin_web_app_deps}) + OUTPUT + ${${PROJECT_NAME}_JS_PROTOS} + ${${PROJECT_NAME}_TS_PROTOS} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND npx + --no-install tsc + -p ${_tsconfig} + DEPENDS + ${_tsconfig} + ${CMAKE_CURRENT_BINARY_DIR}/proto.js +) +add_custom_target( + transpile_type_script + ALL + DEPENDS + ${${PROJECT_NAME}_JS_PROTOS} + ${${PROJECT_NAME}_TS_PROTOS} +) +add_custom_target(generate_fsts_stubs ALL DEPENDS ${${PROJECT_NAME}_FS_PROTOS}) -add_custom_target(admin_web_app_bundle ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bundle.js) +sym_link_dir( + TARGET_NAME link_webapp_public + SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/public + DEST ${CMAKE_CURRENT_BINARY_DIR}/public +) +sym_link_dir( + TARGET_NAME link_webapp_root + SOURCE ${CMAKE_CURRENT_BINARY_DIR} + DEST ${CMAKE_SOURCE_DIR}/build/Spark.Admin.WebApp +) + +file( + GLOB_RECURSE ${PROJECT_NAME}_FS_SOURCES + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + src/*.fs +) +add_custom_command( + OUTPUT + ${CMAKE_CURRENT_BINARY_DIR}/bundle.js + $<$:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map> + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src + COMMAND dotnet restore + COMMAND dotnet tool restore + COMMAND export SPARK_ADMIN_WEB_APP_OUTPUT_DIR=${CMAKE_CURRENT_BINARY_DIR} + COMMAND dotnet fable -o ${CMAKE_CURRENT_BINARY_DIR} + COMMAND npm run build -- --mode $,"development","production"> + DEPENDS + ${${PROJECT_NAME}_FS_SOURCES} + ${${PROJECT_NAME}_FS_PROTOS} + ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj +) +add_custom_target( + bundle_spark_admin_webapp ALL + DEPENDS + ${CMAKE_CURRENT_BINARY_DIR}/bundle.js + $<$:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map> + COMMENT "Bundling Spark.Admin.WebApp" +) install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "${CMAKE_INSTALL_BINDIR}") diff --git a/tools/py/build_admin_web_app.py b/tools/py/build_admin_web_app.py deleted file mode 100644 index cf3081632..000000000 --- a/tools/py/build_admin_web_app.py +++ /dev/null @@ -1,202 +0,0 @@ -from shutil import copytree -import glob -from typing import Iterable -import os -from pathlib import Path -from core.file_utils import CWD, SparkPaths, as_absolute_paths, link_dir, rmdir_or_unlink -from core.build_step import BuildStep -from core.table_formatter import fmt_table -import subprocess -import argparse - - -class BuildArgs: - def __init__(self, args): - self.protos = args.protos - self.mode = args.mode - self.tsconfig = args.tsconfig - self.root_path = SparkPaths.root() - self.src_path = self.root_path.joinpath('src', 'Spark.Admin.WebApp') - self.out_path = Path(args.output_dir).absolute() - self.pbufs_abs = as_absolute_paths( - self.protos, with_respect_to=self.src_path) or [] - - def __repr__(self): - return fmt_table(self.__dict__) - - -def _wrapped_run(*args, require_success=True, **kwargs): - # by default, print header in green and flush buffer immediately to avoid - # non-linear build output - if 'color' not in kwargs: - kwargs['color'] = 'green' - if 'flush' not in kwargs: - kwargs['flush'] = True - cmd = ' '.join(args) - print(f'BEGIN {cmd}', flush=True) - res = subprocess.run(args, shell=True) - if require_success and res.returncode != 0: - raise ValueError( - f'Command \'{cmd}\' exited with code {res.returncode}') - print(f'END {cmd}', flush=True) - - -def _get_arg_parser(): - parser = argparse.ArgumentParser(description='Build spark web server') - - parser.add_argument( - '--mode', - metavar='development|production', - type=str, - nargs='?', - help='The build configuration', - default='development') - - parser.add_argument( - '--tsconfig', - metavar='path', - type=str, - help='The tsconfig file to use', - required=True) - - parser.add_argument( - '-o', - '--output-dir', - metavar='path', - type=str, - nargs='?', - help='The directory in which to build') - - parser.add_argument( - '--protos', - type=str, - nargs='+', - help='The protocol buffers to transpile') - parser.set_defaults(mode='development') - return parser - - -def _gen_fs_pbufs( - root_path: Path, - src_path: Path, - out_path: Path, - protos: Iterable[str], -): - with CWD(src_path): - protos_rel = [os.path.relpath(p, src_path) for p in protos] - protoc_path = root_path.joinpath('externals', 'linux', 'bin', 'protoc') - plugin_script_path = root_path.joinpath( - 'src', 'lib_spark', 'spark', 'api', 'service_compiler', 'plugin.sh') - cmd_parts = [ - f'{protoc_path}', - f'--plugin=protoc-gen-service={plugin_script_path}', - f'--service_out=lang=fs,strip_out_path=1:{out_path.joinpath("fable")}', - f'-I {os.path.relpath(root_path.joinpath("src", "lib_domain"), os.curdir)}', - f'-I {os.path.relpath(root_path.joinpath("src", "lib_spark"), os.curdir)}', - f'-I {os.path.relpath(str(root_path.joinpath("externals", "linux", "include")), os.curdir)}', - ' '.join(protos_rel) - ] - cmd = ' '.join(cmd_parts) - with BuildStep('generate fs protos', cmd): - return subprocess.run(cmd, shell=True) - - -def _gen_js_pbufs(src_path: Path, out_path: Path, protos: Iterable): - with CWD(src_path): - protos_rel = [os.path.relpath(p, src_path) for p in protos] - cmd_parts = [ - f'npx --no-install pbjs', f'-t json-module', f'-w commonjs', - f'-o {out_path.joinpath("proto.js")}', ' '.join(protos_rel) - ] - cmd = ' '.join(cmd_parts) - with BuildStep('generate js protocol buffers', cmd): - return subprocess.run(cmd, shell=True) - - -def _gen_fsts_stubs(api_source_path: Path, api_output_path: Path): - def gen_fs(ts): - # The triple 'with_suffix' chain isn't a typo. It's a hack to remove - # (up to) 2 suffixes to account for the compound ext '.d.ts'. - fs_out = Path(ts).with_suffix('').with_suffix('').with_suffix('.fs') - subprocess.run(f'npx --no-install ts2fable {ts} {fs_out}', shell=True) - - # Generate fs stubs from typescript that was just built - typescripts = glob.glob(f'{api_output_path.joinpath("*.d.ts")}') - with BuildStep('generate fs -> ts stubs', - '\n'.join(typescripts)), CWD(api_source_path): - return [gen_fs(ts) for ts in typescripts] - - -def _tsc(src_path, tscfg_path): - with BuildStep('tsc', str(tscfg_path)), CWD(src_path): - return subprocess.run( - f'npx --no-install tsc -p {tscfg_path}', shell=True) - - -def _fable_webpack(src_path, out_path, mode): - with BuildStep('fable webpack'): - copytree(str(src_path.joinpath('public')), out_path.joinpath('public')) - os.chdir(src_path.joinpath('src')) - _wrapped_run('dotnet restore') - _wrapped_run('dotnet tool restore') - # Note: We used to convert fs -> js via webpack, using the node pkgs - # fable-compiler & fable-loader. However, as of 02/24/2023, they are - # incompatible w/ net7.0. Fortunately the dotnet tool 'fable' IS - # compatible w/ net7.0. - _wrapped_run(f'dotnet fable -o {out_path}') - # Run webpack via 'npm run build' - # (the build cmd is defined in package.json) - _wrapped_run(f'npm run build -- --mode {mode}') - - -def _main(): - parser = _get_arg_parser() - args = BuildArgs(parser.parse_args()) - - # needed bc 'Syntax Error: f-string expression part cannot include a backslash' - newline = '\n' - print( - f'protos: {(newline.join([str(p) for p in args.protos]))}\n\n', - flush=True) - # setup dirs/env vars - rmdir_or_unlink(args.out_path.joinpath('public')) - rmdir_or_unlink(args.out_path.joinpath('fable')) - os.makedirs(args.out_path.joinpath('fable')) - os.putenv('SPARK_ADMIN_WEB_APP_OUTPUT_DIR', str(args.out_path)) - # This build is part of the larger CMake 'sparkd' build, which means the - # output dir depends on the CMake 'configure' step. Dotnet isn't privy to - # CMake configuration, so the output dir is passed to this script as a - # command line argument (per a custom target defined CMakeLists). Since the - # output dir isn't defined in a dotnet proj file, intellisense won't know - # where to look for generated source files. As a workaround, link the - # actual output dir to a path defined in the fsproj file. - out_proxy = SparkPaths.root().joinpath('build', 'Spark.Admin.WebApp') - if args.out_path != out_proxy: - rmdir_or_unlink(out_proxy) - link_dir(args.out_path, out_proxy) - - - os.chdir(args.src_path) - - # 1) Build js protobufs - # _gen_js_pbufs(args.src_path, args.out_path, args.pbufs_abs) - - # 2) Build fs protobufs - # _gen_fs_pbufs(args.root_path, args.src_path, args.out_path, args.pbufs_abs) - - # 3) Build typescript - # _tsc(args.src_path, args.tsconfig) - - # 4) Generate fs stubs from typescript that was just built - _gen_fsts_stubs(args.src_path, args.out_path) - - # 5) Build fs fable (webpack to js) - _fable_webpack(args.src_path, args.out_path, args.mode) - - -if __name__ == '__main__': - try: - _main() - except Exception as e: - print(f'build_admin_web_app.py failed: {e}') - exit(1)