check point

TODO: Add docs for building Spark on Windows

.clang-format

@@ -4,6 +4,7 @@ BinPackParameters: false
 AlignAfterOpenBracket: AlwaysBreak
 AlwaysBreakTemplateDeclarations: true
 PenaltyReturnTypeOnItsOwnLine: 140
+ReflowComments: false
 ---
 Language: Proto
 IndentWidth: 2

.config/dotnet-tools.json

@@ -7,6 +7,24 @@
       "commands": [
         "dotnet-format"
       ]
+    },
+    "fable": {
+      "version": "4.0.0-theta-018",
+      "commands": [
+        "fable"
+      ]
+    },
+    "fantomas": {
+      "version": "5.2.2",
+      "commands": [
+        "fantomas"
+      ]
+    },
+    "fake-cli": {
+      "version": "6.0.0",
+      "commands": [
+        "fake"
+      ]
     }
   }
 }

.gitignore

@@ -50,4 +50,8 @@ trace_data/
 *.tmp/*
 /.venv*/*
 .cache/
-src/web/api/tsconfig.*.g.json
+/src/Spark.Admin.WebApp/tsconfig.*.g.json
+/src/Spark.Admin.WebApp/src/bin/
+**/CMakeDeps.cmake
+**/CMakeDeps_win.cmake
+**/devcontainer.json

.gitmodules

@@ -1,9 +1,3 @@
 [submodule "externals"]
 	path = externals
 	url = git@bitbucket.org:bluetradesys/externals.git
-[submodule "docker/linux"]
-	path = docker/linux
-	url = git@bitbucket.org:bluetradesys/linux_docker.git
-[submodule "docker/windows"]
-	path = docker/windows
-	url = git@bitbucket.org:bluetradesys/windows_docker.git

CMakeLists.txt

@@ -13,101 +13,182 @@ set(
   CACHE
   STRING
   "External dependencies root folder (should contain folders: lib, bin, include")
-set(SPARK_PYTHON "${EXTERNALS_ROOT}/python")
 set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
 set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
-set(SPARK_LIB "${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}")
+set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(BUILD_SPARK_CONFIG OFF CACHE BOOL "Build lib_spark_config")
+set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
+set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
-set(BUILD_WEB ON CACHE BOOL "Build the web server")
+set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
+if (WIN32)
+  set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
+else()
+  set(BUILD_SPARKD ON CACHE BOOL "Build sparkd")
+endif()
 
 include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
 link_directories(${SPARK_LIB})
 
 if(WIN32)
-  # flags
-  set(
-    CMAKE_CXX_FLAGS
-    "${CMAKE_CXX_FLAGS} \
-    /MP \
-    /WX /wd4251 /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 \
-    /bigobj \
-    /DCURL_STATICLIB \
-    /D_WINSOCK_DEPRECATED_NO_WARNINGS \
-    /D__TBB_NO_IMPLICIT_LINKAGE \
-    /DBOOST_ALL_NO_LIB \
-    /std:c++17 \
-    /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} /MT")
-  set(
-    CMAKE_SHARED_LINKER_FLAGS_RELEASE
-    "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} \
-    /DEBUG \
-    /OPT:REF \
-    /OPT:ICF \
-    /LTCG")
-
+  add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
+  add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
   # subdirs
+  add_subdirectory(src/lib_expression)
   add_subdirectory(src/lib)
+elseif(BUILD_SPARK_CONFIG)
+  add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
+  add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
+  add_subdirectory(src/lib_core)
+  add_subdirectory(src/lib_logging)
+  add_subdirectory(src/lib_io)
+  add_subdirectory(src/lib_domain)
+  add_subdirectory(src/lib_protobuf)
+  add_subdirectory(src/lib_threading)
+  add_subdirectory(src/lib_expression)
+  add_subdirectory(src/lib_md)
+  add_subdirectory(src/lib_spark_config)
 else()
   # make sparkd look for libspark.so in its directory
   set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
   set(CMAKE_INSTALL_RPATH ".")
-  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install/${CMAKE_BUILD_TYPE}/sparkd")
-  add_compile_definitions(SPARK_ENABLE_CCAPI)
-  add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_FTX)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
-  add_compile_definitions(CCAPI_ENABLE_LOG_WARN)
+  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
+  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
 
-  # flags
-  set(
-    CMAKE_CXX_FLAGS_DEBUG
-    "${CMAKE_CXX_FLAGS_DEBUG} \
-    -fPIC \
-    -Wall \
-    -pedantic \
-    -Wextra \
-    -Werror \
-    -Wno-missing-braces \
-    -Wno-ignored-qualifiers \
-    --std=c++17 -fdiagnostics-color")
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -DNDEBUG -O3")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG")
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wno-undef")
+  if(BUILD_API_SERVER)
+    add_compile_definitions(SPARK_ENABLE_API_SERVER)
+    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
+    add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
+  elseif(BUILD_SPARKD)
+    add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
+    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
+
+    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
+    # read by their code). We override some below.
+    add_compile_definitions(SPARK_ENABLE_CCAPI)
+
+    # This allows us to override hardcoded exchange-urls with ones read from
+    # config (i.e., resolved at runtime rather than compile-time)
+    add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
+
+    # Enable ccapi exchanges/services
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_FTX)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_HUOBI)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_HUOBI_USDT_SWAP)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_HUOBI_COIN_SWAP)
+
+    set(CCAPI_LOG_LEVEL
+      "Warn"
+      CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
+    string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
+    add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
+
+    # ** Binance **
+    add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
+    add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+
+    # ** Binance US **
+    add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+
+    # ** Binance USDS Futures **
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+
+  endif()
 
   # subdirs
-  add_subdirectory(src/server/sparkd)
+  if (BUILD_SPARKD)
+    add_subdirectory(src/server/sparkd)
+    add_subdirectory(src/data_files)
+    # add_subdirectory(src/lib_md)
+    add_subdirectory(src/lib_core)
+    add_subdirectory(src/lib_logging)
+    add_subdirectory(src/lib_threading)
+    add_subdirectory(src/lib_expression)
+    add_subdirectory(src/lib_spark_config)
+  endif()
   add_subdirectory(src/lib)
-  if (BUILD_WEB)
-    add_subdirectory(src/web)
+  if (BUILD_ADMIN_WEB_APP)
+    add_subdirectory(src/Spark.Admin.WebApp)
+  endif()
+  if (BUILD_API_SERVER)
+    set(_cs_source_dirs
+      spark_api
+      spark_api_server
+      spark_core
+      spark_websocket)
+    foreach(cs_source_dir IN LISTS _cs_source_dirs)
+      file(GLOB cs_source_dir "${CMAKE_SOURCE_DIR}/src/client/${cs_source_dir}/*.cs")
+      list(APPEND _api_server_sources ${cs_source_dir})
+    endforeach()
+
+    list(APPEND _api_server_sources
+      ${CMAKE_SOURCE_DIR}/src/client/spark_api/spark_api.csproj
+      ${CMAKE_SOURCE_DIR}/src/client/spark_api_server/spark_api_server.csproj
+      ${CMAKE_SOURCE_DIR}/src/client/spark_core/spark_core.csproj
+      ${CMAKE_SOURCE_DIR}/src/client/spark_websocket/spark_websocket.csproj)
+
+    set(_out_dir ${CMAKE_BINARY_DIR}/$<CONFIG>)
+    set(_api_server_dlls
+      ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0/spark_api.dll
+      ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0/spark_core.dll
+      ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0/spark_websocket.dll
+      ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0/spark_api_server)
+    add_custom_command(
+      OUTPUT ${_api_server_dlls}
+      COMMAND dotnet build ${CMAKE_SOURCE_DIR}/Spark.Api.Server.sln --verbosity minimal --configuration $<CONFIG>
+      COMMENT "Building Spark.Api.Server.sln"
+      DEPENDS ${_api_server_sources})
+    add_custom_command(
+      OUTPUT
+      ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0/libspark.so
+      COMMAND ln -sf ARGS ${CMAKE_BINARY_DIR}/src/lib/$<CONFIG>/libspark.so ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0
+      COMMENT "Symlink libspark to api server bins"
+      DEPENDS ${_api_server_dlls})
+    add_custom_target(websocket_api_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/$<CONFIG>/bin/net7.0/libspark.so)
+  endif()
+  if(BUILD_DOCS)
+    add_subdirectory(doc/user)
   endif()
-  add_subdirectory(doc/user)
   if(BUILD_SDK)
     add_subdirectory(src/sdk)
   endif()
-  add_subdirectory(src/data_files)
 
-  add_custom_command(
-    OUTPUT ${CMAKE_BINARY_DIR}/server
-    COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
-    COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
-    COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/web ${CMAKE_BINARY_DIR}/server
-    COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
-    COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-    COMMAND
-      ln ARGS -sf
-      ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
-      ${CMAKE_BINARY_DIR}/server
-    COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
-    COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
-    COMMENT "Creating server runtime layout"
-    DEPENDS sparkd data_files_dir)
+  if(BUILD_SPARKD)
+    add_custom_command(
+      OUTPUT ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND
+        ln ARGS -sf
+        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
+        ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
+      COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
+      COMMENT "Creating server runtime layout"
+      DEPENDS sparkd data_files_dir)
 
-  add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+    add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
 
-  install(DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins" DESTINATION bin)
-  install(FILES "${CMAKE_SOURCE_DIR}/externals/linux/lib/${CMAKE_BUILD_TYPE}/libACE-5.7.5.so" DESTINATION bin)
-  install(FILES "${CMAKE_SOURCE_DIR}/externals/linux/lib/${CMAKE_BUILD_TYPE}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0" DESTINATION bin)
-  install(FILES "${CMAKE_SOURCE_DIR}/externals/src/emumba/sparkd.lic" DESTINATION bin)
+    install(
+      DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+    install(
+      FILES
+      "${SPARK_LIB}/libACE-5.7.5.so"
+      "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
+      "${CMAKE_SOURCE_DIR}/externals/src/emumba/sparkd.lic"
+      DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
+  endif()
 endif()

CMakePresets.json

@@ -0,0 +1,283 @@
+{
+  "version": 4,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 23,
+    "patch": 0
+  },
+
+  "configurePresets": [
+    {
+      "name": "spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "generator": "Visual Studio 17 2022",
+      "binaryDir": "${sourceDir}/build",
+      "environment": {
+        "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++17 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
+      },
+      "cacheVariables": {
+        "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
+        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
+        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
+        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/DEBUG /OPT:REF /OPT:ICF /LTCG",
+        "EXTERNALS_ROOT": "${sourceDir}/externals/windows"
+      }
+    },
+
+    {
+      "name": "sparkd",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "hidden": true,
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "platform": "",
+        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CMAKE_INSTALL_RPATH": ".",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
+        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
+        "CMAKE_CXX_STANDARD": "17",
+        "BUILD_SDK": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_ADMIN_WEB_APP": {
+          "type": "BOOL",
+          "value": "ON"
+        }
+      }
+    },
+
+    {
+      "name": "sparkd-debug",
+      "inherits": "sparkd",
+      "hidden": true,
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CCAPI_LOG_LEVEL": "Debug"
+      }
+    },
+
+    {
+      "name": "sparkd-release",
+      "inherits": "sparkd",
+      "hidden": true,
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "CCAPI_LOG_LEVEL": "Warn"
+      }
+    },
+
+    {
+      "name": "sparkd-debug-clang",
+      "inherits": "sparkd-debug",
+      "displayName": "Debug (clang)",
+      "description": "Debug build compiled w/ clang",
+      "binaryDir": "${sourceDir}/build/clang/debug",
+      "cacheVariables": {
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++17 -fdiagnostics-color -DDEBUG",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
+      }
+    },
+
+    {
+      "name": "sparkd-debug-gcc",
+      "inherits": "sparkd-debug",
+      "displayName": "Debug (gcc)",
+      "description": "Debug build compiled w/ gcc",
+      "binaryDir": "${sourceDir}/build/gcc/debug",
+      "cacheVariables": {
+        "CMAKE_C_COMPILER": "gcc",
+        "CMAKE_CXX_COMPILER": "g++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++17 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
+      }
+    },
+
+    {
+      "name": "sparkd-release-clang",
+      "inherits": "sparkd-release",
+      "displayName": "Release (clang)",
+      "description": "Release build compiled w/ clang",
+      "binaryDir": "${sourceDir}/build/clang/release",
+      "cacheVariables": {
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++17 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
+      }
+    },
+
+    {
+      "name": "sparkd-release-gcc",
+      "inherits": "sparkd-release",
+      "displayName": "Release (gcc)",
+      "description": "Release build compiled w/ gcc",
+      "binaryDir": "${sourceDir}/build/gcc/release",
+      "cacheVariables": {
+        "CMAKE_C_COMPILER": "gcc",
+        "CMAKE_CXX_COMPILER": "g++",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++17 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
+      }
+    },
+
+    {
+      "name": "sparkd-publish",
+      "inherits": "sparkd-release-gcc",
+      "displayName": "Publish",
+      "description": "Production build compiled w/ gcc and ready to ship"
+    },
+
+    {
+      "name": "spark-config",
+      "displayName": "Spark Config",
+      "description": "Test build for lib_spark_config",
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "generator": "Unix Makefiles",
+      "cacheVariables": {
+        "platform": "linux",
+        "CMAKE_BUILD_TYPE": "Debug",
+        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "CMAKE_INSTALL_RPATH": ".",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
+        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
+        "CMAKE_CXX_STANDARD": "17",
+        "BUILD_SPARKD": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_DOCS": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SDK": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SPARK_CONFIG": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "BUILD_ADMIN_WEB_APP": {
+          "type": "BOOL",
+          "value": "OFF"
+        }
+      }
+    }
+  ],
+
+  "buildPresets": [
+    {
+      "name": "spark",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
+      "hidden": true,
+      "jobs": 4
+    },
+
+    {
+      "name": "spark-dev",
+      "displayName": "Development",
+      "inherits": "spark",
+      "configurePreset": "spark",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "spark-prod",
+      "displayName": "Production",
+      "inherits": "spark",
+      "configurePreset": "spark",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "sparkd",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "hidden": true,
+      "jobs": 4,
+      "targets": [
+        "data_files_dir",
+        "admin_web_app_bundle",
+        "install",
+        "libspark",
+        "rapidjson_project",
+        "server_layout",
+        "sparkd"
+      ]
+    },
+
+    {
+      "name": "sparkd-debug-clang",
+      "configurePreset": "sparkd-debug-clang",
+      "displayName": "Debug (clang)",
+      "inherits": "sparkd"
+    },
+
+    {
+      "name": "sparkd-debug-gcc",
+      "configurePreset": "sparkd-debug-gcc",
+      "displayName": "Debug (gcc)",
+      "inherits": "sparkd"
+    },
+
+    {
+      "name": "sparkd-release-clang",
+      "configurePreset": "sparkd-release-clang",
+      "displayName": "Release (clang)",
+      "inherits": "sparkd"
+    },
+
+    {
+      "name": "sparkd-release-gcc",
+      "configurePreset": "sparkd-release-gcc",
+      "displayName": "Release (gcc)",
+      "inherits": "sparkd"
+    },
+
+    {
+      "name": "spark-config",
+      "configurePreset": "spark-config",
+      "displayName": "Spark Config",
+      "inherits": "sparkd"
+    },
+
+    {
+      "name": "sparkd-publish",
+      "configurePreset": "sparkd-publish",
+      "displayName": "Publish",
+      "inherits": "sparkd",
+      "targets": "install"
+    }
+  ]
+}

Spark.Api.Server.sln

@@ -0,0 +1,83 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32922.545
+MinimumVisualStudioVersion = 15.0.26124.0
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		MinSizeRel|x64 = MinSizeRel|x64
+		Release|x64 = Release|x64
+		RelWithDebInfo|x64 = RelWithDebInfo|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.ActiveCfg = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Debug|x64.Build.0 = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.MinSizeRel|x64.Build.0 = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.ActiveCfg = Release|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.Release|x64.Build.0 = Release|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
+		{EFC9FC4D-E92B-4C6D-9B97-26D38EAA118D}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.ActiveCfg = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Debug|x64.Build.0 = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.MinSizeRel|x64.Build.0 = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.ActiveCfg = Release|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
+		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
+	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
+EndGlobal

cmake/Protobuf.cmake

@@ -16,13 +16,26 @@ function(Protobuf_Generate_Cpp SRCS HDRS)
 
     list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
     list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
-
+    set(PROTOBUF_INCLUDE_DIR_ARGS ${CMAKE_CURRENT_SOURCE_DIR})
+    # list(APPEND PROTOBUF_INCLUDE_DIR_ARGS " -I ")
+    # list(APPEND PROTOBUF_INCLUDE_DIR_ARGS ${CMAKE_CURRENT_SOURCE_DIR})
+    foreach(INC ${PROTOBUF_INCLUDE_DIRS})
+      set(PROTOBUF_INCLUDE_DIR_ARGS ${PROTOBUF_INCLUDE_DIR_ARGS} -I ${INC})
+      # list(APPEND PROTOBUF_INCLUDE_DIR_ARGS " -I ")
+      # list(APPEND PROTOBUF_INCLUDE_DIR_ARGS ${INC})
+    endforeach()
+    set(PROTOBUF_INCLUDE_DIR_ARGS ${PROTOBUF_INCLUDE_DIR_ARGS} -I ${CMAKE_CURRENT_BINARY_DIR})
+    # string(JOIN " -I " PROTOBUF_INCLUDE_DIR_ARGS_ ${PROTOBUF_INCLUDE_DIR_ARGS})
+    message("pb inc: ${PROTOBUF_INCLUDE_DIR_ARGS}")
+    # set(cmd_text "${Protobuf_PROTOC_EXECUTABLE} --cpp_out=${CMAKE_CURRENT_BINARY_DIR} -I ${PROTOBUF_INCLUDE_DIR_ARGS_} -I ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL}")
+    message("CMD: ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_CURRENT_BINARY_DIR} -I ${PROTOBUF_INCLUDE_DIR_ARGS} ${ABS_FIL}")
     add_custom_command(
       OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
              "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
-      COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src -I
-        ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+      COMMAND ${Protobuf_PROTOC_EXECUTABLE} --cpp_out=${CMAKE_CURRENT_BINARY_DIR} -I ${PROTOBUF_INCLUDE_DIR_ARGS} ${ABS_FIL}
+      # COMMAND
+      #   ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_CURRENT_BINARY_DIR} -I
+      #   ${PROTOBUF_INCLUDE_DIR_ARGS} ${ABS_FIL}
       DEPENDS ${ABS_FIL}
       COMMENT "Running C++ protocol buffer compiler on ${FIL}"
       VERBATIM)
@@ -90,10 +103,10 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
         COMMAND SET PYTHONPATH=${SPARK_PYTHON}
         COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
         COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_BINARY_DIR}/src
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${CMAKE_CURRENT_BINARY_DIR}
           --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.cmd
-          --service_out=lang=cpp:${CMAKE_BINARY_DIR}/src -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+          --service_out=lang=cpp:${CMAKE_CURRENT_BINARY_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL}
         DEPENDS ${ABS_FIL} ${TEMPLATES}
                 ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
         COMMENT "Running C++ service compiler on ${FIL}"
@@ -109,10 +122,10 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
         COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
         ARGS
           ${Protobuf_PROTOC_EXECUTABLE}
-          --cpp_out=${CMAKE_BINARY_DIR}/src
+          --cpp_out=${CMAKE_CURRENT_BINARY_DIR}
           --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py
-          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src
-          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_CURRENT_BINARY_DIR}
+          -I ${PROTOBUF_INCLUDE_DIRS} -I ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL}
         DEPENDS ${ABS_FIL} ${TEMPLATES}
                 ${CMAKE_CURRENT_SOURCE_DIR}/api/service_compiler/plugin.py
         COMMENT "Running C++ service compiler on ${FIL}"
@@ -121,8 +134,6 @@ function(Service_Compiler_Generate_Cpp SRCS HDRS)
 
   endforeach()
 
-  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
-
   # remove some warnings from protobuf generated files
   if(WIN32)
     set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS

dev/README.md

@@ -0,0 +1,76 @@
+[cmake ext]: https://marketplace.visualstudio.com/items?itemName=ms-vscode.cmake-tools
+[cmake presets]: https://cmake.org/cmake/help/latest/manual/cmake-presets.7.html
+[cpp ext]: https://marketplace.visualstudio.com/items?itemName=ms-vscode.cpptools
+[clangd ext]: https://marketplace.visualstudio.com/items?itemName=llvm-vs-code-extensions.vscode-clangd
+[py venv]: https://docs.python.org/3/library/venv.html
+
+# C++ Intellisense for Spark on Windows (VS Code)
+**Steps**
+1. Install the **[C++ extension][cpp ext]**
+2. Install the **[CMake extension][cmake ext]**
+3. Copy `c_cpp_properties.json` from this folder to `$workspaceFolder/.vscode/`
+4. In the VS Code Command Palette, run **CMake: Select Configure Preset**, then
+   select '**spark**' from the dropdown
+
+**Notes**
+* Intellisense won't work for generated files/types until you build libspark
+
+# C++ Intellisense for Sparkd on Linux (VS Code)
+1. Install the **[CMake extension][cmake ext]**
+2. Install the **[Clangd extension][clangd ext]**
+3. (Optional) Install the **[C++ extension][cpp ext]** (*needed for debugging,
+   but intellisense should work without it*).
+4. If the **[C++ extension][cpp ext]** is installed, disable its intellisense in
+  your workspace settings (*Clangd intellisense is better*)
+   ```
+   "C_Cpp.intelliSenseEngine": "disabled"
+   ```
+5. In the VS Code Command Palette, run **CMake: Select Configure Preset**, then select the
+  **[preset][cmake presets]** you wish to build from the dropdown
+   * *The **Clang** presets may work best with the **Clangd** extension*
+   * *Intellisense won't work for generated files/types until you build libspark*
+
+# Build Sparkd on Linux
+
+**Notes**
+* Sparkd is built using **[CMakePresets][cmake presets]**
+
+**Steps**
+1. Set up the **[Python virtual environment][py venv]**<br>
+   * *Run commands from the repo root dir unless otherwise noted*
+   1. Create the virtual environment
+      ```
+      python3 -m venv .venv
+      ```
+   2. Activate the virtual environment
+      ```
+      . .venv/bin/activate
+      ```
+   3. Verify `python` in the venv is first in the path<br>
+      * ***Expected:** output shows the absolute path to `.venv/bin/python`*
+      ```
+      which python
+      ```
+   4. Update `pip`<br>
+      * *The venv module initializes virtual environments with its own version
+        of pip, rather than the system's version. Even if the latter is up-to-date, the
+        former may not be.*
+      ```
+      python -m pip install --upgrade pip
+      ```
+   5. Install the packages listed `requirements.txt`
+      ```
+      pip install -r requirements.txt
+      ```
+2. Build **Sparkd**<br>
+   * From the terminal
+      ```
+      tools/build_sparkd.sh
+      ```
+      * *Run with `--help` to see available options*
+   * From VS Code
+      1. Install the **[CMake extension][cmake ext]**
+      2. In the VS Code Command Palette, run **CMake: Select Configure
+         Preset**, then select the **[preset][cmake presets]** you wish to
+         build from the dropdown
+      3. Run **CMake: Build** in the command palette.

dev/c_cpp_properties.json

@@ -0,0 +1,27 @@
+{
+  "version": 4,
+  "configurations": [
+    {
+      "name": "Windows",
+      "intelliSenseMode": "windows-msvc-x64",
+      "includePath": [
+        "${workspaceFolder}/src/",
+        "${workspaceFolder}/build/src/",
+        "${workspaceFolder}/externals/windows/include/"
+      ],
+      "defines": ["_WIN32", "__x86_64__"],
+      "browse": {
+        "path": [
+          "${workspaceRoot}/src",
+          "${workspaceRoot}/build/src"
+        ],
+        "limitSymbolsToIncludedHeaders": true
+      },
+      "cStandard": "c17",
+      "cppStandard": "c++17",
+      "configurationProvider": "ms-vscode.cmake-tools",
+      "compilerPath": "C:/Program Files/Microsoft Visual Studio/2022/Community/VC/Tools/MSVC/14.34.31933/bin/Hostx64/x64/cl.exe",
+      "mergeConfigurations": true
+    }
+  ]
+}

doc/user/CMakeDeps.cmake

@@ -1 +0,0 @@
-set(sources /home/taha/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)

docker/sparkd/dev/Dockerfile

@@ -0,0 +1,103 @@
+FROM ubuntu:22.10
+
+RUN \
+    apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    clang-13 \
+    clang-format-13 \
+    clangd-13 \
+    clang-15 \
+    clang-format-15 \
+    clangd-15 \
+    cmake \
+    curl \
+    dotnet7 \
+    fish \
+    g++-11 \
+    g++-12 \
+    gnupg \
+    liblttng-ust-dev \
+    libssh2-1-dev \
+    libpcap0.8-dev \
+    locales \
+    ninja-build \
+    nodejs \
+    npm \
+    openssh-server \
+    python3-pip \
+    python3-venv \
+    rapidjson-dev \
+    software-properties-common \
+    sqlite3 \
+    sudo \
+    tar \
+    vim \
+    wget
+
+# this is done here so that ccache sees clang++ installed, not clang-10
+RUN \
+    apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y ccache 
+
+# emumba deps
+RUN \
+    apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    tzdata \
+    build-essential \
+    cmake-curses-gui \
+    git-all \
+    doxygen \
+    python3-sphinx \
+    graphviz \
+    libgtest-dev \
+    libssl-dev \
+    libboost-filesystem-dev \
+    libboost-program-options-dev \
+    libboost-test-dev \
+    libboost-date-time-dev \
+    zlib1g-dev \
+    netcat-traditional \
+    net-tools \
+    gdb gdbserver \
+    pkg-config
+
+# set up default executables (update-alternatives)
+RUN \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-13 100; \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 100; \
+    update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100; \
+    update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-13 100; \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100; \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
+
+# Add compiler symlinks for ccache 
+RUN \
+    ln -s /usr/bin/ccache /usr/local/bin/clang; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-13; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/clang++-15; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-11; \
+    ln -s /usr/bin/ccache /usr/local/bin/gcc-12; \
+    ln -s /usr/bin/ccache /usr/local/bin/g++-12
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+    groupadd -g "$docker_uid" "$docker_user" \
+    && useradd \
+    -m \
+    -u "$docker_uid" \
+    -g "$docker_uid" \
+    -G sudo \
+    --password "$(openssl passwd -1 "$docker_pass")" \
+    "$docker_user"

docker/sparkd/dev/README.md

@@ -0,0 +1,64 @@
+[dev containers]: https://code.visualstudio.com/docs/devcontainers/containers
+[dev containers ext]: https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers
+[docker repo sparkd-dev]: https://hub.docker.com/repository/docker/bluetradingsystems/sparkd-dev/general
+# Sparkd Dev Container
+## How to create a VS Code [Dev Container][dev containers]
+
+1. Make sure the **[Dev Containers][dev containers ext]** extension is installed
+1. Make sure your Docker image is up to date
+   ```
+   docker pull bluetradingsystems/sparkd-dev:latest
+   ```
+1. Copy **devcontainer.json** from this folder to **$workspaceFolder/.devcontainer/**.
+
+1. In the VS Code Command Palette, run **Dev Containers: Reopen in Container**
+
+---
+## How to create a Sparkd Container in the terminal (without VS Code)
+
+1. Make sure your Docker image is up to date
+   ```
+   docker pull bluetradingsystems/sparkd-dev:latest
+   ```
+2. Start the container
+   ```
+   # Replace <cont_name> with a name for the container. The container's
+   # "$HOME" dir will be mounted to "$HOME/.sparkd-dev/mounts/<cont_name> on
+   # the host
+   
+   ./start_docker.sh --container <cont_name> --mount-existing
+   ```
+   * *Replace <cont_name> with a name for the container*
+   * *The container's `$HOME` dir will be mounted to
+   `$HOME/.sparkd-dev/mounts<cont_name>` on the host*<br>
+   * *Run: `./start_docker.sh --help` to see additional options*
+3. *(Optional) Attach VS Code to the running container.*<br>
+Run **Dev Containers: Attach to Running Container** from the command palette,
+then select your container in the dropdown
+
+---
+## How to update the Docker image
+
+1. Change the image by modifying the Dockerfile (don't tag arbitrary
+  images on your machine and push them to our repo on Docker Hub)
+
+1. After modifying the Dockerfile, (re)build the image
+   ```
+   # Run in this dir
+   docker build -t bluetradingsystems/sparkd-dev:latest .
+   ```
+   ***Note:** Your docker login must have write permissions to push to the
+   [sparkd-dev][docker repo sparkd-dev] repository*
+
+2. Commit the changes to Dockerfile in git
+
+3. Push the new image to Docker Hub
+
+   ```
+   docker push johncolvin/bluetradingsystems:sparkd-dev
+   ```
+   ***Note:** The image doesn't have to be on Docker Hub, we can simply build
+   it from scratch on our local machines (with the Dockerfile). However, this
+   takes time. It is significantly quicker to download a prebuilt image, so we
+   store it on a remote repository to save time when setting up a new dev
+   machine.*

docker/sparkd/dev/bashrc

@@ -0,0 +1,119 @@
+# ~/.bashrc: executed by bash(1) for non-login shells.
+# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc)
+# for examples
+
+# If not running interactively, don't do anything
+case $- in
+    *i*) ;;
+      *) return;;
+esac
+
+# don't put duplicate lines or lines starting with space in the history.
+# See bash(1) for more options
+HISTCONTROL=ignoreboth
+
+# append to the history file, don't overwrite it
+shopt -s histappend
+
+# for setting history length see HISTSIZE and HISTFILESIZE in bash(1)
+HISTSIZE=1000
+HISTFILESIZE=2000
+
+# check the window size after each command and, if necessary,
+# update the values of LINES and COLUMNS.
+shopt -s checkwinsize
+
+# If set, the pattern "**" used in a pathname expansion context will
+# match all files and zero or more directories and subdirectories.
+#shopt -s globstar
+
+# make less more friendly for non-text input files, see lesspipe(1)
+[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)"
+
+# set variable identifying the chroot you work in (used in the prompt below)
+if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then
+    debian_chroot=$(cat /etc/debian_chroot)
+fi
+
+# set a fancy prompt (non-color, unless we know we "want" color)
+case "$TERM" in
+    xterm-color|*-256color) color_prompt=yes;;
+esac
+
+# uncomment for a colored prompt, if the terminal has the capability; turned
+# off by default to not distract the user: the focus in a terminal window
+# should be on the output of commands, not on the prompt
+#force_color_prompt=yes
+
+if [ -n "$force_color_prompt" ]; then
+    if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then
+	# We have color support; assume it's compliant with Ecma-48
+	# (ISO/IEC-6429). (Lack of such support is extremely rare, and such
+	# a case would tend to support setf rather than setaf.)
+	color_prompt=yes
+    else
+	color_prompt=
+    fi
+fi
+
+if [ "$color_prompt" = yes ]; then
+    PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
+else
+    PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ '
+fi
+unset color_prompt force_color_prompt
+
+# If this is an xterm set the title to user@host:dir
+case "$TERM" in
+xterm*|rxvt*)
+    PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
+    ;;
+*)
+    ;;
+esac
+
+# enable color support of ls and also add handy aliases
+if [ -x /usr/bin/dircolors ]; then
+    test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
+    alias ls='ls --color=auto'
+    #alias dir='dir --color=auto'
+    #alias vdir='vdir --color=auto'
+
+    alias grep='grep --color=auto'
+    alias fgrep='fgrep --color=auto'
+    alias egrep='egrep --color=auto'
+fi
+
+# colored GCC warnings and errors
+#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01'
+
+# some more ls aliases
+alias ll='ls -alF'
+alias la='ls -A'
+alias l='ls -CF'
+
+# Add an "alert" alias for long running commands.  Use like so:
+#   sleep 10; alert
+alias alert='notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"'
+
+# Alias definitions.
+# You may want to put all your additions into a separate file like
+# ~/.bash_aliases, instead of adding them here directly.
+# See /usr/share/doc/bash-doc/examples in the bash-doc package.
+
+if [ -f ~/.bash_aliases ]; then
+    . ~/.bash_aliases
+fi
+
+# enable programmable completion features (you don't need to enable
+# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
+# sources /etc/bash.bashrc).
+if ! shopt -oq posix; then
+  if [ -f /usr/share/bash-completion/bash_completion ]; then
+    . /usr/share/bash-completion/bash_completion
+  elif [ -f /etc/bash_completion ]; then
+    . /etc/bash_completion
+  fi
+fi
+
+export PATH="$HOME/.local/bin:$PATH"

docker/sparkd/dev/config.fish

@@ -0,0 +1,22 @@
+set fish_config_home $XDG_CONFIG_HOME/fish
+if test -f "$fish_config_home/aliases.fish"
+    source "$fish_config_home/aliases.fish"
+end
+# enable general colors and case insensitive search in 'less'
+set -x LESS "$LESS -R -Q -I"
+set -x LC_ALL en_US.UTF-8
+# enable syntax highlighting in 'less'
+# set highlight color scheme in 'less' (fg)(bg)
+set -g LESS_TERMCAP_md (tput bold)(tput setaf 36)
+set -g LESS_TERMCAP_me (tput sgr0)
+set -g LESS_TERMCAP_so (tput setaf 232)(tput setab 49)
+
+set -q TERM
+if test $status != 0; or test $TERM = 'xterm'
+    set -g TERM 'xterm-256color'
+end
+
+# Fix impossible-to-see navy blue command text color
+set fish_color_command F8F8F2
+
+set -g PATH "$HOME/.local/bin" "$PATH"

docker/sparkd/dev/devcontainer.json

@@ -0,0 +1,26 @@
+{
+  "image": "bluetradingsystems/sparkd-dev:latest",
+  "initializeCommand": "${localWorkspaceFolder}/docker/sparkd/dev/devcontainer_initialize.sh",
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
+  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
+  "updateRemoteUserUID": true,
+  "shutdownAction": "none",
+  // Note: If running multiple containers on the same host, each must use a
+  // unique set of ports. Usually we just add 1000 to each port as needed. For
+  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
+  "runArgs": [
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003"
+  ],
+  "mounts": [
+    "source=${localEnv:HOME}/.sparkd-dev/mounts/sparkd-dev,target=/home/bts,type=bind"
+  ]
+}

docker/sparkd/dev/devcontainer_initialize.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+mkdir -p "$HOME/.sparkd-dev/mounts/sparkd-dev"

docker/sparkd/dev/devcontainer_post_create.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+repo_dir="$(cd "$this_dir" && git rev-parse --show-toplevel)"
+tools_dir="$repo_dir/tools"
+gen_tag="GENERATED BY $(basename "${BASH_SOURCE[0]}")"
+
+function copy_if_not_exist() {
+  src="$1"
+  dest="$2"
+  if [[ ! -e "$src" ]]; then
+    echo "Unable to copy $src to $dest. Source file doesn't exist"
+    exit 1
+  fi
+  if [[ ! -e "$dest" ]]; then
+    cp "$src" "$dest"
+  fi
+}
+
+# Init bashrc
+echo "Initializing bashrc"
+copy_if_not_exist "$this_dir/bashrc" "$HOME/.bashrc"
+
+# Init fish config
+echo "Initializing fish config"
+fish_cfg_dir="$HOME/.config/fish/"
+mkdir -p "$fish_cfg_dir"
+exec "$tools_dir/write_tag_block.sh" \
+  --tag "$gen_tag" \
+  --replace \
+  --in-place \
+  --file "$fish_cfg_dir/config.fish" \
+  --text "$(cat "$this_dir/config.fish")"

docker/sparkd/dev/set_user.sh

@@ -0,0 +1,250 @@
+#!/usr/bin/env bash
+
+function _exit() {
+  if [[ "$#" -gt 1 ]]; then
+    # When there are multiple args, the last arg is the exit code.
+    echo "${@:1:($# - 1)}"
+    exit "${@: -1}"
+  fi
+  echo "$1"
+  exit 0
+}
+
+help_text="\
+Add or update a user's uid
+
+This script is intended to be run in a Docker container as a means to obtain
+file permissions in mounted directories owned by the specified UID.
+
+Usage: set_user.sh [OPTIONS] USER UID
+
+Positionals:
+  USER               The name of the user
+
+Options:
+  --user USER        The name of the user (defaults to 'bts')
+  --uid UID          The new uid to give USER (defaults to '1000')
+  --group GROUP      The primary group to give USER (defaults to USER)
+  --gid GID          The primary gid to give USER (defaults to UID)
+  --pass PASSWORD    The password to give USER (defaults to USER, only applies
+                     if USER hasn't been created yet) 
+  --modify-existing  Update UID/GID even if USER already exists
+  -h,--help          Print this help message and exit
+
+"
+
+function _add_group() {
+  if [[ "$#" -ne 2 ]]; then
+    _exit "'_add_group' requires exactly 2 args (group, gid). Received $#." 1
+  fi
+  local group="$1"
+  local gid="$2"
+  # Check if gid already exists
+  local existing_group=""
+  existing_group="$(getent group "$gid" | cut -d : -f 1)"
+  if [[ -n "$existing_group" ]]; then
+    if [[ "$existing_group" != "$group" ]]; then
+      echo "\
+The gid '$gid' is already in use by group '$existing_group' (will not rename \
+the group)"
+    fi
+    return
+  fi
+
+  # Check if group with same name already exists
+  local existing_gid=""
+  existing_gid="$(getent group "$group" | cut -d : -f 3)"
+  if [[ -n "$existing_gid" ]]; then
+    # The group already exists. Only modify if --modify-existing was specified.
+    if [[ "$modify_existing" -eq 0 ]]; then
+      _exit "\
+The group '$group' already exists with gid '$existing_gid'.
+Specify --modify-existing to change the gid anyways." 1
+    fi
+    echo "Changing gid for group '$group' from '$existing_gid' to '$gid'"
+    groupmod -g "$gid" "$group"
+  else
+    # The group doesn't exist. Add it
+    echo "Adding group '$group:$gid'"
+    groupadd -g "$gid" "$group"
+  fi
+}
+
+function _add_user() {
+  local user="$1"
+  local uid="$2"
+  local group="$3"
+  local gid="$4"
+  local pass="$5"
+  if [[ "$#" -ne 5 ]]; then
+    _exit "'_add_user' requires exactly 5 args (user, uid, group, gid, pass). \
+Received $#." 1
+  fi
+
+  # Check if uid is being used by another user
+  local existing_user=""
+  existing_user="$(getent passwd "$uid" | cut -d : -f 1)"
+  if [[ -n "$existing_user" ]]; then
+    if [[ "$existing_user" != "$user" ]]; then
+      _exit "The uid '$uid' is already in use by user '$existing_user'." 1
+    fi
+
+    echo "The user '$user' already has the specified uid '$uid'."
+    # Modify gid if necessary
+    local existing_gid=""
+    existing_gid="$(id -g "$user" 2> /dev/null)"
+    if [[ "$existing_gid" = "$gid" ]]; then
+      echo "The user '$user' already has the specified gid '$gid'."
+      return
+    fi
+
+    if [[ "$modify_existing" -eq 0 ]]; then
+      _exit "\
+The user '$user' is associated with the primary group '$existing_gid'.
+Specify --modify-existing to change the gid anyways." 1
+    fi
+
+    echo "Changing primary group for '$user' from '$existing_gid' to '$gid'."
+    usermod -g "$gid" "$user"
+    return
+  fi
+
+  # Check if user with same name already exists
+  local existing_uid=""
+  existing_uid="$(id -u "$user" 2> /dev/null)"
+  if [[ -n "$existing_uid" ]]; then
+    if [[ "$existing_uid" = "$uid" ]]; then
+      _exit "The user '$user' already has the specified uid '$uid'" 0
+    fi
+
+    if [[ "$modify_existing" -eq 0 ]]; then
+      _exit "\
+User '$user' already exists with uid '$existing_uid'.
+Specify --modify-existing to change the uid anyways." 1
+    fi
+
+    echo "Changing uid for user '$user' from '$existing_uid' to '$uid'"
+    if ! usermod -u "$uid" -g "$gid" "$user"; then
+      _exit "usermod exited with code $?" 1
+    fi
+    return
+  fi
+
+  # The user doesn't exist. Add it
+  echo "Adding user '$user:$uid' (group='$group:$gid', password='$pass')"
+  useradd -m -u "$uid" -g "$gid" -G sudo --password "$(openssl passwd -1 "$pass")" "$user"
+}
+
+function _update_fish_dir_perms() {
+  local user="$1"
+  local uid="$2"
+  local gid="$3"
+  # When a fish shell session starts, a temp dir named '/tmp/fish.<user>' is
+  # created with the user's current uid/gid. If such a dir exists for the
+  # specified USER, make sure the uid/gid match the new values.
+  fish_dir="/tmp/fish.$user"
+  if [[ -d "$fish_dir" ]]; then
+    perms="$uid:$gid"
+    existing_perms="$(stat -c %u:%g "$fish_dir")"
+    if [[ "$existing_perms" != "$perms" ]]; then
+      echo "Changing uid:gid for dir '$fish_dir' from '$existing_perms' to '$perms'"
+      chown -R "$perms" "$fish_dir"
+    fi
+  fi
+}
+
+user=""
+group=""
+pass=""
+declare -i uid=0
+declare -i gid=0
+declare -i modify_existing=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --user)
+      if [[ -z "$2" ]]; then
+        _exit "USER (--user) must be non-empty" 1
+      elif [[ "$2" = "root" ]]; then
+        _exit "USER (--user) cannot be 'root'" 1
+      fi
+      user="$2"
+      shift 2
+      ;;
+    --group)
+      if [[ -z "$2" ]]; then
+        _exit "GROUP (--group) must be non-empty" 1
+      elif [[ "$2" = "root" ]]; then
+        _exit "GROUP (--group) cannot be 'root'" 1
+      fi
+      group="$2"
+      shift 2
+      ;;
+    --uid)
+      if [[ -z "$2" ]]; then
+        _exit "UID (--uid) must be non-empty" 1
+      fi
+      uid="$2"
+      shift 2
+      ;;
+    --gid)
+      if [[ -z "$2" ]]; then
+        _exit "GID (--gid) must be non-empty" 1
+      fi
+      gid="$2"
+      shift 2
+      ;;
+    --pass)
+      if [[ -z "$2" ]]; then
+        _exit "PASS (--pass) must be non-empty" 1
+      fi
+      pass="$2"
+      shift 2
+      ;;
+    --modify-existing)
+      modify_existing=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      _exit "$help_text" 0
+      ;;
+    *)
+      if [[ -z "$1" ]]; then
+        _exit "Invalid positional argument '$1'" 1
+      elif [[ -z "$user" ]]; then
+        user="$1"
+      elif [[ -z "$uid" ]]; then
+        uid="$1"
+      else
+        _exit "Unexpected argument '$1'" 1
+      fi
+      shift 1
+      ;;
+  esac
+done
+
+if [[ -z "$user" ]]; then
+  user="bts"
+fi
+
+if [[ $uid -eq 0 ]]; then
+  uid=1000
+fi
+
+# In general, a user's primary group/gid matches the user's name/uid.
+# If group/gid args weren't specified, set them to name/uid.
+if [[ -z "$group" ]]; then
+  group="$user"
+fi
+if [[ $gid -eq 0 ]]; then
+  gid=$uid
+fi
+if [[ -z "$pass" ]]; then
+  pass="$user"
+fi
+
+_add_group "$group" "$gid"
+_add_user "$user" "$uid" "$group" "$gid" "$pass"
+_update_fish_dir_perms "$user" "$uid" "$gid"

docker/sparkd/dev/start_docker.sh

@@ -0,0 +1,230 @@
+#!/usr/bin/env bash
+
+function _exit() {
+  if [[ "$#" -gt 1 ]]; then
+    # When there are multiple args, the last arg is the exit code.
+    echo "${@:1:($# - 1)}"
+    exit "${@: -1}"
+  fi
+  echo "$1"
+  exit 0
+}
+
+help_text="\
+Start a docker dev container
+Usage: start_docker.sh [OPTIONS] image container
+
+Positionals:
+  image             The image to instantiate
+  container         The name for the container
+
+Options:
+  -u,--user USER      The user to add in the container (defaults to 'bts')
+  -p,--pass PASS      The password for USER in the container (defaults to USER)
+  -i,--image IMAGE    The image to instantiate (defaults to 'bluetradingsystems/sparkd-dev:latest')
+  -c,--container CONT The name for the container
+  -m,--mount MOUNT    The name for the folder (outside the container) to create/mount to the home
+                      directory in the container. Mount folders are added here: $mount_prefix
+  --mount-existing    Mount the specified dir even if it already exists
+  --port <cont_port>:<host_port> OR [<cont_port>:<host_port>,...] Port mapping(s). May be
+                      specified multiple times.
+  -h,--help           Print this help message and exit
+
+"
+
+mount_prefix="$HOME/.sparkd-dev/mounts"
+declare docker_user=""
+declare docker_pass=""
+declare image_name=""
+declare cont_name=""
+declare mount_name=""
+declare -i mount_existing=0
+declare -a port_mappings=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -u)
+      ;&
+    --user)
+      if [[ -z "$2" ]]; then
+        _exit "User (-u,--user) must be non-empty" 1
+      fi
+      docker_user="$2"
+      shift 2
+      ;;
+    -p)
+      ;&
+    --pass)
+      if [[ -z "$2" ]]; then
+        _exit "Password (-p,--pass) must be non-empty" 1
+      fi
+      docker_pass="$2"
+      shift 2
+      ;;
+    -i)
+      ;&
+    --image)
+      if [[ -n "$image_name" ]]; then
+        _exit "Unexpected argument '$1'" 1
+      elif [[ -z "$2" ]]; then
+        _exit "Image (-i,--image) must be non-empty" 1
+      fi
+      image_name="$2"
+      shift 2
+      ;;
+    -c)
+      ;&
+    --container)
+      if [[ -n "$cont_name" ]]; then
+        _exit "Unexpected argument '$1'" 1
+      elif [[ -z "$2" ]]; then
+        _exit "Container (-c,--container) must be non-empty" 1
+      fi
+      cont_name="$2"
+      shift 2
+      ;;
+    --port)
+      # Parse port(s) arg, and append the result to 'port_mappings'
+      # Note: port(s) may be specified in bracketed notation:
+      #   [a:b,c:d,...,y:z]
+      next_ports="$(echo "$2" | sed -e 's/\[//g' -e 's/\]//g' -e 's/,/ /g')"
+      if [[ "${#port_mappings[@]}" -gt 0 ]]; then
+        next_ports=" $next_ports"
+      fi
+      port_mappings+=("$next_ports")
+      shift 2
+      ;;
+    -m)
+      ;&
+    --mount)
+      if [[ -z "$2" ]]; then
+        echo "Mount (-m,--mount) must be non-empty"
+      fi
+      mount_name="$2"
+      shift 2
+      ;;
+    --mount-existing)
+      mount_existing=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      _exit "$help_text" 0
+      ;;
+    *)
+      if [[ -z "$1" ]]; then
+        _exit "Invalid positional argument '$1'" "$help_text" 1
+      elif [[ -z "$image_name" ]]; then
+        image_name="$1"
+      elif [[ -z "$cont_name" ]]; then
+        cont_name="$1"
+      else
+        _exit "Unexpected argument '$1'" "$help_text" 1
+      fi
+      shift 1
+      ;;
+  esac
+done
+
+
+if [[ -z "$image_name" ]]; then
+  image_name="bluetradingsystems/sparkd-dev:latest"
+fi
+
+host_name="$(hostname)"
+if [[ -n "$cont_name" ]]; then
+  host_name="$host_name.$cont_name"
+fi
+
+if [[ -z "$mount_name" ]]; then
+  mount_name="$cont_name"
+fi
+
+if [[ -z "$mount_name" && -z "$cont_name" ]]; then
+  _exit "Please specify mount (-m,--mount) and/or container (-c,--container)" 1
+fi
+
+if [[ -z "$docker_user" ]]; then
+  docker_user="bts"
+fi
+
+if [[ "$docker_user" = "root" ]]; then
+  _exit "USER (--user) cannot be 'root'" 1
+fi
+
+cont_home="/home/$docker_user"
+home_src="$mount_prefix/$mount_name"
+home_tgt="$cont_home"
+this_dir="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+repo_src="$(cd "$this_dir" && git rev-parse --show-toplevel)"
+repo_tgt="$cont_home/spark"
+working_dir="$cont_home"
+
+if [[ $mount_existing -eq 0 && -e "$home_src" ]]; then
+  # Don't overwrite existing dir (might be mounted in another container)
+  _exit "Mount dir '$home_src' already exists (use --mount-existing to mount \
+an existing directory)" 1
+fi
+
+mkdir -p "$home_src"
+
+if [[ "${#port_mappings[@]}" -eq 0 ]]; then
+  port_mappings=("8000:8000" "8001:8001" "8002:8002" "8003:8003")
+fi
+
+# Adapt port_mappings into args for `docker run` by prepending "-p " to each
+# item
+declare -a docker_port_args=()
+for port_mapping in "${port_mappings[@]}"; do
+  docker_port_args+=("-p $port_mapping")
+done
+
+echo "Starting container '$cont_name'
+image_name  : $image_name
+cont_name   : $cont_name
+home_src    : $home_src
+home_tgt    : $home_tgt
+repo_src    : $repo_src
+repo_tgt    : $repo_tgt
+working dir : $working_dir
+ports       : ${port_mappings[*]}
+"
+
+# DON'T wrap $docker_port_args in quotes. We want bash to split args by
+# whitespace
+docker run \
+  --net bridge \
+  -h "$host_name" \
+  -e "TZ=America/New_York" \
+  --cap-add=SYS_PTRACE \
+  --security-opt seccomp=unconfined \
+  --name "$cont_name" \
+  -v "$home_src:$home_tgt" \
+  -v "$repo_src:$repo_tgt" \
+  -w "$working_dir" \
+  ${docker_port_args[*]} \
+  --shm-size=4024m \
+  -t \
+  -i \
+  -d \
+  -e DISPLAY="$DISPLAY" \
+  "$image_name"
+
+# Add user w/ same uid and gid as the current user outside of docker, so the
+# user inside the docker has file permissions in the mounted dir.
+docker_group="$docker_user"
+docker_uid="$(id -u "$USER")"
+docker_gid="$(id -g "$USER")"
+if [[ -z "$docker_pass" ]]; then
+  docker_pass="$docker_user"
+fi
+docker exec -u root -it "$cont_name" "$repo_tgt/docker/sparkd/dev/set_user.sh" \
+--user "$docker_user" \
+--uid "$docker_uid" \
+--group "$docker_group" \
+--gid "$docker_gid" \
+--pass "$docker_pass"
+
+# Run same script VS Code runs to initialize the container
+docker exec -u "$docker_user" -it "$cont_name" "$repo_tgt/docker/sparkd/dev/devcontainer_post_create.sh"

docker/sparkd/docker_shell.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+function print_help_and_exit() {
+  cat <<EOF
+Open a shell inside the docker container
+Usage: docker_shell.sh [OPTIONS] [CONTAINER]
+
+Options:
+  -u,--user         The user to login with in the container (defaults to the current user)
+  -c,--container    The container name (may also be specified as positional arg)
+  -h,--help         Print this help message and exit
+
+EOF
+  exit "$1"
+}
+
+
+declare docker_user=""
+declare cont_name=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -u)
+      ;&
+    --user)
+      if [[ -z "$2" ]]; then
+        echo "User (-u,--user) must be non-empty"
+        print_help_and_exit 1
+      fi
+      docker_user="$2"
+      shift 2
+      ;;
+    -h)
+      ;&
+    --help)
+      print_help_and_exit 0
+      ;;
+    -c)
+      ;&
+    --container)
+      shift 1
+      ;&
+    *)
+      if [[ ! -z "$cont_name" ]]; then
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      fi
+      cont_name="$1"
+      shift 1
+      ;;
+  esac
+done
+
+
+if [[ -z "$cont_name" ]]; then
+  echo "Please specify the container ([-c,--container])"
+  print_help_and_exit 1
+fi
+
+if [[ -z "$docker_user" ]]; then
+  docker_user="$USER"
+fi
+
+docker exec -u "$docker_user" -it "$cont_name" bash

docker/sparkd/start_docker.sh

@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+
+mount_prefix="$HOME/.spark-dev/mounts"
+
+function print_help_and_exit() {
+  cat <<EOF
+Start a docker dev container
+Usage: start_docker.sh [OPTIONS] image container
+
+Positionals:
+  image             The image to instantiate
+  container         The name for the container
+
+Options:
+  -u,--user USER      The user to add in the container (defaults to the current user)
+  -p,--pass PASS      The password for USER in the container (defaults to USER)
+  -i,--image IMAGE    The image to instantiate
+  -c,--container CONT The name for the container
+  -m,--mount MOUNT    The name for the folder (outside the container) to create/mount to the home
+                      directory in the container. Mount folders are added here: $mount_prefix
+  --mount-existing    Mount the specified dir even if it already exists
+  --port <cont_port>:<host_port> OR [<cont_port>:<host_port>,...] Port mapping(s). May be
+                      specified multiple times.
+  -h,--help           Print this help message and exit
+
+EOF
+  exit "$1"
+}
+
+
+declare docker_user=""
+declare docker_pass=""
+declare image_name=""
+declare cont_name=""
+declare mount_name=""
+declare -i mount_existing=0
+declare -a port_mappings=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -u)
+      ;&
+    --user)
+      if [[ -z "$2" ]]; then
+        echo "User (-u,--user) must be non-empty"
+        print_help_and_exit 1
+      fi
+      docker_user="$2"
+      shift 2
+      ;;
+    -p)
+      ;&
+    --pass)
+      if [[ -z "$2" ]]; then
+        echo "Password (-p,--pass) must be non-empty"
+        print_help_and_exit 1
+      fi
+      docker_pass="$2"
+      shift 2
+      ;;
+    -i)
+      ;&
+    --image)
+      if [[ -n "$image_name" ]]; then
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      elif [[ -z "$2" ]]; then
+        echo "Image (-i,--image) must be non-empty"
+        print_help_and_exit 1
+      fi
+      image_name="$2"
+      shift 2
+      ;;
+    -c)
+      ;&
+    --container)
+      if [[ -n "$cont_name" ]]; then
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      elif [[ -z "$2" ]]; then
+        echo "Container (-c,--container) must be non-empty"
+        print_help_and_exit 1
+      fi
+      cont_name="$2"
+      shift 2
+      ;;
+    --port)
+      # Parse port(s) arg, and append the result to 'port_mappings'
+      # Note: port(s) may be specified in bracketed notation:
+      #   [a:b,c:d,...,y:z]
+      next_ports="$(echo "$2" | sed -e 's/\[//g' -e 's/\]//g' -e 's/,/ /g')"
+      if [[ "${#port_mappings[@]}" -gt 0 ]]; then
+        next_ports=" $next_ports"
+      fi
+      port_mappings+=("$next_ports")
+      shift 2
+      ;;
+    -m)
+      ;&
+    --mount)
+      if [[ -z "$2" ]]; then
+        echo "Mount (-m,--mount) must be non-empty"
+      fi
+      mount_name="$2"
+      shift 2
+      ;;
+    --mount-existing)
+      mount_existing=1
+      shift 1
+      ;;
+    -h)
+      ;&
+    --help)
+      print_help_and_exit 0
+      ;;
+    *)
+      if [[ -z "$1" ]]; then
+        echo "Invalid positional argument '$1'"
+        print_help_and_exit 1
+      elif [[ -z "$image_name" ]]; then
+        image_name="$1"
+      elif [[ -z "$cont_name" ]]; then
+        cont_name="$1"
+      else
+        echo "Unexpected argument '$1'"
+        print_help_and_exit 1
+      fi
+      shift 1
+      ;;
+  esac
+done
+
+
+if [[ -z "$image_name" ]]; then
+  image_name="bluetradingsystems/spark-crypto-dev"
+fi
+
+host_name="$(hostname)"
+if [[ -n "$cont_name" ]]; then
+  host_name="$host_name.$cont_name"
+fi
+
+if [[ -z "$mount_name" ]]; then
+  mount_name="$cont_name"
+fi
+
+if [[ -z "$mount_name" && -z "$cont_name" ]]; then
+  echo "Please specify mount (-m,--mount) and/or container (-c,--container)"
+  print_help_and_exit 1
+fi
+
+if [[ -z "$docker_user" ]]; then
+  docker_user="$USER"
+fi
+
+home_src=$mount_prefix/$mount_name
+home_tgt=$HOME
+repo_src=$(git rev-parse --show-superproject-working-tree)
+repo_tgt=$HOME/$(basename "$repo_src")
+working_dir=$HOME
+
+if [[ $mount_existing -eq 0 && -e "$home_src" ]]; then
+  # Don't overwrite existing dir (might be mounted in another container)
+  echo "Mount dir '$home_src' already exists (use --mount-existing to mount an existing directory)"
+  print_help_and_exit 1
+fi
+
+mkdir -p "$home_src"
+
+if [[ "${#port_mappings[@]}" -eq 0 ]]; then
+  port_mappings=("8000:8000" "8001:8001" "8002:8002" "8003:8003")
+fi
+
+# Adapt port_mappings into args for `docker run` by prepending "-p " to each
+# item
+declare -a docker_port_args=()
+for port_mapping in "${port_mappings[@]}"; do
+  docker_port_args+=("-p $port_mapping")
+done
+
+echo "Starting container '$cont_name'
+image_name  : $image_name
+cont_name   : $cont_name
+home_src    : $home_src
+home_tgt    : $home_tgt
+repo_src    : $repo_src
+repo_tgt    : $repo_tgt
+working dir : $working_dir
+ports       : ${port_mappings[*]}
+"
+
+# DON'T wrap $docker_port_args in quotes. We want bash to split args by
+# whitespace
+docker run \
+  --net bridge \
+  -h "$host_name" \
+  -e "TZ=America/New_York" \
+  --cap-add=SYS_PTRACE \
+  --security-opt seccomp=unconfined \
+  --name "$cont_name" \
+  -v "$home_src:$home_tgt" \
+  -v "$repo_src:$repo_tgt" \
+  -w "$working_dir" \
+  ${docker_port_args[*]} \
+  --shm-size=4024m \
+  -t \
+  -i \
+  -d \
+  -e DISPLAY="$DISPLAY" \
+  "$image_name" \
+  bash
+
+# Add user w/ same uid and gid as the current user outside of docker, so the
+# user inside the docker has file permissions in the mounted dir.
+docker_group="$docker_user"
+docker_user_uid="$(id -u "$USER")"
+docker_user_gid="$(id -g "$USER")"
+
+echo "Adding user in container '$cont_name'
+name : $docker_user
+uid  : $docker_user_uid
+gid  : $docker_user_gid"
+
+if [[ -z "$docker_pass" ]]; then
+  docker_pass="$docker_user"
+  echo "Giving user '$docker_user' a secure password: $docker_pass"
+fi
+
+docker exec -u root -it "$cont_name" groupadd -g "$docker_user_gid" "$docker_group"
+docker exec -u root -it "$cont_name" useradd -u "$docker_user_uid" -g "$docker_user_gid" "$docker_user"
+docker exec -u root -it "$cont_name" usermod --password "$(openssl passwd -1 "$docker_pass")" "$docker_user"
+docker exec -u root -it "$cont_name" usermod -aG sudo "$docker_user"
+docker exec -u root -it "$cont_name" "$repo_tgt/externals/src/emumba/install_emumba-deribit-connector.sh"

requirements.txt

@@ -6,7 +6,7 @@ isort==5.9.3
 lazy-object-proxy==1.6.0
 mccabe==0.6.1
 platformdirs==2.3.0
-protobuf==3.19.1
+protobuf==4.21.12
 pybars3==0.9.7
 pycodestyle==2.7.0
 pycparser==2.20

server_sln/spark_server.sln

@@ -1,41 +0,0 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 15
-VisualStudioVersion = 15.0.26730.10
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sparkd", "src\server\sparkd\sparkd.vcxproj", "{BBBAFD38-312D-4A8D-9AF0-F2BDEE72CE6F}"
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libspark", "build\src\lib\libspark.vcxproj", "{502FD050-679A-3B50-B438-098500700919}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
-		MinSizeRel|x64 = MinSizeRel|x64
-		Release|x64 = Release|x64
-		RelWithDebInfo|x64 = RelWithDebInfo|x64
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{BBBAFD38-312D-4A8D-9AF0-F2BDEE72CE6F}.Debug|x64.ActiveCfg = Debug|x64
-		{BBBAFD38-312D-4A8D-9AF0-F2BDEE72CE6F}.Debug|x64.Build.0 = Debug|x64
-		{BBBAFD38-312D-4A8D-9AF0-F2BDEE72CE6F}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{BBBAFD38-312D-4A8D-9AF0-F2BDEE72CE6F}.MinSizeRel|x64.Build.0 = Release|x64
-		{BBBAFD38-312D-4A8D-9AF0-F2BDEE72CE6F}.Release|x64.ActiveCfg = Release|x64
-		{BBBAFD38-312D-4A8D-9AF0-F2BDEE72CE6F}.Release|x64.Build.0 = Release|x64
-		{BBBAFD38-312D-4A8D-9AF0-F2BDEE72CE6F}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{BBBAFD38-312D-4A8D-9AF0-F2BDEE72CE6F}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{502FD050-679A-3B50-B438-098500700919}.Debug|x64.ActiveCfg = Debug|x64
-		{502FD050-679A-3B50-B438-098500700919}.Debug|x64.Build.0 = Debug|x64
-		{502FD050-679A-3B50-B438-098500700919}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64
-		{502FD050-679A-3B50-B438-098500700919}.MinSizeRel|x64.Build.0 = MinSizeRel|x64
-		{502FD050-679A-3B50-B438-098500700919}.Release|x64.ActiveCfg = Release|x64
-		{502FD050-679A-3B50-B438-098500700919}.Release|x64.Build.0 = Release|x64
-		{502FD050-679A-3B50-B438-098500700919}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64
-		{502FD050-679A-3B50-B438-098500700919}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {6A7249D7-B40C-4656-9347-60072D0B8FDB}
-	EndGlobalSection
-EndGlobal

spark_client.sln

@@ -8,7 +8,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libspark", "build\src\lib\l
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "sparkcmd", "src\client\sparkcmd\sparkcmd.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject

src/Spark.Admin.WebApp/CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.5)
-project(web)
+project(Spark.Admin.WebApp)
 
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/Protobuf.cmake)
@@ -24,45 +24,48 @@ file(
   ${PROTO_GLOBS})
 
 file(
-  GLOB_RECURSE _api_ts_SOURCES
+  GLOB_RECURSE _ts_SOURCES
   RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  api/src/*.ts)
+  src/*.ts)
 
 file(
-  GLOB_RECURSE _ui_fs_SOURCES
+  GLOB_RECURSE _fs_SOURCES
   RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ui/src/*.fs)
+  src/*.fs)
 
 set(_tools ${CMAKE_SOURCE_DIR}/tools)
 set(_quiet_exec ${_tools}/quiet_exec.sh)
 set(_venv_exec ${_tools}/venv_exec.sh)
 set(_tools_py ${CMAKE_SOURCE_DIR}/tools/py)
 
-string(TOLOWER ${CMAKE_BUILD_TYPE} _build_type)
-set(_tsconfig_base ${CMAKE_CURRENT_SOURCE_DIR}/api/tsconfig.${_build_type}.json)
-set(WEB_TSCONFIG_NAME "last" CACHE STRING "Name for the generated tsconfig file")
-set(_tsconfig "${CMAKE_CURRENT_SOURCE_DIR}/api/tsconfig.${WEB_TSCONFIG_NAME}.g.json")
+# The tsconfig file is generated as a function of the cmake build type, and
+# binary dir. Depending on the build type, it extends either
+# 'tsconfig.debug.json' or 'tsconfig.release.json'.
+set(_tsconfig_base "${CMAKE_CURRENT_SOURCE_DIR}/tsconfig.$<IF:$<CONFIG:Debug>,debug,release>.json")
+set(WEBPACK_MODE $<IF:$<CONFIG:Debug>,"development","production">)
+set(_tsconfig "${CMAKE_CURRENT_BINARY_DIR}/tsconfig.json")
 
 set(_venv_python ${CMAKE_SOURCE_DIR}/.venv/bin/python)
 set(_gen_tsconfig_py ${_tools_py}/gen_tsconfig.py)
-set(_build_web_py ${_tools_py}/build_web.py)
+set(_build_admin_web_app_py ${_tools_py}/build_admin_web_app.py)
 
 set(_gen_tsconfig_deps ${_venv_exec} ${_venv_python} ${_gen_tsconfig_py} ${_tsconfig_base})
 
-set(_build_web_deps
+set(_build_admin_web_app_deps
+  ${_quiet_exec}
   ${_venv_exec}
   ${_venv_python}
-  ${_build_web_py}
-  ${CMAKE_CURRENT_SOURCE_DIR}/ui/src/ui.fsproj
-  ${CMAKE_CURRENT_SOURCE_DIR}/ui/webpack.config.js
+  ${_build_admin_web_app_py}
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
+  ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
   ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs
-  ${CMAKE_SOURCE_DIR}/src/web/api/package-lock.json
+  ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
   ${_proto_rel_SOURCES}
   ${_api_ts_SOURCES}
   ${_ui_fs_SOURCES}
   ${_tsconfig})
 
-update_deps_file("${_build_web_deps}")
+update_deps_file("${_build_admin_web_app_deps}")
 
 function(npm_install path)
   add_custom_command(
@@ -73,8 +76,7 @@ function(npm_install path)
     COMMENT "Run 'npm install' in ${path}")
 endfunction()
 
-npm_install(${CMAKE_CURRENT_SOURCE_DIR}/api)
-npm_install(${CMAKE_CURRENT_SOURCE_DIR}/ui)
+npm_install(${CMAKE_CURRENT_SOURCE_DIR})
 
 add_custom_command(
   OUTPUT ${_tsconfig}
@@ -84,22 +86,23 @@ add_custom_command(
   ${_gen_tsconfig_py}
   --config-base ${_tsconfig_base}
   --config-out ${_tsconfig}
-  --build-out ${CMAKE_BINARY_DIR}/web/api
+  --build-out ${CMAKE_CURRENT_BINARY_DIR}
   DEPENDS ${_gen_tsconfig_deps})
 
 add_custom_command(
-  OUTPUT ${CMAKE_BINARY_DIR}/web/ui/bundle.js
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
   COMMAND ${_quiet_exec}
   ARGS
   ${_venv_exec}
   ${_venv_python}
-  ${_build_web_py}
+  ${_build_admin_web_app_py}
   --protos ${_proto_abs_SOURCES}
   --tsconfig ${_tsconfig}
-  --output-dir ${CMAKE_BINARY_DIR}/web
-  COMMENT "Building Web UI"
-  DEPENDS ${_build_web_deps})
+  --mode ${WEBPACK_MODE}
+  --output-dir ${CMAKE_CURRENT_BINARY_DIR}
+  COMMENT "Building Spark.Admin.WebApp"
+  DEPENDS ${_build_admin_web_app_deps})
 
-add_custom_target(web_ui_bundle ALL DEPENDS ${CMAKE_BINARY_DIR}/web/ui/bundle.js)
+add_custom_target(admin_web_app_bundle ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bundle.js)
 
-install(DIRECTORY "${CMAKE_BINARY_DIR}/web" DESTINATION bin)
+install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "${SPARK_INSTALL_PREFIX}/bin")

src/Spark.Admin.WebApp/Spark.Admin.WebApp.sln

@@ -0,0 +1,21 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.31903.59
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Spark.Admin.WebApp", "src\Spark.Admin.WebApp.fsproj", "{C3143A6F-8D32-4F9E-84EE-B34AF3F3A862}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{C3143A6F-8D32-4F9E-84EE-B34AF3F3A862}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C3143A6F-8D32-4F9E-84EE-B34AF3F3A862}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C3143A6F-8D32-4F9E-84EE-B34AF3F3A862}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C3143A6F-8D32-4F9E-84EE-B34AF3F3A862}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+EndGlobal

src/Spark.Admin.WebApp/package.json

@@ -0,0 +1,49 @@
+{
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "npx webpack serve",
+    "build": "npx webpack"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@babel/plugin-transform-runtime": "^7.21.0",
+    "@babel/preset-env": "^7.20.2",
+    "@babel/runtime": "^7.21.0",
+    "babel-loader": "^9.1.2",
+    "fable-utils": "^1.2.0",
+    "node-sass": "^8.0.0",
+    "protobufjs": "^7.2.2",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "remotedev": "^0.2.9",
+    "ts2fable": "^0.7.1",
+    "tslib": "^2.5.0",
+    "watch": "^1.0.2",
+    "webpack": "^5.75.0"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.21.0",
+    "@fortawesome/fontawesome-free": "^5.15.4",
+    "bulma": "^0.9.4",
+    "copy-webpack-plugin": "^11.0.0",
+    "core-js": "^3.28.0",
+    "css-loader": "^6.7.3",
+    "html-webpack-plugin": "^5.5.0",
+    "jest": "^29.4.3",
+    "mini-css-extract-plugin": "^2.7.2",
+    "npx": "^10.2.2",
+    "protobufjs-cli": "^1.1.1",
+    "sass": "^1.58.3",
+    "sass-loader": "^13.2.0",
+    "style-loader": "^3.3.1",
+    "ts-jest": "^29.0.5",
+    "ts-loader": "^9.4.2",
+    "tsutils": "^3.21.0",
+    "typescript": "^4.9.5",
+    "webpack-cli": "^5.0.1",
+    "webpack-dev-server": "^4.11.1",
+    "ws": "^8.12.1"
+  }
+}

src/Spark.Admin.WebApp/public/index.html

@@ -4,8 +4,8 @@
   <title>BTS Spark Admin</title>
   <meta http-equiv='Content-Type' content='text/html; charset=utf-8'>
   <meta name="viewport" content="width=device-width, initial-scale=1">
-  <link rel="shortcut icon" type="image/ico" href="img/favicon.ico" />
-  <link rel="stylesheet" href="font-awesome.min.css" />
+  <link rel="shortcut icon" type="image/ico" href="public/img/favicon.ico" />
+  <link rel="stylesheet" href="public/font-awesome.min.css" />
   <script src="https://cdn.polyfill.io/v2/polyfill.js?features=es6"></script>
 </head>
 <body>

src/Spark.Admin.WebApp/src/Accounts/State.fs

@@ -170,8 +170,8 @@ let marginSub model dispatch =
 
 let selectSub (tab:Tab option) model =
   match tab, model.tab with
-  | Some PositionManagementTab, _ | None, PositionManagementTab -> Cmd.ofSub (positionSub model)
-  | Some MarginTab, _ | None, MarginTab -> Cmd.ofMsg GetMarginData //Cmd.ofSub (marginSub model)
+  | Some PositionManagementTab, _ | None, PositionManagementTab -> Cmd.ofEffect (positionSub model)
+  | Some MarginTab, _ | None, MarginTab -> Cmd.ofMsg GetMarginData //Cmd.ofEffect (marginSub model)
   | _ ->
     match model.unsub with
     | Some (unsub) -> unsub.Dispose()

src/Spark.Admin.WebApp/src/Accounts/View.fs

@@ -1225,7 +1225,7 @@ let accountSelectionGrid (accountList:BTSAccount list) (currentAccount:BTSAccoun
         onClick = None
       }
     ]
-  div[] [
+  div [] [
     TableControl.createSelectorTable (Some headerRow) rws
   ]
 
@@ -1265,7 +1265,7 @@ let menuItem account selectedAccount model dispatch =
                  ]
              ]
          ]
-      else span[][])
+      else span [] [])
     ]
 
 let NavRoot (accounts:BTSAccount[]) selectedAccount model dispatch =
@@ -1312,7 +1312,7 @@ let root (model:Model) dispatch =
               | "" -> span [] []
               | _ -> Modal.showConfirm (fst model.ConfirmModal) (fun () -> (snd model.ConfirmModal) |> dispatch) (fun _ -> OpenConfirmModal ("", Msg.SelectTab model.tab ) |> dispatch)
   match model.adminData with
-    | None -> div[][str "Loading account list..."]
+    | None -> div [] [str "Loading account list..."]
     | Some data ->
       div
         [ Class "columns is-fullheight"

src/Spark.Admin.WebApp/src/App.fs

@@ -7,10 +7,6 @@ open Types
 open App.State
 open App.Types
 open Global
-open Fable.Core.JsInterop
-open Types
-open App.State
-open Global
 
 importAll "../sass/main.scss"
 

src/Spark.Admin.WebApp/src/Controls/ReactGrid.fs

@@ -146,8 +146,8 @@ let buttonColumn action header icon misc =
         rowIdx = 0;
         misc = misc;
     }
-    [ Fable.React.Props.HTMLAttr.Name header
-      Fable.React.Props.HTMLAttr.Style [ Fable.React.Props.CSSProp.Width 60 ]
+    [ HTMLAttr.Name header
+      Style [ Fable.React.Props.CSSProp.Width 60 ]
     ]
 
 //   [ Name header

src/Spark.Admin.WebApp/src/Login/Login.fs

@@ -52,7 +52,7 @@ let update msg model =
 // View
 
 let root model dispatch =
-    let sparkImg = "img/BTS-300.png"
+    let sparkImg = "public/img/BTS-300.png"
     let handler msg (ev:Browser.Types.Event) =
         !!ev.target?value |> msg |> dispatch
     section [ ClassName "hero is-fullheight is-primary" ] [

src/Spark.Admin.WebApp/src/Spark.Admin.WebApp.fsproj

@@ -1,22 +1,26 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Version>1.0.0</Version>
     <AssemblyVersion>1.0.0</AssemblyVersion>
   </PropertyGroup>
   <Choose>
-    <When Condition="'$(WEB_BUILD_PATH)'==''">
+    <When Condition="'$(SPARK_ADMIN_WEB_APP_OUTPUT_DIR)'==''">
       <PropertyGroup>
-        <API>../../../../build/web/api</API>
+        <API>../../../build/Spark.Admin.WebApp</API>
       </PropertyGroup>
     </When>
     <Otherwise>
       <PropertyGroup>
-        <API>$(WEB_BUILD_PATH)/api</API>
+        <API>$(SPARK_ADMIN_WEB_APP_OUTPUT_DIR)</API>
       </PropertyGroup>
     </Otherwise>
   </Choose>
   <ItemGroup>
+    <!-- Global to the app -->
+    <Compile Include="Lens.fs" />
+    <Compile Include="Cmd.fs" />
+
     <Compile Include="$(API)/ServiceContext.fs" />
     <Compile Include="$(API)/DynamicService.fs" />
     <Compile Include="Imports.fs" />
@@ -37,9 +41,7 @@
     <Compile Include="$(API)/fable/co2.fable.fs" />
     <Compile Include="$(API)/fable/MarketDataRelay.fable.fs" />
     <!-- Global to the app -->
-    <Compile Include="Lens.fs" />
     <Compile Include="Global.fs" />
-    <Compile Include="Cmd.fs" />
     <Compile Include="Controls/nbsp.fs" />
     <Compile Include="Controls/Button.fs" />
     <Compile Include="Controls/ReactGrid.fs" />
@@ -68,13 +70,17 @@
     <Compile Include="App.fs" />
   </ItemGroup>
   <ItemGroup>
+    <PackageReference Include="Elmish" Version="4.0.1" />
     <PackageReference Include="Elmish.Navigation" Version="1.0.0" />
-    <PackageReference Include="Fable.Core" Version="4.0.0-snake-island-alpha-005" />
-    <PackageReference Include="Fable.Elmish" Version="4.0.0-beta-3" />
-    <PackageReference Include="Fable.Elmish.React" Version="4.0.0-beta-2" />
-    <PackageReference Include="Fable.Elmish.UrlParser" Version="1.0.0-beta-1" />
-    <PackageReference Include="Fable.Elmish.Debugger" Version="4.0.0-beta-1" />
-    <PackageReference Include="Fable.Elmish.HMR" Version="6.0.0-beta-002" />
-    <PackageReference Include="Fable.Fetch" Version="2.4.0" />
+    <PackageReference Include="Fable.Core" Version="4.0.0-theta-007" />
+    <PackageReference Include="Fable.Elmish" Version="4.0.1" />
+    <PackageReference Include="Fable.Elmish.Browser" Version="4.0.1" />
+    <PackageReference Include="Fable.Elmish.React" Version="4.0.0" />
+    <PackageReference Include="Fable.Elmish.UrlParser" Version="1.0.1" />
+    <PackageReference Include="Fable.Elmish.Debugger" Version="4.0.0" />
+    <PackageReference Include="Fable.Elmish.HMR" Version="7.0.0" />
+    <PackageReference Include="Fable.Promise" Version="3.2.0" />
+    <PackageReference Include="Fable.React" Version="9.2.0" />
+    <PackageReference Include="System.Runtime" Version="4.3.1" />
   </ItemGroup>
 </Project>

src/Spark.Admin.WebApp/src/Users/State.fs

@@ -6,7 +6,6 @@ open System
 open System.Runtime.InteropServices
 open Fable.Core
 open Fable.Core.JsInterop
-open System.Xml
 open bts.api.admin
 open Global
 

src/Spark.Admin.WebApp/src/Users/View.fs

@@ -119,7 +119,7 @@ let generalTabView (model:Model) (dispatch:Msg->Unit) (readonly: bool)=
         Fields.text "Company" model.scratchUser.company [Disabled readonly] (Company >> UpdateField >> dispatch)
         Fields.checkbox "Can Login?" "Unchecking this stops the user from logging into Spark" model.scratchUser.isActive [Disabled can_login_disabled] (IsActive >> UpdateField >> dispatch)
         Fields.checkbox "Can Create?" "Checking this allows the user to create other users and accounts" model.scratchUser.canCreate [Disabled (readonly || not can_create)] (CanCreate  >> UpdateField >> dispatch)
-        (if (State.isNewUser model.scratchUser) then modal else span[][])
+        (if (State.isNewUser model.scratchUser) then modal else span [] [])
       ]
 
 let accountNameLookup (accountsList:BTSAccount list) (id:string) =
@@ -334,7 +334,7 @@ let tabContent (model:Model) (dispatch:Msg->Unit) display_md display_risk =
     | Tab.MarketData, true -> MarketDataTabView model dispatch (not display_md)
     | Tab.OrderRouting, true -> orderRoutingTabView model dispatch (not display_risk) )
 
-    (if (State.userIsSelected model.scratchUser State.defaultScratchUser) then userControls model dispatch else span[][])
+    (if (State.userIsSelected model.scratchUser State.defaultScratchUser) then userControls model dispatch else span [] [])
   ]
 
 let usersMenu (usersList:BTSUser list) (currentUser:BTSUser) (dispatch:Msg->Unit) =
@@ -395,7 +395,7 @@ let userSelectionGrid (usersList:BTSUser list) (currentUser:BTSUser) (dispatch:M
         onClick = None
       }
     ]
-  div[][
+  div [] [
     TableControl.createSelectorTable (Some headerRow) rws
   ]
 
@@ -444,7 +444,7 @@ let menuItem (user:BTSUser) (selectedUser:BTSUser) model dispatch =
              ]
          ]
       else
-         span[][])
+         span [] [])
     ]
 
 let NavRoot (usersList:BTSUser list) (currentUser:BTSUser) model (dispatch:Msg->Unit) =
@@ -496,7 +496,7 @@ let root (model:Model) (dispatch:Msg->Unit) =
               | "" -> span [] []
               | _ -> Modal.showConfirm (fst model.ConfirmModal) (fun () -> (snd model.ConfirmModal) |> dispatch) (fun _ -> OpenConfirmModal ("", Msg.SwitchTab model.currentTab ) |> dispatch)
   (match model.adminData with
-    | None  -> div[][str "Loading user list..."]
+    | None  -> div [] [str "Loading user list..."]
     | Some data ->
       div
         [ Class "columns is-fullheight"

src/Spark.Admin.WebApp/webpack.config.js

@@ -25,15 +25,15 @@ process.env.NODE_ENV = isProduction ? 'production' : 'development'
 console.log("Bundling for " + (isProduction ? "production" : "development") + "...");
 
 module.exports = env => {
-  const build_path = process.env.WEB_BUILD_PATH
-  const output_path = build_path + "/ui/";
-  const api_path = build_path + "/api/";
+  const build_path = process.env.SPARK_ADMIN_WEB_APP_OUTPUT_DIR
+  const output_path = build_path
+  const api_path = build_path
   return {
     watchOptions: {
       ignored: "./**/*"
     },
     devtool: isProduction ? undefined : "source-map",
-    entry: resolve('src/ui.fsproj'),
+    entry: resolve(`${output_path}/App.js`),
     output: {
         filename: 'bundle.js',
         path: output_path,
@@ -59,14 +59,10 @@ module.exports = env => {
     module: {
         rules: [
             {
-                test: /\.fs(x|proj)?$/,
-                use: {
-                    loader: "fable-loader",
-                    options: {
-                        babel: babelOptions,
-                        define: isProduction ? [] : ["DEBUG"]
-                    }
-                }
+              test: /\.([cm]?ts|tsx)$/,
+              use: [
+                "ts-loader"
+              ]
             },
             {
                 test: /\.js$/,

src/client/Directory.Build.Props

@@ -1,53 +0,0 @@
-<Project>
-
-  <PropertyGroup>
-    <ComputerSpecificConfig>..\..\tools\override\$(computername).props</ComputerSpecificConfig>
-    <SparkRoot>$([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\..))</SparkRoot>
-    <SourceRoot>$([System.IO.Path]::GetFullPath($(SparkRoot)\src))</SourceRoot>
-    <Externals>$(SparkRoot)\externals\windows</Externals>
-    <DepotName>$([System.IO.Path]::GetFileName($([System.IO.Path]::GetFullPath($(SparkRoot)))))</DepotName>
-    <ErrorReport>None</ErrorReport>
-    <Prefer32Bit>False</Prefer32Bit>
-    <Deterministic>true</Deterministic>
-    <EnableDefaultContentItems>False</EnableDefaultContentItems>
-    <ShowMissingItemTypes>false</ShowMissingItemTypes>
-    <Configuration Condition="'$(configuration)'==''">Debug</Configuration>
-    <EnablePreviewFeatures>true</EnablePreviewFeatures>
-  </PropertyGroup>
-
-  <PropertyGroup Label="Output">
-    <BaseOutputPath>$(SparkRoot)\build\$(Configuration)</BaseOutputPath>
-    <OutputPath>$(BaseOutputPath)\bin</OutputPath>
-    <BaseIntermediateOutputPath>$(BaseOutputPath)\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
-    <IntermediateOutputPath>$(BaseOutputPath)\obj\$(MSBuildProjectName)</IntermediateOutputPath>
-    <GenDir>$(BaseOutputPath)\gen\$(MSBuildProjectName)</GenDir>
-  </PropertyGroup>
-
-  <Import
-      Project="$(ComputerSpecificConfig)"
-      Condition="Exists($(ComputerSpecificConfig))" />
-
-  <PropertyGroup Label="ProtoBuf">
-    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
-    <protoc>$(Externals)\bin\protoc.exe</protoc>
-    <protogen>$(Externals)\bin\protogen.exe</protogen>
-  </PropertyGroup>
-
-  <ItemDefinitionGroup>
-    <Proto2></Proto2>
-    <Proto3>
-      <OutBase>$(IntermediateOutputPath)\gen\%(relativedir)</OutBase>
-      <OutCS>%(OutBase)%(FileName).cs</OutCS>
-      <OutCSClient>%(OutBase)%(FileName).client.pb.cs</OutCSClient>
-      <OutCSServer>%(OutBase)%(FileName).server.pb.cs</OutCSServer>
-      <OutCSImmutable>%(OutBase)%(FileName).immutable.pb.cs</OutCSImmutable>
-      <CompileOptions></CompileOptions>
-    </Proto3>
-    <Proto3Excel>
-      <OutBase>$(IntermediateOutputPath)\gen\%(relativedir)</OutBase>
-      <OutCSExcel>%(OutBase)%(FileName).excel.pb.cs</OutCSExcel>
-      <CompileOptions></CompileOptions>
-    </Proto3Excel>
-  </ItemDefinitionGroup>
-
-</Project>

src/client/Directory.Build.Targets

@@ -1,94 +0,0 @@
-<Project>
-
-  <Target
-      Name="CompileProto2"
-      Condition="'@(Proto2)'!=''"
-      Inputs="$(MSBuildProjectFile);$(protogen);%(Proto2.FullPath)"
-      Outputs="$(GenDir)\proto2\%(Proto2.filename).cs"
-      AfterTargets="PreBuildEvent">
-    <PropertyGroup>
-      <SourceRelativeFile>$([System.String]::Copy('%(proto2.fullpath)').Replace($(SourceRoot), ''))</SourceRelativeFile>
-      <SourceRelativeDir>$([System.IO.Path]::GetDirectoryName($(SourceRelativeFile)))</SourceRelativeDir>
-      <WorkingDirectory>$(GenDir)\proto2</WorkingDirectory>
-    </PropertyGroup>
-    <MakeDir Directories="$(WorkingDirectory)" />
-    <Copy
-        SourceFiles="%(Proto2.FullPath)"
-        DestinationFolder="$(WorkingDirectory)"
-        SkipUnchangedFiles="True"
-        UseHardLinksIfPossible="True" />
-    <Error
-        Text="protogen.exe does not exist. If you forgot to clone with externals try: git submodule update --init --recursive"
-        Condition="!Exists($(protogen))" />
-    <Exec
-        Command="$(Protogen) %(Proto2.filename)%(proto2.extension) --csharp_out=. -I. -I$(SourceRoot) -I$(Externals)\include"
-        WorkingDirectory="$(WorkingDirectory)" />
-    <ItemGroup>
-      <Compile Include="$(WorkingDirectory)\%(Proto2.filename).cs" />
-    </ItemGroup>
-  </Target>
-
-  <!-- 
-    Run this after GetAssemblyVersion because it happens after the Xaml compiler 
-    and exists in both xaml and non-xaml projects.  One of the (many) xaml compile
-    passes wipes out the additional compile includes that we add.
-  -->
-  <Target
-      Name="CompileProto3"
-      AfterTargets="GetAssemblyVersion"
-      Condition="'@(Proto3)'!=''"
-      Inputs="$(protoc);
-    %(Proto3.Identity);$(ProtocPluginBase)\plugin.py;
-    $(ProtocPluginBase)\templates\c#\client.pb.cs;
-    $(ProtocPluginBase)\templates\c#\server.pb.cs;
-    $(ProtocPluginBase)\templates\c#\immutable.pb.cs;"
-      Outputs="%(Proto3.OutCS);%(Proto3.OutCSServer);%(Proto3.OutCSClient);%(Proto3.OutCSImmutable)">
-    <MakeDir Directories="%(Proto3.OutBase)" />
-
-    <Message
-        Importance="High"
-        Text="Generating service %(Proto3.FileName)" />
-    <Exec Command="$(SparkRoot)\.venv\Scripts\activate.bat" />
-    <Exec Command="set PYTHONPATH=$(SparkRoot)\.venv\Scripts\python" />
-    <Exec
-        Command="
-       $(protoc) %* %(Proto3.FullPath) ^
-       --error_format=msvs ^
-       --proto_path=$(SourceRoot) ^
-       --plugin=&quot;protoc-gen-service=$(ProtocPluginBase)\plugin.cmd&quot; ^
-       --service_out=lang=c#,strip_out_path=1:%(Proto3.OutBase) ^
-       --csharp_out=%(CompileOptions)%(Proto3.OutBase)"
-        WorkingDirectory="%(relativedir)" />
-    <ItemGroup>
-      <Compile Include="%(Proto3.OutCS)" />
-      <Compile Include="%(Proto3.OutCSClient)" />
-      <Compile Include="%(Proto3.OutCSServer)" />
-      <Compile Include="%(Proto3.OutCSImmutable)" />
-    </ItemGroup>
-  </Target>
-
-  <Target Name="RemoveSystemRuntimeFromRefPack"
-          BeforeTargets="_HandlePackageFileConflicts"
-          Condition="'@(Reference -> WithMetadataValue('NugetPackageId', 'System.Runtime.Experimental'))' != ''">
-    <ItemGroup>
-      <Reference Remove="@(Reference)"
-                 Condition="$([System.String]::Copy(%(Reference.Identity)).Contains('System.Runtime.dll')) and
-                            '%(Reference.NuGetPackageId)' == 'Microsoft.NETCore.App.Ref'" />
-    </ItemGroup>
-  </Target>
-
-  <Target
-      Name="AfterClean"
-      AfterTargets="Clean">
-    <Delete Files="%(Proto3.OutCS)" />
-    <Delete Files="%(Proto3.OutCSClient)" />
-    <Delete Files="%(Proto3.OutCSServer)" />
-    <Delete Files="%(Proto3.OutCSImmutable)" />
-    <Delete Files="$(GenDir)\proto2\*" />
-  </Target>
-
-  <Target
-      Name="Test"
-      DependsOnTargets="Build"></Target>
-
-</Project>

src/client/Directory.Build.props

@@ -0,0 +1,82 @@
+<Project>
+
+  <PropertyGroup>
+    <ComputerSpecificConfig>..\..\tools\override\$(computername).props</ComputerSpecificConfig>
+    <SparkRoot>$([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\..))</SparkRoot>
+    <SourceRoot>$([System.IO.Path]::GetFullPath($(SparkRoot)\src))</SourceRoot>
+    <DepotName>$([System.IO.Path]::GetFileName($([System.IO.Path]::GetFullPath($(SparkRoot)))))</DepotName>
+    <ErrorReport>None</ErrorReport>
+    <Prefer32Bit>False</Prefer32Bit>
+    <Deterministic>true</Deterministic>
+    <EnableDefaultContentItems>False</EnableDefaultContentItems>
+    <ShowMissingItemTypes>false</ShowMissingItemTypes>
+    <Configuration Condition="'$(configuration)'==''">Debug</Configuration>
+    <EnablePreviewFeatures>true</EnablePreviewFeatures>
+    <LangVersion>11</LangVersion>
+  </PropertyGroup>
+
+  <Choose>
+    <When Condition="'$(OS)' == 'Windows_NT'">
+      <PropertyGroup>
+        <Externals>$(SparkRoot)/externals/windows</Externals>
+      </PropertyGroup>
+    </When>
+    <Otherwise>
+      <PropertyGroup>
+        <Externals>$(SparkRoot)/externals/linux</Externals>
+      </PropertyGroup>
+    </Otherwise>
+  </Choose>
+
+  <PropertyGroup Label="Output">
+    <BaseOutputPath>$(SparkRoot)\build\$(Configuration)</BaseOutputPath>
+    <OutputPath>$(BaseOutputPath)\bin</OutputPath>
+    <BaseIntermediateOutputPath>$(BaseOutputPath)\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <IntermediateOutputPath>$(BaseOutputPath)\obj\$(MSBuildProjectName)</IntermediateOutputPath>
+    <ProtoOutputPath>$(BaseOutputPath)\obj\$(MSBuildProjectName)</ProtoOutputPath>
+  </PropertyGroup>
+
+  <Import
+      Project="$(ComputerSpecificConfig)"
+      Condition="Exists($(ComputerSpecificConfig))" />
+
+  <PropertyGroup Label="ProtoBuf">
+    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
+  </PropertyGroup>
+  <Choose>
+    <When Condition="'$(OS)' == 'Windows_NT'">
+      <PropertyGroup Label="ProtoBuf">
+        <protoc>$(Externals)/bin/protoc.exe</protoc>
+        <ProtocPlugin>$(ProtocPluginBase)/plugin.cmd</ProtocPlugin>
+      </PropertyGroup>
+    </When>
+    <Otherwise>
+      <PropertyGroup Label="ProtoBuf">
+        <protoc>$(Externals)/bin/protoc</protoc>
+        <ProtocPlugin>$(ProtocPluginBase)/plugin.sh</ProtocPlugin>
+      </PropertyGroup>
+    </Otherwise>
+  </Choose>
+
+  <ItemDefinitionGroup>
+    <!--
+    Each .proto input file produces 4 output files. For example,
+    'foo_bar.proto' produces:
+      * FooBar.cs
+      * foo_bar.client.pb.cs
+      * foo_bar.server.pb.cs
+      * foo_bar.immutable.pb.cs
+    Google automatically converts .proto filenames to TitleCase when
+    generating .cs files. Unless the .proto file is already in TitleCase
+    (against Google's recommended practice to use snake_case .proto files),
+    the 'CSName' must be overridden with the TitleCase formatting.
+    For example:
+      <Proto3 Include="foo_bar.proto" CSName="FooBar" />
+    -->
+    <Proto3>
+      <OutDir>$(ProtoOutputPath)/</OutDir>
+      <CSName>%(FileName)</CSName>
+    </Proto3>
+  </ItemDefinitionGroup>
+
+</Project>

src/client/Directory.Build.targets

@@ -0,0 +1,68 @@
+<Project>
+
+  <!-- 
+    Run this after GetAssemblyVersion because it happens after the Xaml compiler 
+    and exists in both xaml and non-xaml projects.  One of the (many) xaml compile
+    passes wipes out the additional compile includes that we add.
+  -->
+  <Target
+      Name="CompileProto3"
+      AfterTargets="GetAssemblyVersion"
+      Condition="'@(Proto3)'!=''"
+      Inputs="$(protoc);
+        %(Proto3.Identity);$(ProtocPluginBase)/plugin.py;
+        $(ProtocPluginBase)/templates/c#/client.pb.cs;
+        $(ProtocPluginBase)/templates/c#/server.pb.cs;
+        $(ProtocPluginBase)/templates/c#/immutable.pb.cs;"
+      Outputs="
+        %(Proto3.OutDir)/%(Proto3.CSName).cs;
+        %(Proto3.OutDir)/%(Proto3.FileName).client.pb.cs;
+        %(Proto3.OutDir)/%(Proto3.FileName).server.pb.cs;
+        %(Proto3.OutDir)/%(Proto3.FileName).immutable.pb.cs">
+    <MakeDir Directories="%(Proto3.OutDir)" />
+
+    <Message
+        Importance="High"
+        Text="Generating service %(Proto3.FileName)" />
+
+    <PropertyGroup>
+      <GenProtosCmd>$(protoc) %(Proto3.FullPath)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-service=$(ProtocPlugin)&quot;</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --service_out=lang=c#,strip_out_path=1:%(Proto3.OutDir)</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(Proto3.OutDir)</GenProtosCmd>
+    </PropertyGroup>
+    <Exec Command="$(GenProtosCmd)" WorkingDirectory="%(Proto3.RelativeDir)" />
+    <ItemGroup>
+      <Compile Include="%(Proto3.OutDir)/%(Proto3.CSName).cs" />
+      <Compile Include="%(Proto3.OutDir)/%(Proto3.FileName).client.pb.cs" />
+      <Compile Include="%(Proto3.OutDir)/%(Proto3.FileName).server.pb.cs" />
+      <Compile Include="%(Proto3.OutDir)/%(Proto3.FileName).immutable.pb.cs" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="RemoveSystemRuntimeFromRefPack"
+          BeforeTargets="_HandlePackageFileConflicts"
+          Condition="'@(Reference -> WithMetadataValue('NugetPackageId', 'System.Runtime.Experimental'))' != ''">
+    <ItemGroup>
+      <Reference Remove="@(Reference)"
+                 Condition="$([System.String]::Copy(%(Reference.Identity)).Contains('System.Runtime.dll')) and
+                            '%(Reference.NuGetPackageId)' == 'Microsoft.NETCore.App.Ref'" />
+    </ItemGroup>
+  </Target>
+
+  <Target
+      Name="AfterClean"
+      AfterTargets="Clean">
+    <Delete Files="%(Proto3.OutDir)/%(Proto3.CSName).cs" />
+    <Delete Files="%(Proto3.OutDir)/%(Proto3.FileName).client.pb.cs" />
+    <Delete Files="%(Proto3.OutDir)/%(Proto3.FileName).server.pb.cs" />
+    <Delete Files="%(Proto3.OutDir)/%(Proto3.FileName).immutable.pb.cs" />
+  </Target>
+
+  <Target
+      Name="Test"
+      DependsOnTargets="Build"></Target>
+
+</Project>

src/client/Spark.Console/Spark.Console.csproj

@@ -2,7 +2,8 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFrameworks>net6.0</TargetFrameworks>
+    <TargetFrameworks>net7.0</TargetFrameworks>
+    <Nullable>enable</Nullable>
     <RuntimeIdentifiers>win10-x64;ubuntu.16.04-x64</RuntimeIdentifiers>
     <Authors>Blue Trading Systems</Authors>
     <Company>Blue Trading Systems</Company>
@@ -14,10 +15,8 @@
 
   <ItemGroup>
     <ProjectReference Include="../spark_api/spark_api.csproj" />
-    <!--<ProjectReference Include="..\spark\Spark.csproj" />-->
-    <ProjectReference Include="..\spark_core\spark_core.csproj" />
-    <ProjectReference Include="..\spark_test\spark_test.csproj" />
-    <!--<Proto3 Include="**/*.proto" />-->
+    <ProjectReference Include="../spark_core/spark_core.csproj" />
+    <ProjectReference Include="../spark_test/spark_test.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/client/Spark.Console/src/CommandParserEx.cs

@@ -0,0 +1,32 @@
+namespace Spark.Console {
+   using CommandLine;
+
+   using System;
+   using System.Collections.Generic;
+
+   public static class CommandParserEx {
+      public static TResult parse_and_run<T1, T2, T3, T4, T5, T6, T7, T8, TResult>(
+         this Parser parser,
+         IEnumerable<string> args,
+         Func<T1, TResult> on_args1,
+         Func<T2, TResult> on_args2,
+         Func<T3, TResult> on_args3,
+         Func<T4, TResult> on_args4,
+         Func<T5, TResult> on_args5,
+         Func<T6, TResult> on_args6,
+         Func<T7, TResult> on_args7,
+         Func<T8, TResult> on_args8,
+         Func<IEnumerable<Error>, TResult> on_error) =>
+         parser.ParseArguments<T1, T2, T3, T4, T5, T6, T7, T8>(args)
+            .MapResult(
+               on_args1,
+               on_args2,
+               on_args3,
+               on_args4,
+               on_args5,
+               on_args6,
+               on_args7,
+               on_args8,
+               on_error);
+   }
+}

src/client/Spark.Console/src/Commands/ConfigCommand.cs

@@ -0,0 +1,27 @@
+namespace Spark.Console {
+   using bts;
+
+   using mbts.config;
+
+   using System;
+   using System.Linq;
+
+   class ConfigCommand {
+      public static int run(ConfigCommandOptions opts) {
+         if (opts.sparkd is not null) {
+            var args = Environment.GetCommandLineArgs();
+            BTS.set_config_arg(args, opts.sparkd);
+         }
+         var keys = opts.keys?.ToArray() ?? Array.Empty<string>();
+         var excluded_keys = opts.excluded_keys?.ToArray() ?? Array.Empty<string>();
+         return Configuration.run_config_query(
+            keys,
+            keys.Length,
+            excluded_keys,
+            excluded_keys.Length,
+            opts.all,
+            opts.list_keys,
+            opts.verbose);
+      }
+   }
+}

src/client/Spark.Console/src/Commands/ConfigCommandOptions.cs

@@ -0,0 +1,30 @@
+namespace Spark.Console {
+   using CommandLine;
+
+   using System.Collections.Generic;
+
+   [Verb("config", HelpText = "Get config values")]
+   class ConfigCommandOptions {
+      [Option(
+         "query", MetaValue = "keys", HelpText = "Keys to query (supports regular expressions)")]
+      public IEnumerable<string>? keys { get; set; }
+
+      [Option(
+         "exclude",
+         MetaValue = "keys",
+         HelpText = "Keys to exclude from query (supports regular expressions)")]
+      public IEnumerable<string>? excluded_keys { get; set; }
+
+      [Option("all", HelpText = "Query all keys")]
+      public bool all { get; set; }
+
+      [Option("list-keys", HelpText = "List available keys")]
+      public bool list_keys { get; set; }
+
+      [Option("verbose", HelpText = "Show verbose output")]
+      public bool verbose { get; set; }
+
+      [Option("sparkd", MetaValue = "URL", HelpText = "The sparkd server")]
+      public string? sparkd { get; set; }
+   }
+}

src/client/Spark.Console/src/Commands/MBOCommand.cs

@@ -0,0 +1,47 @@
+namespace Spark.Console {
+   using bts.api;
+   using bts.api.core;
+   using bts.md;
+
+   using System;
+   using System.Linq;
+
+   static class MBOCommand {
+      public static int run(MBOCommandOptions o) {
+         _dump_mbo(o.key, o.side);
+         return 0;
+      }
+
+      static void _dump_mbo(uint key, Side side) {
+         using var ctx = new ServiceContext();
+         var mbo = ctx.locate<MBOService>();
+         var data = mbo.get_orders_for_security(
+            new MBOSecurityQuery { SecurityId = key, PriceLevel = double.NaN });
+
+         Console.WriteLine($"Order data for {data.SecurityId}");
+         var price = 0.0;
+         var color = Console.ForegroundColor;
+         if (side == Side.Sell || side == Side.Undefined) {
+            Console.ForegroundColor = ConsoleColor.Red;
+            foreach (var e in data.Ask.Reverse()) {
+               if (e.Price != price) {
+                  price = e.Price;
+                  Console.WriteLine("================");
+               }
+               Console.WriteLine(e);
+            }
+         }
+         if (side == Side.Buy || side == Side.Undefined) {
+            Console.ForegroundColor = ConsoleColor.Blue;
+            foreach (var e in data.Bid) {
+               if (e.Price != price) {
+                  price = e.Price;
+                  Console.WriteLine("================");
+               }
+               Console.WriteLine(e);
+            }
+         }
+         Console.ForegroundColor = color;
+      }
+   }
+}

src/client/Spark.Console/src/Commands/MBOCommandOptions.cs

@@ -0,0 +1,14 @@
+namespace Spark.Console {
+   using bts.api;
+
+   using CommandLine;
+
+   [Verb("mbo", HelpText = "MBO diagnostic command")]
+   public class MBOCommandOptions {
+      [Option('k', "key", Required = true)]
+      public uint key { get; set; }
+
+      [Option('s', "side", Default = Side.Undefined)]
+      public Side side { get; set; }
+   }
+}

src/client/Spark.Console/src/Commands/OrderStressCommand.cs

@@ -1,168 +1,159 @@
-using bts.api;
-using bts.api.core;
-using bts.api.orderman;
-using bts.co2;
-using bts.test;
-using CommandLine;
-using mbts.co2;
-using mbts.config;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-
-namespace bts {
-
-   [Verb("orderstress", HelpText = "Create a bunch of orders")]
-   public class OrderStressOptions {
-      // Maybe options to create fewer orders or sleep
-
-      [Option('s', "sleep", HelpText = "Sleep ms between order submission", Default = 100)]
-      public int sleep { get; set; }
-
-      [Option('d', "depth", HelpText = "How many orders per side", Default = 10u)]
-      public uint depth { get; set; }
-
-      [Option('q', "quanityt", HelpText = "Order quantity", Default = 1u)]
-      public uint quantity { get; set; }
-   };
-
-   public struct Market {
-      public SecurityImmutable security;
-      public IntPrice? bid;
-      public IntPrice? ask;
-   }
-
-   public static class OrderStress {
-      static IEnumerable<Market> all_markets(ServiceContext ctx) {
-         var channels =
-            UserSymbol.get_all()
-            .Select(u =>
-               new SecurityChannel() {
-                  Time = 0,
-                  Flags = (int)(MDRequest.Types.Flags.Direct | MDRequest.Types.Flags.Implied),
-                  Type = u.type,
-                  Symbol = u.symbol,
-                  Exchange = u.exchange
-               }
-            );
-         var request = new MDRequest() { Host = "system_test" };
-         request.Channels.AddRange(channels);
-
-         var relay = ctx.locate<MarketDataRelay>();
-         System.Threading.Thread.Sleep(100);
-         var result = relay.requestmd(request);
-
-         foreach (var ch in result.Channels) {
-            // if (ch.Platform != Platform.Sim) continue;
-            foreach (var update in ch.Updates) {
-               if (update.Security != null) {
-                  var m = new Market() {
-                     security = new SecurityImmutable(update.Security),
-                     bid = best(
-                        (a,b) => a > b ? a : b,
-                        update.Direct?.Bids?.FirstOrDefault(), update.Implied?.Bids?.FirstOrDefault()),
-                     ask = best(
-                        (a,b) => a < b ? a : b,
-                        update.Direct?.Asks?.FirstOrDefault(), update.Implied?.Asks?.FirstOrDefault())
-                  };
-                  if (m.bid.HasValue || m.ask.HasValue) {
-                     yield return m;
-                  }
-               }
-            }
-         }
-      }
-
-      private static IntPrice? best(Func<IntPrice, IntPrice,IntPrice> comp, params PriceSizeMessage[] ps) {
-         IntPrice? result = null;
-         foreach (var p in ps) {
-            if (p != null) {
-               var ip = new IntPrice(p.Mantissa, (sbyte)p.Exponent);
-               if (result == null) {
-                  result = ip;
-               }
-               else {
-                  result = comp(result.Value, ip);
-               }
-            }
-         }
-         return result;
-      }
-
-      static Create mkorder(uint security, double price, uint size, Side side) {
-         return new Create {
-            Price = price,
-            Quantity = size,
-            NewOrder = new OrderDefinition {
-               SecurityKey = security,
-               Side = side,
-               TimeInForce = TimeInForce.Day,
-               ManualOrderIndicator = ManualOrderIndicator.Yes,
-               CtiCode = CTICode.CtiCode1,
-               CustomerOrFirm = CustomerOrFirm.Firm,
-               AccountId = 1,
-               UserId = 1,
-               CustomDef = new CustomOrderDefinition {
-                  Limit = new LimitOrderDefinition()
-               }
-            }
-         };
-      }
-
-      public static int Run(OrderStressOptions o) {
-
-         var ctx = new ServiceContext();
-         ConsoleEx.WriteLine("Please log in to Spark", ConsoleColor.Yellow);
-         ConsoleEx.Write("Username: ", ConsoleColor.Yellow);
-         var name = Console.ReadLine();
-         ConsoleEx.Write("Password: ", ConsoleColor.Yellow);
-         var pw = Console.ReadLine();
-
-         var login_result = ctx.login(name, pw);
-         if (!login_result.Result.success) {
-            ConsoleEx.WriteLine("Invalid Credentials", ConsoleColor.Red);
-         }
-
-         if (Configuration.get<config.DeploymentType>().type.ToUpper() == "PROD") {
-            ConsoleEx.WriteLine("Refusing to run on production system", ConsoleColor.Red);
-            return -1;
-         }
-
-         var orders = ctx.locate<OrderService>();
-         var factor = TimeSpan.TicksPerMillisecond / 1000;
-         foreach (var m in all_markets(ctx)) {
-            ConsoleEx.WriteLine(m.security.security_desc(), ConsoleColor.White);
-            var tick = m.security.ContractSpec.LowTick;
-            if (m.ask.HasValue) {
-               for (var x = 0; x < o.depth; ++x) {
-                  var price = m.ask.Value.price() + (tick * x);
-                  ConsoleEx.WriteLine($"Sell {o.quantity} at {price}", ConsoleColor.Red);
-                  var sw = new System.Diagnostics.Stopwatch();
-                  sw.Start();
-                  var r = orders.create_order(mkorder(m.security.Id, price, o.quantity, Side.Sell));
-                  sw.Stop();
-                  // Log.Api.info($"Order: id {r.OrderKey} time {sw.ElapsedTicks/factor}us");
-                  Log.Api.info($"Order {m.security.security_desc()}: id {r.OrderKey} {r.ErrorCode} time {sw.ElapsedTicks/factor}us");
-                  System.Threading.Thread.Sleep(o.sleep);
-               }
-            }
-            if (m.bid.HasValue) {
-               // start at bid and place 1 order at 10 levels
-               for (var x=0; x < o.depth; ++x) {
-                  var price = m.bid.Value.price() - (tick * x);
-                  ConsoleEx.WriteLine($"Buy {o.quantity} at {price}", ConsoleColor.Blue);
-                  var sw = new System.Diagnostics.Stopwatch();
-                  sw.Start();
-                  var r = orders.create_order(mkorder(m.security.Id, price, o.quantity, Side.Buy));
-                  sw.Stop();
-                  Log.Api.info($"Order {m.security.security_desc()}: id {r.OrderKey} {r.ErrorCode} time {sw.ElapsedTicks/factor}us");
-                  System.Threading.Thread.Sleep(o.sleep);
-               }
-            }
-         }
-         return 3;
-      }
-   }
-
-}
+namespace Spark.Console {
+   using bts;
+   using bts.api;
+   using bts.api.core;
+   using bts.api.orderman;
+   using bts.co2;
+   using bts.config;
+   using bts.test;
+
+   using mbts.co2;
+   using mbts.config;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Diagnostics;
+   using System.Linq;
+   using System.Threading;
+
+   static class OrderStressCommand {
+      public static int run(OrderStressOptions o) {
+         var ctx = new ServiceContext();
+         ConsoleEx.WriteLine("Please log in to Spark", ConsoleColor.Yellow);
+         var name = ConsoleEx.prompt("Username: ", ConsoleColor.Yellow);
+         var pw = ConsoleEx.prompt("Password: ", ConsoleColor.Yellow);
+
+         var login_result = ctx.login(name, pw);
+         if (!login_result.Result.success) {
+            ConsoleEx.WriteLine("Invalid Credentials", ConsoleColor.Red);
+         }
+
+         if (Configuration.get<DeploymentType>().Type.ToUpper() == "PROD") {
+            ConsoleEx.WriteLine("Refusing to run on production system", ConsoleColor.Red);
+            return -1;
+         }
+
+         var orders = ctx.locate<OrderService>();
+         var factor = TimeSpan.TicksPerMillisecond / 1000;
+         
+         string fmt_order(Market m, Response r, long elapsed_ticks) =>
+            $"Order {m.security.security_desc()}. id: {r.OrderKey}; err: {r.ErrorCode}; " +
+            $"time: {elapsed_ticks / factor}us";
+         
+         foreach (var m in _all_markets(ctx)) {
+            ConsoleEx.WriteLine(m.security.security_desc(), ConsoleColor.White);
+            var tick = m.security.ContractSpec.LowTick;
+            if (m.ask.HasValue) {
+               for (var x = 0; x < o.depth; ++x) {
+                  var price = m.ask.Value.price() + (tick * x);
+                  ConsoleEx.WriteLine($"Sell {o.quantity} at {price}", ConsoleColor.Red);
+                  var sw = new Stopwatch();
+                  sw.Start();
+                  var r = orders.create_order(_mkorder(m.security.Id, price, o.quantity, Side.Sell));
+                  sw.Stop();
+                  Log.Api.info(fmt_order(m, r, sw.ElapsedTicks));
+                  Thread.Sleep(o.sleep);
+               }
+            }
+            if (m.bid.HasValue) {
+               // start at bid and place 1 order at 10 levels
+               for (var x = 0; x < o.depth; ++x) {
+                  var price = m.bid.Value.price() - (tick * x);
+                  ConsoleEx.WriteLine($"Buy {o.quantity} at {price}", ConsoleColor.Blue);
+                  var sw = new Stopwatch();
+                  sw.Start();
+                  var r = orders.create_order(_mkorder(m.security.Id, price, o.quantity, Side.Buy));
+                  sw.Stop();
+                  Log.Api.info(fmt_order(m, r, sw.ElapsedTicks));
+                  Thread.Sleep(o.sleep);
+               }
+            }
+         }
+         return 3;
+      }
+
+      static IEnumerable<Market> _all_markets(ServiceContext ctx) {
+         var channels =
+            UserSymbol.get_all()
+            .Select(u =>
+               new SecurityChannel() {
+                  Time = 0,
+                  Flags = (int)(MDRequest.Types.Flags.Direct | MDRequest.Types.Flags.Implied),
+                  Type = u.type,
+                  Symbol = u.symbol,
+                  Exchange = u.exchange
+               }
+            );
+         var request = new MDRequest() { Host = "system_test" };
+         request.Channels.AddRange(channels);
+
+         var relay = ctx.locate<MarketDataRelay>();
+         Thread.Sleep(100);
+         var result = relay.requestmd(request);
+
+         foreach (var ch in result.Channels) {
+            // if (ch.Platform != Platform.Sim) continue;
+            foreach (var update in ch.Updates) {
+               if (update.Security != null) {
+                  var m = new Market() {
+                     security = new SecurityImmutable(update.Security),
+                     bid = _best(
+                        (a, b) => a > b ? a : b,
+                        update.Direct.Bids.FirstOrDefault(),
+                        update.Implied.Bids.FirstOrDefault()),
+                     ask = _best(
+                        (a, b) => a < b ? a : b,
+                        update.Direct.Asks.FirstOrDefault(),
+                        update.Implied.Asks.FirstOrDefault())
+                  };
+                  if (m.bid.HasValue || m.ask.HasValue) {
+                     yield return m;
+                  }
+               }
+            }
+         }
+      }
+
+      static IntPrice? _best(
+         Func<IntPrice, IntPrice, IntPrice> comp, params PriceSizeMessage?[] ps) {
+         IntPrice? result = null;
+         foreach (var p in ps) {
+            if (p != null) {
+               var ip = new IntPrice(p.Mantissa, (sbyte)p.Exponent);
+               if (result == null) {
+                  result = ip;
+               } else {
+                  result = comp(result.Value, ip);
+               }
+            }
+         }
+         return result;
+      }
+
+      static Create _mkorder(uint security, double price, uint size, Side side) {
+         return new Create {
+            Price = price,
+            Quantity = size,
+            NewOrder = new OrderDefinition {
+               SecurityKey = security,
+               Side = side,
+               TimeInForce = TimeInForce.Day,
+               ManualOrderIndicator = ManualOrderIndicator.Yes,
+               CtiCode = CTICode.CtiCode1,
+               CustomerOrFirm = CustomerOrFirm.Firm,
+               AccountId = 1,
+               UserId = 1,
+               CustomDef = new CustomOrderDefinition {
+                  Limit = new LimitOrderDefinition()
+               }
+            }
+         };
+      }
+
+      struct Market {
+         public SecurityImmutable security;
+         public IntPrice? bid;
+         public IntPrice? ask;
+      }
+   }
+}

src/client/Spark.Console/src/Commands/OrderStressCommandOptions.cs

@@ -0,0 +1,17 @@
+namespace Spark.Console {
+   using CommandLine;
+
+   [Verb("order-stress", HelpText = "Create a bunch of orders")]
+   public class OrderStressOptions {
+      // Maybe options to create fewer orders or sleep
+
+      [Option('s', "sleep", HelpText = "Sleep ms between order submission", Default = 100)]
+      public int sleep { get; set; }
+
+      [Option('d', "depth", HelpText = "How many orders per side", Default = 10u)]
+      public uint depth { get; set; }
+
+      [Option('q', "quanityt", HelpText = "Order quantity", Default = 1u)]
+      public uint quantity { get; set; }
+   };
+}

src/client/Spark.Console/src/Commands/ScriptCommand.cs

@@ -0,0 +1,110 @@
+namespace Spark.Console {
+   using bts.api.core;
+   using bts.api.diagnostic;
+
+   using System;
+   using System.IO;
+   using System.Reflection;
+   using System.Reactive.Linq;
+   using Spark.Core;
+   using System.Threading;
+
+   static class ScriptCommand {
+      public static int run(ScriptCommandOptions o) {
+         if (o.server) {
+            _run_server();
+            return 0;
+         }
+         if (o.host is null) {
+            Console.WriteLine("Please specify host");
+            return 1;
+         }
+         if (!o.file.is_null_or_empty() && File.Exists(o.file)) {
+            _run_client(o.host, File.OpenRead(o.file));
+            return 0;
+         }
+         _run_client(o.host, Console.OpenStandardInput());
+         return 0;
+      }
+
+      static void _run_client(string qualifier, Stream stream) {
+         using var ctx = new ServiceContext();
+         ShellDiagnosticService diag = ctx.locate<ShellDiagnosticService>(qualifier);
+         var script_id = diag.create(null);
+         ShellScriptHost script = ctx.locate<ShellScriptHost>(script_id.Id);
+         try {
+            using var sub = script.subscribei_output().Subscribe(_script_output);
+            using var reader = new StreamReader(stream);
+            var status = ScriptResult.Complete;
+
+            for (var line = string.Empty; line != "#q"; line = prompt_input()) {
+               var result = script.run(new ScriptLines { Lines = { line } });
+               status = result.Res;
+               switch (status) {
+                  case ScriptResult.Invalid:
+                     Console.WriteLine($"Syntax error: '{result.Message}'");
+                     break;
+                  case ScriptResult.RunError:
+                     Console.WriteLine($"Exception: {result.Message}");
+                     break;
+                  case ScriptResult.Complete:
+                     if (!result.Message.is_null_or_empty()) {
+                        Console.WriteLine(result.Message);
+                     }
+                     break;
+               }
+            }
+
+            string prompt_input() {
+               _write_prompt(status);
+               return reader.ReadLine() ??
+                  throw new InvalidOperationException("Received null input");
+            }
+         } catch (Exception e) {
+            Console.WriteLine($"Done. {e.Message}");
+         } finally {
+            try {
+               script.dispose(null);
+               script.Dispose();
+            } catch (Exception) { }
+         }
+      }
+
+      static void _script_output(Update<ScriptLinesImmutable, int> update) {
+         foreach (var lines in update.added) {
+            foreach (var line in lines.Lines) {
+               Console.WriteLine(line);
+            }
+         }
+      }
+
+      static void _write_prompt(ScriptResult status) {
+         switch (status) {
+            case ScriptResult.Complete:
+               Console.Write(">>> ");
+               break;
+            case ScriptResult.Incomplete:
+               Console.Write("... ");
+               break;
+            case ScriptResult.Invalid:
+               Console.Write("?>> ");
+               break;
+            case ScriptResult.RunError:
+               Console.Write("!>> ");
+               break;
+         }
+      }
+
+      static void _run_server() {
+         // for debugging
+         using var ctx = new ServiceContext();
+         var stype = SparkReflectionHelper.find_type(
+            "bts.api.diagnostic.ShellDiagnosticsServiceServer");
+         var server = Activator.CreateInstance(stype, ctx);
+         while (true) {
+            Thread.Sleep(1000);
+            GC.KeepAlive(server);
+         }
+      }
+   }
+}

src/client/Spark.Console/src/Commands/ScriptCommandOptions.cs

@@ -0,0 +1,15 @@
+namespace Spark.Console {
+   using CommandLine;
+
+   [Verb("script", HelpText = "Remote control for shell diagnostic host")]
+   public class ScriptCommandOptions {
+      [Option('s', "server", HelpText = "Run a script server for testing.")]
+      public bool server { get; set; }
+
+      [Option('h', "host", HelpText = "Connect to {host}")]
+      public string? host { get; set; }
+
+      [Option('f', "fromfile", HelpText = "Read commands from file instead of stdin")]
+      public string? file { get; set; }
+   }
+}

src/client/Spark.Console/src/Commands/SettingsCommand.cs

@@ -0,0 +1,16 @@
+namespace Spark.Console {
+   using bts;
+
+   using System;
+   using System.Reflection;
+
+   class SettingsCommand {
+      public static int run(SettingsCommandOptions _) {
+         var type = SparkReflectionHelper.find_type(
+            "bts.api.diagnostic.ShellDiagnosticsServiceServer");
+         var run = type.GetMethod("run", BindingFlags.Static | BindingFlags.Public);
+         Log.notnull(run).Invoke(null, Array.Empty<object>());
+         return 0;
+      }
+   }
+}

src/client/Spark.Console/src/Commands/SettingsCommandOptions.cs

@@ -0,0 +1,6 @@
+namespace Spark.Console {
+   using CommandLine;
+
+   [Verb("settings", HelpText = "Copy workspaces around")]
+   class SettingsCommandOptions { }
+}

src/client/Spark.Console/src/Commands/TailCommand.cs

@@ -0,0 +1,95 @@
+namespace Spark.Console {
+   using bts.api.core;
+   using bts.test;
+
+   using Newtonsoft.Json;
+
+   using System;
+   using System.Linq;
+   using System.Threading;
+
+   class TailService {
+      static PrintFormat _format = PrintFormat.ToString;
+      static bool _timestamp = false;
+
+      enum PrintFormat {
+         ToString,
+         Json
+      }
+
+      public static int run(TailServiceOptions o) {
+         var split = o.method?.Split('.');
+         if (split?.Length != 2) {
+            Console.WriteLine("Try --method [ServiceName].[Method]");
+            return -1;
+         }
+
+         if (o.json) {
+            _format = PrintFormat.Json;
+         }
+         if (o.time) {
+            _timestamp = true;
+         }
+
+         _tail(split[0], split[1]);
+         return 0;
+      }
+
+      class SubscriptionLogger<T> : IObserver<T> {
+         public void OnCompleted() =>
+            ConsoleEx.WriteLine("Completed.", ConsoleColor.Yellow);
+
+         public void OnError(Exception error) =>
+            ConsoleEx.WriteLine($"Error. '{error}'", ConsoleColor.Red);
+
+         public void OnNext(T value) {
+            if (_timestamp) {
+               Console.Write(DateTime.Now.ToString("HH:mm:ss.fff"));
+               Console.Write(" ");
+            }
+            if (value is null) {
+               return;
+            }
+
+            switch (_format) {
+               case PrintFormat.Json:
+                  dynamic dv = value;
+                  var json = JsonConvert.SerializeObject(new { dv.added, dv.removed, dv.changed });
+                  Console.WriteLine(json);
+                  break;
+               case PrintFormat.ToString:
+                  Console.WriteLine(value);
+                  break;
+            }
+         }
+      }
+
+      static void _tail(string service_name, string method_name) {
+         var service_type = SparkReflectionHelper.find_service_type(service_name);
+         var subscribe_method = service_type.GetMethod("subscribei_" + method_name) ??
+            throw new InvalidOperationException();
+         using var ctx = new ServiceContext();
+         var locate = typeof(ServiceContext).GetMethod("locate")?.MakeGenericMethod(service_type) ??
+            throw new InvalidOperationException();
+         var service = locate.Invoke(ctx, new[] { string.Empty });
+         // Subscribe with null qualifier
+         object observable = subscribe_method.Invoke(service, new object?[] { null })
+            ?? throw new InvalidOperationException();
+         Type update_type = observable.GetType().GetGenericArguments()
+            .First(arg => arg.Name.StartsWith("Update"));
+         Type observer_type = typeof(SubscriptionLogger<>).MakeGenericType(update_type);
+         var observer = Activator.CreateInstance(observer_type)
+            ?? throw new InvalidOperationException(
+               $"Activator failed to instantiate observer of type '{observer_type}'");
+
+         var subscribe = observable.GetType().GetMethod("Subscribe", new[] { observer_type }) ??
+            throw new InvalidOperationException(
+               "Failed to find 'IObservable.Subscribe' method via reflection");
+         var dispose = subscribe.Invoke(observable, new[] { observer });
+
+         while (true) {
+            Thread.Sleep(1000);
+         }
+      }
+   }
+}