refactor: Replace if/else with switch cases

We don't need explicit subcommands to 'run onixs' md (just use config).

.config/dotnet-tools.json

@@ -25,6 +25,12 @@
       "commands": [
         "fake"
       ]
+    },
+    "dotnet-t4": {
+      "version": "3.0.0",
+      "commands": [
+        "t4"
+      ]
     }
   }
 }

.editorconfig

@@ -207,17 +207,23 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
 
+# local symbols
+dotnet_naming_symbols.local_symbols.applicable_kinds = local
+
+# parameter symbols
+dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
+
 # Naming styles
 
-dotnet_naming_style.snake_case.required_prefix =
-dotnet_naming_style.snake_case.required_suffix =
-dotnet_naming_style.snake_case.word_separator = _
-dotnet_naming_style.snake_case.capitalization = all_lower
+dotnet_naming_style.pascal_case.required_prefix =
+dotnet_naming_style.pascal_case.required_suffix =
+dotnet_naming_style.pascal_case.word_separator =
+dotnet_naming_style.pascal_case.capitalization = pascal_case
 
-dotnet_naming_style.private_snake_case.required_prefix = _
-dotnet_naming_style.private_snake_case.required_suffix =
-dotnet_naming_style.private_snake_case.word_separator = _
-dotnet_naming_style.private_snake_case.capitalization = all_lower
+dotnet_naming_style.camel_case.required_prefix =
+dotnet_naming_style.camel_case.required_suffix =
+dotnet_naming_style.camel_case.word_separator =
+dotnet_naming_style.camel_case.capitalization = camel_case
 
 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -229,6 +235,10 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case
 
+dotnet_naming_style.local.capitalization = camel_case
+
+dotnet_naming_style.parameter.capitalization = camel_case
+
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -238,14 +248,18 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion
 
-dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
-dotnet_naming_rule.public_members_snake_case.style = snake_case
-dotnet_naming_rule.public_members_snake_case.severity = suggestion
+dotnet_naming_rule.public_members.symbols = public_symbols
+dotnet_naming_rule.public_members.style = pascal_case
+dotnet_naming_rule.public_members.severity = suggestion
 
-dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
-dotnet_naming_rule.private_members_snake_case.style = private_snake_case
-dotnet_naming_rule.private_members_snake_case.severity = suggestion
+dotnet_naming_rule.private_members.symbols = private_symbols
+dotnet_naming_rule.private_members.style = camel_case
+dotnet_naming_rule.private_members.severity = suggestion
 
-dotnet_naming_rule.parameters.symbols = parameters
-dotnet_naming_rule.parameters.style = snake_case
+dotnet_naming_rule.variables_are_camel_case.severity = suggestion
+dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
+dotnet_naming_rule.variables_are_camel_case.style = local
+
+dotnet_naming_rule.parameters.symbols = parameter_symbols
+dotnet_naming_rule.parameters.style = parameter
 dotnet_naming_rule.parameters.severity = suggestion

.vscode/launch.json

@@ -14,6 +14,68 @@
       "type": "coreclr",
       "request": "attach"
     },
+    {
+      "name": "Api Server",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "preLaunchTask": "Build Api Server (Debug)",
+        "program": "spark_api_server.exe",
+        "args": [
+          "run",
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
+        ],
+        "cwd": "build/Debug/bin/net7.0",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Api Server (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "program": "spark_api_server.exe",
+        "args": [
+          "run",
+          "--sparkd",
+          "http://127.0.0.1:8000/config"
+        ],
+        "cwd": "build/Debug/bin/net7.0",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Bts.DataStreamWebApp.Server",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream WebApp (Debug)",
+      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+      "env": {
+        "ASPNETCORE_HTTP_PORT": "8080",
+        "ASPNETCORE_HTTPS_PORT": "8443",
+        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    {
+      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+      "env": {
+        "ASPNETCORE_HTTP_PORT": "8080",
+        "ASPNETCORE_HTTPS_PORT": "8443",
+        "ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
     {
       "name": "DataStreamPublisher",
       "type": "coreclr",
@@ -26,11 +88,28 @@
       "windows": {
         "program": "DataStreamPublisher.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
-      ],
       "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DataStreamPublisher (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "DataStreamPublisher",
+      },
+      "windows": {
+        "program": "DataStreamPublisher.exe",
+      },
+      "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -82,7 +161,7 @@
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
@@ -110,11 +189,28 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
       "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "PubSubServer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "PubSubServer",
+      },
+      "windows": {
+        "program": "PubSubServer.exe",
+      },
+      "cwd": "build/Debug/bin/net7.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -252,26 +348,41 @@
   ],
   "inputs": [
     {
-      "id": "dataStreamPublisherConfig",
+      "id": "dataStreamPublisherEnvironment",
       "type": "pickString",
       "description": "Publisher Configuration",
       "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
+        "Development",
+        "MarketData",
+        "Production",
+        "ProductionAws",
+        "Staging",
       ],
       "default": "MarketDataPublisher",
     },
     {
-      "id": "pubSubServerConfig",
+      "id": "pubSubServerEnvironment",
       "type": "pickString",
       "description": "PubSub Configuration",
       "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
+        "Development",
+        "EchoServer",
+        "Production",
+        "ProductionAws",
+        "Staging",
       ],
-      "default": "PubSubServer",
+      "default": "Development",
+    },
+    {
+      "id": "dataStreamWebAppServerEnvironment",
+      "type": "pickString",
+      "description": "DataStream WebApp Environment",
+      "options": [
+        "Development",
+        "Production",
+        "ProductionAws",
+        "Staging",
+      ]
     },
     {
       "id": "sparkConfigServer",
@@ -289,5 +400,15 @@
       "type": "promptString",
       "description": "Test to run",
     },
+    {
+      "id": "isSparkApiEnabled",
+      "type": "pickString",
+      "description": "Enable Spark Api (connect to sparkd)",
+      "options": [
+        "True",
+        "False",
+      ],
+      "default": "True",
+    }
   ]
 }

.vscode/tasks.json

@@ -121,6 +121,117 @@
         "color": "terminal.ansiCyan"
       },
     },
+    {
+      "label": "Build DataStreamQueryAnalyzer",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
     {
       "label": "Build Spark",
       "group": "build",
@@ -303,6 +414,22 @@
         "color": "terminal.ansiCyan"
       },
     },
+    {
+      "label": "Publish DataStreamWebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/publish_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/publish_data_stream_web_app.cmd",
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
     {
       "label": "Edit DataStream Config",
       "type": "process",
@@ -353,7 +480,7 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStream",
+      "dependsOn": "Build DataStreamQueryAnalyzer",
       "type": "process",
       "command": "",
       "windows": {
@@ -373,7 +500,7 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
       "type": "process",
       "command": "",
       "windows": {
@@ -393,7 +520,7 @@
     },
     {
       "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStream (Release)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
       "type": "process",
       "command": "",
       "windows": {
@@ -480,6 +607,89 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run DataStream WebApp (Debug)",
+      "dependsOn": "Build DataStream WebApp (Debug)",
+      "command": "",
+      "linux": {
+        "command": "Bts.DataStreamWebApp.Server",
+      },
+      "windows": {
+        "command": "Bts.DataStreamWebApp.Server.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+        "env": {
+          "ASPNETCORE_HTTP_PORT": "8080",
+          "ASPNETCORE_HTTPS_PORT": "8443",
+          "ASPNETCORE_ENVIRONMENT": "Development",
+          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStream WebApp (Release)",
+      "dependsOn": "Build DataStream WebApp (Release)",
+      "command": "",
+      "linux": {
+        "command": "Bts.DataStreamWebApp.Server",
+      },
+      "windows": {
+        "command": "Bts.DataStreamWebApp.Server.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0",
+        "env": {
+          "ASPNETCORE_HTTP_PORT": "8080",
+          "ASPNETCORE_HTTPS_PORT": "8443",
+          "ASPNETCORE_ENVIRONMENT": "Production",
+          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStream WebApp (Release | Skip Build)",
+      "command": "",
+      "linux": {
+        "command": "Bts.DataStreamWebApp.Server",
+      },
+      "windows": {
+        "command": "Bts.DataStreamWebApp.Server.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0",
+        "env": {
+          "ASPNETCORE_HTTP_PORT": "8080",
+          "ASPNETCORE_HTTPS_PORT": "8443",
+          "ASPNETCORE_ENVIRONMENT": "Production",
+          "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+        }
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run PubSubServer",
       "dependsOn": "Build DataStream",

CMakeLists.txt

@@ -21,7 +21,7 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
+set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
 set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
 set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
 set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
@@ -40,9 +40,8 @@ include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include
 link_directories(${EXTERNALS_LIB})
 
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
-if(DEFINED APP_DATA_DIR_OVERRIDE)
-  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
+if(DEFINED APP_DATA_DIR)
+  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)

CMakePresets.json

@@ -20,8 +20,6 @@
         "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
       },
       "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
         "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
         "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
         "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
@@ -36,8 +34,7 @@
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "spark"
+        }
       }
     },
 
@@ -53,8 +50,6 @@
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
         "platform": "",
-        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
         "CMAKE_BUILD_WITH_INSTALL_RPATH": {
           "type": "BOOL",
           "value": "OFF"
@@ -77,8 +72,7 @@
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "sparkd"
+        }
       }
     },
 
@@ -159,11 +153,7 @@
     {
       "name": "spark-api-server-windows",
       "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)",
-      "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
-        "SHARED_MEMORY_PREFIX": "spark_api_server"
-      }
+      "displayName": "Spark Api Server (Windows)"
     },
 
     {
@@ -172,7 +162,6 @@
       "displayName": "Spark Api Server (Linux)",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
@@ -191,7 +180,7 @@
           "value": "ON"
         },
         "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net8.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
@@ -229,10 +218,8 @@
       "description": "Data Stream utils ported over from Edge",
       "binaryDir": "${sourceDir}/build/${presetName}",
       "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
         "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
-        "SHARED_MEMORY_PREFIX": "data_stream"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
       }
     },
 
@@ -248,8 +235,6 @@
       "inherits": "sparkd",
       "generator": "Ninja Multi-Config",
       "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
         "BUILD_ADMIN_WEB_APP": {
           "type": "BOOL",
           "value": "OFF"
@@ -271,8 +256,7 @@
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "data_stream"
+        }
       }
     }
   ],

cmake/Protobuf.cmake

@@ -29,36 +29,43 @@ function(Protobuf_Generate_Cpp)
     list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
 
     add_custom_command(
-      OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-             "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
       COMMAND
-        ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+        ${Protobuf_PROTOC_EXECUTABLE}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
       DEPENDS ${ABS_FIL}
       COMMENT "Running C++ protocol buffer compiler on ${FIL}"
-      VERBATIM)
+      VERBATIM
+    )
   endforeach()
 
-  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+  set_source_files_properties(
+    ${${_ARGS_CXX_SOURCES}}
+    ${${_ARGS_CXX_HEADERS}}
+    PROPERTIES
+      GENERATED TRUE
+  )
 
   # remove some warnings from protobuf generated files
   if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
-                                                      "/wd4309 /wd4838")
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
   else()
     set_source_files_properties(
       ${${_ARGS_CXX_SOURCES}}
       PROPERTIES
-        COMPILE_FLAGS
-        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
     )
   endif()
 
-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
-      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
-      PARENT_SCOPE)
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
 endfunction()
 
 function(Service_Compiler_Generate_Cpp)
@@ -92,40 +99,51 @@ function(Service_Compiler_Generate_Cpp)
     list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
 
     set(TEMPLATES
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
-        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+      ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
     )
 
     set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
     if(WIN32)
       set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
       add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
         COMMAND SET PYTHONPATH=${SPARK_PYTHON}
         COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
         COMMAND
-          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
-          --plugin=protoc-gen-service=${_plugin_cmd}
-          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
-          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+          ${Protobuf_PROTOC_EXECUTABLE}
+          ARGS
+            --cpp_out=${_ARGS_CXX_OUT_DIR}
+            --plugin=protoc-gen-service=${_plugin_cmd}
+            --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
+            -I ${PROTOBUF_INCLUDE_DIRS}
+            -I ${_protobuf_src_dir}
+            ${ABS_FIL}
+        DEPENDS
+          ${ABS_FIL}
+          ${TEMPLATES}
+          ${_plugin_cmd}
+          ${_plugin_py}
         COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM)
+        VERBATIM
+      )
     else()
       add_custom_command(
-        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
-               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        OUTPUT
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+          "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
         COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
         ARGS
           ${Protobuf_PROTOC_EXECUTABLE}
@@ -133,32 +151,32 @@ function(Service_Compiler_Generate_Cpp)
           --plugin=protoc-gen-service=${_plugin_py}
           --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
           ${_proto_include_cli_args} ${ABS_FIL}
-        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+        DEPENDS
+          ${ABS_FIL}
+          ${TEMPLATES}
+          ${_plugin_py}
         COMMENT "Running C++ service compiler on ${FIL}"
-        VERBATIM)
+        VERBATIM
+      )
     endif()
-
   endforeach()
 
   # remove some warnings from protobuf generated files
   if(WIN32)
-    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
-                                                      "/wd4309 /wd4838")
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS "/wd4309 /wd4838"
+    )
   else()
     set_source_files_properties(
       ${${_ARGS_CXX_SOURCES}}
       PROPERTIES
-        COMPILE_FLAGS
-        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
     )
   endif()
 
-  set(${_ARGS_CXX_SOURCES}
-      ${${_ARGS_CXX_SOURCES}}
-      PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS}
-      ${${_ARGS_CXX_HEADERS}}
-      PARENT_SCOPE)
-
+  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
 endfunction()
 

compose.yaml

@@ -109,6 +109,57 @@ services:
         RUN apt-get update && apt-get -y install iputils-ping
     command: [ "ping", "sparkd" ]
 
+  pubsub:
+    environment:
+      - DOTNET_ENVIRONMENT=Staging
+    build:
+      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
+    ports:
+      - 22019:22019
+    command: [ "./PubSubServer" ]
+
+  md-publisher:
+    build:
+      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
+    depends_on:
+      - pubsub
+      - sparkd
+    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
+    environment:
+      - DOTNET_ENVIRONMENT=MarketData
+
+  webapp:
+    ports:
+      - 8080:8080
+      - 8443:8443
+    build:
+      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
+      args:
+        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
+        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
+        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
+    command: [ "./Bts.DataStreamWebApp.Server" ]
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+
+  api-server:
+    depends_on:
+      - sparkd
+    ports:
+      - 8008:8008
+      - 9443:9443
+    build:
+      dockerfile: src/client/spark_api_server/Dockerfile
+    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
+
 volumes:
   sparkd_data:
     name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"

docker/ubuntu-22.04/dotnet-7/Dockerfile

@@ -0,0 +1,39 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+  && apt-get install -y pkg-config \
+  && apt-get install -y locales \
+  && apt-get install -y ca-certificates \
+  && apt-get install -y libssh2-1-dev \
+  && apt-get install -y software-properties-common \
+  && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

src/DataStreamSuite/ArithmeticExpressions/ArithmeticExpressionGrammar.y

@@ -8,7 +8,6 @@
    public int Integer;
    public double Real;
    public string Identifier;
-   public DateTime DateTime; 
    public ImmutableList<IArithmeticExpression> Arguments;
    public ImmutableList<Assignment> Assignments;
    public IArithmeticExpression Argument;

src/DataStreamSuite/BlueRegistry/DoubleRegistryHandler.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Runtime.Versioning;
+
 using Microsoft.Win32;
 
 namespace BCG
@@ -10,6 +12,7 @@ namespace BCG
          rk.SetValue (name, value.ToString ());
       }
 
+      [SupportedOSPlatform("windows")]
       public static double getValue (RegistryKey rk, string name)
       {
          object v = rk.GetValue (name);

src/DataStreamSuite/BooleanExpressions/BooleanExpression.cs

@@ -1,19 +1,11 @@
 namespace Bts.BooleanExpressions;
 
-using bts;
-
 class BooleanExpression : IBooleanExpression {
    readonly Func<IDictionary<string, object?>, bool> evaluate;
 
    public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
       this.evaluate = evaluate;
 
-   public bool Evaluate(IDictionary<string, object?> env) {
-      try {
-         return evaluate(env);
-      } catch (Exception ex) {
-         Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
-         throw;
-      }
-   }
+   public bool Evaluate(IDictionary<string, object?> env) =>
+      evaluate(env);
 }

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -47,11 +47,6 @@ IsOfType                 '/'
 
 %%
 
-{StringConstant} {
-   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
-   return (int)Token.STRING_CONSTANT;
-} 
-
 {BooleanConstant} {
    yylval.Boolean = yytext.ToLower() switch {
       "true" => true,
@@ -62,6 +57,11 @@ IsOfType                 '/'
    return (int)Token.BOOLEAN_CONSTANT;
 }
 
+{StringConstant} {
+   yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
+   return (int)Token.STRING_CONSTANT;
+} 
+
 {NullConstant} {
    yylval.Null = DBNull.Value;
    return (int)Token.NULL_CONSTANT;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.env.development

@@ -0,0 +1 @@
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.env.production

@@ -0,0 +1 @@
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -0,0 +1,20 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.3.1' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

src/DataStreamSuite/Bts.DataStreamWebApp.Client/Bts.DataStreamWebApp.Client.esproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
+  <PropertyGroup>
+    <StartupCommand>npm run dev</StartupCommand>
+    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
+    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
+    <!-- Allows the build (or compile) script located on package.json to run on Build -->
+    <ShouldRunBuildScript>true</ShouldRunBuildScript>
+    <!-- Folder where production build objects will be placed -->
+    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
+  </PropertyGroup>
+</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/README.md

@@ -0,0 +1,8 @@
+# React + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh

src/DataStreamSuite/Bts.DataStreamWebApp.Client/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/ico" href="/logo.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>BTS Cloud Streams</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/nuget.config

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "bts.datastreamwebapp.client",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@angular/material": "^18.0.4",
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@fontsource/noto-sans": "^5.0.22",
+    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
+    "@mui/icons-material": "^5.15.20",
+    "@mui/material": "^5.15.20",
+    "@mui/x-data-grid": "^7.7.1",
+    "bulma": "^1.0.1",
+    "classnames": "^2.5.1",
+    "react": "^18.3.1",
+    "react-data-grid": "^7.0.0-beta.44",
+    "react-data-table-component": "^7.6.2",
+    "react-dom": "^18.3.1",
+    "react-json-view-lite": "^1.4.0",
+    "react-pro-sidebar": "^1.1.0",
+    "react-use-websocket": "^4.8.1",
+    "rpc-websockets": "^7.11.2"
+  },
+  "devDependencies": {
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^8.57.0",
+    "eslint-plugin-react": "^7.34.2",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-react-refresh": "^0.4.7",
+    "react-mui-sidebar": "^1.3.5",
+    "vite": "^5.2.13"
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/public/vite.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/AnimatedSideBar.jsx

@@ -0,0 +1,40 @@
+import { Sidebar, Menu, MenuItem, Submenu, Logo } from "react-mui-sidebar";
+import './App.css';
+
+const AnimatedSideBar = () => {
+  return (
+    <Sidebar className={"sidebar-container"} width={"270px"}>
+      <Logo img="https://adminmart.com/wp-content/uploads/2024/03/logo-admin-mart-news.png">
+        AdminMart
+      </Logo>
+      <Menu subHeading="HOME">
+        <MenuItem link="/" badge="true">
+          Modern
+        </MenuItem>
+        <MenuItem>eCommerce</MenuItem>
+        <MenuItem>Analytical</MenuItem>
+      </Menu>
+      <Menu subHeading="APPS">
+        <MenuItem>Chat</MenuItem>
+        <MenuItem>Calendar</MenuItem>
+      </Menu>
+      <Menu subHeading="OTHERS">
+        <Submenu title="Menu Level">
+          <MenuItem>Post</MenuItem>
+          <MenuItem>Details</MenuItem>
+          <Submenu title="Level 2">
+            <MenuItem>new</MenuItem>
+            <MenuItem>Hello</MenuItem>
+          </Submenu>
+        </Submenu>
+
+        <MenuItem>Chip</MenuItem>
+        <MenuItem target="_blank" link="google.com">
+          External Link
+        </MenuItem>
+      </Menu>
+    </Sidebar>
+  );
+};
+
+export default AnimatedSideBar;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.css

@@ -0,0 +1,101 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+.root-div {
+  display: flex;
+  flex-direction: row;
+  align-items: flex-start;
+}
+
+.main-content-div {
+  display: flex;
+  flex-direction: row;
+}
+
+.title-div {
+  width: 100%;
+}
+
+.container {
+  display: flex;
+}
+
+.container-column {
+  display: flex;
+  flex-direction: column;
+}
+
+.container-row {
+  display: flex;
+  flex-direction: row;
+  align-items: stretch;
+}
+
+box.query-input-form {
+   /* p: 2, display: "flex", flexFlow: "column", alignItems: "stretch"  */
+  display: flex;
+  flex-flow: column;
+  align-items: stretch;
+}
+
+/* li.ps-menuitem-root:hover {
+  background-color: #ff9922;  
+} */
+
+.container-row-centered {
+  display: flex;
+  flex-direction: row;
+  justify-content: center;
+  align-items: center;
+}
+
+h1#main-header {
+  padding: 4px 8px 4px 8px;
+  text-align: center;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}
+
+/* Sidebar.css */
+/* .sidebar-container {
+  display: flex;
+  flex-direction: column;
+}
+
+.sidebar {
+  position: relative;
+  top: 0;
+  left: 0;
+  height: 100vh;
+  background-color: #333;
+  color: white;
+  overflow: hidden;
+  transition: transform 0.3s ease;
+}
+
+.sidebar.closed {
+  width: 0;
+}
+
+.sidebar.open {
+  width: 250px;
+} */

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/App.jsx

@@ -0,0 +1,173 @@
+import { ThemeProvider, CssBaseline, GlobalStyles } from '@mui/material';
+import QueryStatusView from './QueryStatusView';
+import useMediaQuery from '@mui/material/useMediaQuery';
+import QueryControl from './QueryControl';
+import DataTable from './DataTable';
+import { useState, useEffect, useMemo } from 'react';
+import useWebSocket, { ReadyState } from 'react-use-websocket';
+import Sidebar from './SideBar';
+import getTheme from './Theme';
+
+import '@fontsource/noto-sans';
+import './App.css';
+import './SideBar.css';
+
+const webSocketUrl = import.meta.env.VITE_DATASTREAM_WEBSOCKET_URL;
+// Always use the same subscription id (1), because we only show 1 table at a time.
+// The server will automatically dispose the previous subscription and start a new one
+// if we're currently subscribed to another query.
+const subscriptionId = "1";
+
+var rpcReqIdCounter = 1;
+
+function subscribeReq(query) {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "subscribe",
+    params: {
+      subscription_id: subscriptionId,
+      query: query
+    }
+  };
+}
+
+function unsubscribeReq() {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "unsubscribe",
+    params: {
+      subscription_id: subscriptionId,
+    }
+  };
+}
+
+const App = () => {
+  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
+  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
+  const globalStyles = {
+    ':root': {
+      '--primary-color': theme.palette.primary.main,
+      '--primary-dark-color': theme.palette.primary.dark,
+      '--secondary-color': theme.palette.secondary.main,
+      '--secondary-dark-color': theme.palette.secondary.dark,
+      '--background-color': theme.palette.background.default,
+    },
+  };
+  const [blueTable, setBlueTable] = useState({ columns: null, rows: null });
+  const [queryError, setQueryError] = useState(null);
+  const [queryState, setQueryState] = useState("Unsubscribed");
+  const queryStateDetails = useMemo(() => {
+    if (queryState == "Unsubscribed") {
+      return "Enter query above, then click the play button to subscribe";
+    }
+    if (queryState == "Subscribed") {
+      return null;
+    }
+  }, [queryState]);
+  const [sidebarIsOpen, setSidebarIsOpen] = useState(false);
+  const [cannedQuery, setCannedQuery] = useState(null);
+
+  const onCannedQueryClicked = (cannedQuery) => {
+    console.info(`Executing canned query: ${JSON.stringify(cannedQuery)}`);
+    const queryText = cannedQuery.query.join("\n");
+    setCannedQuery({ queryText: queryText, clickTime: Date.now() });
+    subscribe(queryText);
+  };
+
+  const { sendJsonMessage, getWebSocket, readyState } = useWebSocket(
+    webSocketUrl,
+    {
+      onOpen: (e) => console.info(`WebSocket open. ${JSON.stringify(e)}`),
+      onClose: (e) => console.info(`WebSocket closed. ${JSON.stringify(e)}`),
+      onError: (e) => console.error(`WebSocket error: ${JSON.stringify(e)}`),
+      shouldReconnect: () => true,
+    });
+
+  useEffect(() => {
+    console.info(`WebSocket readyState: ${readyState}`)
+    if (readyState != ReadyState.OPEN) {
+      return;
+    }
+    const ws = getWebSocket();
+    ws.onmessage = e => {
+      console.info(`Received message: ${e.data}`);
+      const msg = JSON.parse(e.data);
+      setQueryError(msg.error);
+      if (msg.error != null) {
+        setQueryState("Error");
+      } else if (msg.id != null) {
+        setQueryState(msg.result.status);
+      } else {
+        console.assert(msg.params.subscription_id == subscriptionId);
+        console.info(`Received rows: ${msg.params.table.all}`);
+        setBlueTable({ columns: msg.params.table.columns, rows: msg.params.table.all });
+      }
+    };
+  }, [getWebSocket, readyState]);
+
+  const subscribe = (query) => {
+    const req = subscribeReq(query);
+    console.info(`Subscribing: ${JSON.stringify(subscribeReq(query))}`);
+    sendJsonMessage(req);
+  };
+
+  const unsubscribe = () => {
+    const req = unsubscribeReq();
+    console.info(`Unsubscribing: ${JSON.stringify(req)}`);
+    sendJsonMessage(req);
+  };
+
+  const onOpened = () => {
+    setSidebarIsOpen(true);
+  };
+
+  const onClosed = () => {
+    setSidebarIsOpen(false);
+  };
+
+  const sidebarWidth = sidebarIsOpen ? '250px' : '65px';
+  return (
+    <ThemeProvider theme={theme}>
+      <CssBaseline />
+      <GlobalStyles styles={globalStyles} />
+      <div className='root-div'>
+        <div style={{
+          display: 'flex',
+          flexDirection: 'column',
+          alignItems: 'flex-start',
+          flexWrap: 'nowrap',
+          width: sidebarWidth,
+          transition: 'width 0.3s ease',
+        }}>
+          <Sidebar onSelection={onCannedQueryClicked} onOpened={onOpened} onClosed={onClosed} />
+        </div>
+        <div style={{
+          display: 'flex',
+          flexDirection: 'column',
+          left: sidebarWidth,
+          width: '100%',
+          transition: 'width 0.3s ease',
+        }}>
+          <div style={{ width: '100%', display: 'flex', flexDirection: 'column', alignItems: 'flex-start', margin: theme.spacing(2, 0, 0, 0) }}>
+            <div style={{ width: '90%', rightMargin: '100px' }}>
+              <QueryControl
+                queryState={queryState}
+                queryText={cannedQuery?.queryText}
+                queryTextResetTime={cannedQuery?.clickTime}
+                subscribe={subscribe}
+                unsubscribe={unsubscribe} />
+            </div>
+            <div style={{ width: '90%', rightMargin: '100px' }}>
+              <QueryStatusView queryState={queryState} details={queryStateDetails} error={queryError} />
+              <DataTable columns={blueTable?.columns ?? []} rows={blueTable?.rows ?? []} />
+            </div>
+          </div>
+        </div>
+      </div>
+    </ThemeProvider>
+  );
+};
+
+export default App;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataGrid.css

@@ -0,0 +1,53 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+.data-grid-container {
+  overflow: auto;
+  white-space: nowrap;
+}
+
+
+.data-grid {
+  max-height: 1000px;
+}
+
+div[role=columnheader] {
+  border-top: 0px;
+  border-bottom: 0px;
+  background-color: var(--primary-dark-color);
+}
+
+/* div[role=columnheader] {
+  border-top: 0px;
+  border-bottom: 0px;
+  background-color: var(--primary-dark-color);
+}
+
+.rdg-row {
+  border: 0px;
+  border-top: 0px;
+  border-bottom: 0px;
+  background-color: rgba(var(--primary-dark-color), 0.8);
+  color: var(--primary-color);
+}
+
+.rdg-row-odd {
+  border: 0px;
+  border-top: 0px;
+  border-bottom: 0px;
+  background-color: var(--primary-dark-color);
+  color: var(--primary-color);
+}
+
+.rdg-cell {
+  border-bottom: 0px;
+}
+
+.rdg-cell.even {
+  border-bottom: 5px;
+}
+
+.rdg-cell.odd {
+  border-bottom: 5px;
+} */

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataTable.css

@@ -0,0 +1,32 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+th {
+  font-size: large;  
+  padding: 15px;
+}
+
+td {
+  padding: 15px;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/DataTable.jsx

@@ -0,0 +1,74 @@
+import PropTypes from 'prop-types';
+import { useTheme } from '@mui/material/styles';
+import { DataGrid } from '@mui/x-data-grid';
+import './DataGrid.css'
+import { useMemo, useState } from 'react';
+
+export default function DataTable({ columns, rows }) {
+  const theme = useTheme();
+  const [columnWidths, setColumnWidths] = useState({});
+
+  const handleColumnResize = (params) => {
+    const newWidth = params.width;
+    setColumnWidths((prevWidths) => ({
+      ...prevWidths,
+      [params.colDef.field]: params.width,
+    }));
+  };
+
+  const getGridColDef = useMemo(() => {
+    return col => {
+      return {
+        field: col.name,
+        headerName: col.name,
+        width: columnWidths[col.name],
+      };
+    }
+  }, [columnWidths]);
+
+  return (
+    <DataGrid
+      className='data-grid-container'
+      autosizeOnMount={false}
+      columns={columns?.map(getGridColDef) ?? []}
+      onColumnResize={handleColumnResize}
+      rows={rows ?? []}
+      sx={{
+        margin: theme.spacing(2, 0, 0, 0),
+        borderColor: 'primary.main',
+        backgroundColor: 'primary.dark',
+        color: 'text.primary',
+        '& .MuiDataGrid-columnHeader': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-filler': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-cell': {
+          color: 'text.primary',
+        },
+        '& .MuiDataGrid-cell:hover': {
+          color: 'primary.main',
+        },
+        '& .MuiDataGrid-row': {
+          '--DataGrid-rowBorderColor': 'secondary.dark'
+        }
+      }}
+    />
+  );
+}
+
+DataTable.propTypes = {
+  columns: PropTypes.arrayOf(
+    PropTypes.shape({
+      name: PropTypes.string.isRequired,
+      type: PropTypes.string.isRequired,
+    })
+  ).isRequired,
+  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
+  apiRef: PropTypes.object,
+};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/QueryControl.jsx

@@ -0,0 +1,119 @@
+import PropTypes from 'prop-types';
+import DataTable from './DataTable';
+import { useRef, useState, useEffect } from 'react';
+import { useGridApiRef } from '@mui/x-data-grid';
+import Box from '@mui/material/Box';
+import Button from '@mui/material/Button';
+import Grid from '@mui/material/Grid';
+import Paper from '@mui/material/Paper';
+import PlayArrowIcon from '@mui/icons-material/PlayArrow';
+import StopIcon from '@mui/icons-material/Stop';
+import TextField from '@mui/material/TextField';
+import Tooltip from '@mui/material/Tooltip';
+
+const QueryControl = (props) => {
+  const { queryState, queryText, queryTextResetTime, subscribe, unsubscribe } = props;
+  // const theme = useTheme();
+  let textInput = useRef();
+  const [queryTextChangeTime, setQueryTextChangeTime] = useState(null);
+  useEffect(() => {
+    if (queryText == null || textInput.current == null) {
+      return;
+    }
+    // Init text field (unless a non-empty string is already present, or the user has modified
+    // the query - we don't want to overwrite their changes).
+    if (queryTextChangeTime == null || queryTextResetTime > queryTextChangeTime) {
+      console.info(`Initializing query text: ${queryText}`)
+      textInput.current.value = queryText;
+    }
+  }, [queryText, queryTextChangeTime, queryTextResetTime]);
+  const onClearClick = () => {
+    textInput.current.value = '';
+  };
+
+  const onRunClick = () => {
+    subscribe(textInput.current.value);
+  };
+
+  const onStopClick = () => {
+    unsubscribe();
+  };
+
+  const onKeyDown = (e) => {
+    // Detect keyboard shortcuts.
+    //   Ctrl+Enter : Run
+    if (e.ctrlKey && e.key === 'Enter') {
+      onRunClick();
+    }
+  };
+
+  const onChange = (e) => {
+    // So we know not to overwrite the changes with the canned query on re-render
+    setQueryTextChangeTime(Date.now());
+  };
+
+  return (
+    <Box sx={{ display: "flex", flexFlow: "column" }}>
+      <Paper elevation={3} sx={{ color: 'primary.main' }}>
+        <Box sx={{ p: 2 }}>
+          <TextField
+            fullWidth
+            multiline
+            rows={10}
+            variant="outlined"
+            onChange={onChange}
+            onKeyDown={onKeyDown}
+            inputRef={textInput}
+            placeholder="Enter Query..."
+          />
+        </Box>
+        <Box sx={{ p: 2, borderTop: 1 }}>
+          <Grid container spacing={2} justifyContent="flex-start">
+            <Grid item xs>
+              <Tooltip title="Clear Query Text">
+                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
+                  Clear
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Run (Ctrl+Enter)">
+                <Button onClick={onRunClick} sx={{
+                  color: `queryStartButton.${queryState}.color`,
+                  bgcolor: `queryStartButton.${queryState}.backgroundColor`,
+                  padding: '11px'
+                }}>
+                  <PlayArrowIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Stop">
+                <Button onClick={onStopClick} sx={{
+                  color: `queryStopButton.${queryState}.color`,
+                  bgcolor: `queryStopButton.${queryState}.backgroundColor`,
+                  padding: '11px'
+                }}>
+                  <StopIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+          </Grid>
+        </Box>
+      </Paper>
+    </Box>
+  );
+};
+
+QueryControl.propTypes = {
+  columns: PropTypes.array,
+  rows: PropTypes.array,
+  isRunning: PropTypes.bool,
+  queryText: PropTypes.string,
+  queryTextResetTime: PropTypes.number,
+  queryState: PropTypes.string,
+  subscribe: PropTypes.func,
+  unsubscribe: PropTypes.func,
+};
+
+export default QueryControl;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/QueryStatusView.jsx

@@ -0,0 +1,73 @@
+import { useState } from 'react';
+import PropTypes from 'prop-types';
+import { Box, Typography, Collapse } from '@mui/material';
+import Alert from '@mui/material/Alert';
+import AlertTitle from '@mui/material/AlertTitle';
+import { useTheme } from '@mui/material/styles';
+import { alpha } from '@mui/system';
+
+const QueryStatusView = ({ queryState, details, error }) => {
+  const theme = useTheme();
+  const [open, setOpen] = useState(false);
+
+  let header = queryState;
+  let headerContent = null;
+  let detailElements = [];
+  let severity = queryState == "Subscribed" ? "success" : "info";
+  let cursor = details == null ? "default" : "pointer";
+  if (error != null) {
+    severity = "error";
+    cursor = "pointer";
+    header = `Error Code: ${error.code}`;
+    headerContent = error.message;
+    detailElements.push(<Typography variant="body1">{error.data.error_message}</Typography>);
+    detailElements.push(<Typography variant="subtitle1">Stack Trace:</Typography>);
+    (error.data.stack_trace ?? []).forEach(line => detailElements.push(
+      <Typography variant="body2" component="pre" sx={{ whiteSpace: 'pre-wrap' }}>
+        {line}
+      </Typography>));
+  } else if (details != null) {
+    detailElements.push(<Typography variant="body1">{details}</Typography>);
+  }
+  return (
+    <Box sx={{ margin: theme.spacing(2, 0, 0, 0) }}>
+      <Alert
+        severity={severity}
+        onClick={() => setOpen(!open)}
+        sx={{
+          margin: theme.spacing(2, 0, 0, 0),
+          cursor: cursor,
+        }}>
+        <AlertTitle>{header}</AlertTitle>
+        {headerContent}
+      </Alert>
+      {detailElements.length == 0
+        ? null
+        : <Collapse in={open}>
+          <Box
+            sx={{
+              backgroundColor: alpha(theme.palette[severity].light, 0.75),
+              color: theme.palette[severity].contrastText,
+              padding: theme.spacing(2),
+              marginTop: theme.spacing(0),
+              borderRadius: theme.shape.borderRadius,
+            }}
+          >
+            {detailElements}
+          </Box>
+        </Collapse>}
+    </Box>
+  );
+};
+
+QueryStatusView.propTypes = {
+  queryState: PropTypes.string,
+  details: PropTypes.string,
+  error: PropTypes.shape({
+    code: PropTypes.number,
+    message: PropTypes.string,
+    data: PropTypes.object,
+  })
+};
+
+export default QueryStatusView;

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/ReactDataGrid.jsx

@@ -0,0 +1,27 @@
+import 'react-data-grid/lib/styles.css';
+import PropTypes from 'prop-types';
+import DataGrid from 'react-data-grid';
+import './DataGrid.css';
+
+function getGridColDef(col) {
+  return {
+    key: col.name,
+    name: col.name,
+  };
+}
+
+export default function ReactDataGrid({columns, rows}) {
+  return (
+    <DataGrid className={"data-grid"} rows={rows} columns={columns.map(getGridColDef)} />
+  )
+}
+
+ReactDataGrid.propTypes = {
+  columns: PropTypes.arrayOf(
+    PropTypes.shape({
+      name: PropTypes.string.isRequired,
+      type: PropTypes.string.isRequired,
+    })
+  ).isRequired,
+  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
+};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/SideBar.css

@@ -0,0 +1,98 @@
+/* Sidebar.css */
+.app-container {
+  display: flex;
+  width: 100%;
+  overflow: hidden;
+  flex-direction: row;
+  justify-content: flex-start;
+  align-items: flex-start;
+}
+
+.sidebar {
+  display: flex;
+  flex-direction: row;
+  justify-content: flex-start;
+  align-items: flex-start;
+  background-color: transparent;
+  color: white;
+  overflow: hidden;
+  transition: width 0.3s ease;
+}
+
+.sidebar.open {
+  width: 100%;
+}
+
+.sidebar.closed {
+  width: 0%;
+}
+
+.sidebar-toggle-button {
+  border: 0px;
+  border-radius: 50%;
+  box-shadow: rgba(9, 30, 66, 0.08) 0px 0px 0px 1px, rgba(9, 30, 66, 0.08) 0px 2px 4px 1px;
+  color: var(--primary-color);
+  cursor: pointer;
+  height: 24px;
+  opacity: 1;
+  outline: 0px;
+  padding: 0px;
+  background-color: transparent;
+  transition: background-color 100ms linear 0s, color 100ms linear 0s, opacity 300ms cubic-bezier(0.2, 0, 0, 1) 0s, transform 300ms cubic-bezier(0.2, 0, 0, 1) 0s;
+  width: 20%;
+}
+
+.sidebar-toggle-button:hover {
+  transition: background-color 100ms linear 0s, color 100ms linear 0s, opacity 300ms cubic-bezier(0.2, 0, 0, 1) 0s, transform 300ms cubic-bezier(0.2, 0, 0, 1) 0s;
+}
+
+.main-content.open {
+  width: 165px;
+  transition: width 0.3s ease;
+  background-color: transparent;
+}
+
+.main-content.closed {
+  width: 0%;
+  /* min-width: 25px; */
+  transition: width 0.3s ease;
+  background-color: transparent;
+}
+
+.sidebar-menu-item {
+  width: 100%;
+  box-sizing: border-box;
+}
+
+.sidebar-menu-item.open {
+  width: 100%;
+  /* justify-content: flex-start; */
+  transition: width 0.3s ease;
+}
+
+.sidebar-menu-item.closed {
+  width: 0%;
+  min-width: 0px;
+  /* justify-content: flex-start; */
+  transition: width 0.3s ease;
+}
+
+.sidebar-menu-item-content.open {
+  width: 100%;
+  /* justify-content: flex-start; */
+  transition: width 0.3s ease;
+}
+
+.sidebar-menu-item-content.closed {
+  width: 0px;
+  /* justify-content: flex-start; */
+  transition: width 0.3s ease;
+}
+
+/* Ensure span can shrink */
+.button-text {
+  display: block;    /* Make span a block element */
+  width: 100%;       /* Ensure it takes the full width of its parent */
+  overflow: hidden;  /* Hide any overflow content */
+  white-space: nowrap; /* Prevent text wrapping */
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/SideBar.jsx

@@ -0,0 +1,109 @@
+import { useState, useEffect } from 'react';
+import Button from '@mui/material/Button';
+import { useTheme } from '@mui/material/styles';
+import PropTypes from 'prop-types';
+import Tooltip from '@mui/material/Tooltip';
+import ChevronLeftRounded from "@mui/icons-material/ChevronLeftRounded"
+import ChevronRightRounded from '@mui/icons-material/ChevronRightRounded';
+import FileCopyIcon from '@mui/icons-material/FileCopy';
+import './SideBar.css';
+
+const Sidebar = ({ onSelection, onOpened, onClosed }) => {
+  const theme = useTheme();
+  const [isOpen, setIsOpen] = useState(false);
+  const [cannedQueries, setCannedQueries] = useState([]);
+  const Icon = isOpen ? ChevronLeftRounded : ChevronRightRounded;
+
+  const toggleIsOpen = () => {
+    if (isOpen) {
+      setIsOpen(false);
+      if (onClosed != null) {
+        onClosed();
+      }
+    } else {
+      setIsOpen(true);
+      if (onOpened != null) {
+        onOpened();
+      }
+    }
+  };
+
+  const onMenuItemClick = (event, cannedQuery) => {
+    console.info(`menu item clicked: ${cannedQuery.name}`);
+    onSelection(cannedQuery);
+  };
+
+  // const onMenuItemClick = async (event, cannedQuery) => {
+  //   try {
+  //     await navigator.clipboard.writeText(cannedQuery.query.join("\n"));
+  //     alert(`Copied ${cannedQuery.name} to clipboard`);
+  //   } catch (err) {
+  //     console.error('Failed to copy query text: ', err);
+  //   }
+  // };
+
+  useEffect(() => {
+    const fetchCannedQueries = async () => {
+      try {
+        const response = await fetch('api/cannedqueries');
+        const data = await response.json();
+        setCannedQueries(data);
+      } catch (error) {
+        console.error('Error fetching WebSocket config:', error);
+      }
+    }
+    fetchCannedQueries();
+  }, []);
+
+  const mainContentClass = `main-content ${isOpen ? 'open' : 'closed'}`;
+  const menuItemClass = `sidebar-menu-item ${isOpen ? 'open' : 'closed'}`;
+  const menuItemContentClass = `sidebar-menu-item-content ${isOpen ? 'open' : 'closed'}`;
+  const toggleButtonTitle = `${isOpen ? 'Hide' : 'Show'} Common Queries`;
+
+  return (
+    <div
+      className={"sidebar"}
+      style={{
+        display: 'flex',
+        flexDirection: 'row',
+        justifyContent: 'flexStart',
+        alignItems: 'flexStart',
+        margin: theme.spacing(2, 0, 0, 0)
+      }}
+    >
+      <div className={mainContentClass}>
+        {cannedQueries.map(cq =>
+          <Button key={cq.name} className={menuItemClass} onClick={(event) => onMenuItemClick(event, cq)}>
+            <div className={menuItemContentClass}>
+              <span className='button-text'>{cq.name}</span>
+            </div>
+          </Button>)
+        }
+        {/* {cannedQueries.map(cq =>
+          <Button
+            variant="contained"
+            color="primary"
+            startIcon={<FileCopyIcon />}
+            onClick={onMenuItemClick}
+          >
+            <div className={menuItemContentClass}>
+              <span className='button-text'>{cq.name}</span>
+            </div>
+          </Button>)} */}
+      </div>
+      <Tooltip title={toggleButtonTitle}>
+        <Button onClick={toggleIsOpen}>
+          <Icon className="w-5 h-5" />
+        </Button>
+      </Tooltip>
+    </div>
+  );
+};
+
+export default Sidebar;
+
+Sidebar.propTypes = {
+  onSelection: PropTypes.func,
+  onOpened: PropTypes.func,
+  onClosed: PropTypes.func,
+};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/Theme.js

@@ -0,0 +1,112 @@
+import { createTheme } from '@mui/material/styles';
+import PropTypes from 'prop-types';
+import '@fontsource/noto-sans';
+import './App.css';
+
+const getLightPalette = () => {
+  return {
+    mode: 'light',
+    primary: {
+      light: '#6573c3',
+      main: '#3f51b5',
+      dark: '#2c387e',
+      contrastText: '#ffffff',
+    },
+    secondary: {
+      light: '#f73378',
+      main: '#f50057',
+      dark: '#ab003c',
+      contrastText: '#ffffff',
+    },
+    text: {
+      primary: '#000000c4',
+      secondary: '#00000086',
+      disabled: '#0000005b',
+      hint: '#0000005b',
+    }
+  };
+};
+
+const getDarkPalette = () => {
+  return {
+    mode: 'dark',
+    primary: {
+      light: '#bcc5ff',
+      main: '#33baf9',
+      dark: '#202334',
+      contrastText: '#78a2ac',
+    },
+    secondary: {
+      light: '#9fd0b8',
+      main: '#5eb289',
+      dark: '#ffffff12',
+      contrastText: '#c8d4f7',
+    },
+    action: {
+      hoverOpacity: 0.08
+    },
+    text: {
+      primary: '#eaf8f8',
+      dark: '#eaf8f8a0'
+    },
+    queryStartButton: {
+      Subscribed: {
+        color: '#33baf9',
+        backgroundColor: '#ffffff40',
+      },
+      Unsubscribed: {
+        color: '#ffffff40',
+        backgroundColor: '#ffffff00',
+      }
+    },
+    queryStopButton: {
+      Subscribed: {
+        color: '#ff4444a0',
+        backgroundColor: '#ffffff40',
+      },
+      Unsubscribed: {
+        color: '#ff444450',
+        backgroundColor: '#ffffff00',
+      }
+    },
+    error: {
+      light: '#e57373',
+      main: '#f44336',
+      dark: '#d32f2f',
+      contrastText: '#fff',
+    },
+    warning: {
+      light: '#ffb74d',
+      main: '#ff9800',
+      dark: '#f57c00',
+      contrastText: '#000',
+    },
+    info: {
+      light: '#64b5f6',
+      main: '#2196f3',
+      dark: '#303030',
+      contrastText: '#fff',
+    },
+    success: {
+      light: '#81c784',
+      main: '#4caf50',
+      dark: '#388e3c',
+      contrastText: '#000',
+    },
+  };
+};
+
+const getTheme = ({ mode }) => {
+  return createTheme({
+    typography: {
+      fontFamily: '"Helvetica"'
+    },
+    palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
+  });
+};
+
+export default getTheme;
+
+getTheme.propTypes = {
+  mode: PropTypes.string.isRequired,
+};

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/assets/react.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/dark-hc.css

@@ -0,0 +1,51 @@
+.dark-high-contrast {
+  --md-sys-color-primary: rgb(251 250 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 0 0);
+  --md-sys-color-primary-container: rgb(173 204 255);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(250 250 255);
+  --md-sys-color-on-secondary: rgb(0 0 0);
+  --md-sys-color-secondary-container: rgb(159 208 255);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(255 249 251);
+  --md-sys-color-on-tertiary: rgb(0 0 0);
+  --md-sys-color-tertiary-container: rgb(222 193 231);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 249 249);
+  --md-sys-color-on-error: rgb(0 0 0);
+  --md-sys-color-error-container: rgb(255 186 177);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(255 255 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(251 250 255);
+  --md-sys-color-outline: rgb(200 202 212);
+  --md-sys-color-outline-variant: rgb(200 202 212);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(0 0 0);
+  --md-sys-color-inverse-primary: rgb(0 42 83);
+  --md-sys-color-primary-fixed: rgb(219 231 255);
+  --md-sys-color-on-primary-fixed: rgb(0 0 0);
+  --md-sys-color-primary-fixed-dim: rgb(173 204 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
+  --md-sys-color-secondary-fixed: rgb(214 233 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 0 0);
+  --md-sys-color-secondary-fixed-dim: rgb(159 208 255);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
+  --md-sys-color-tertiary-fixed: rgb(249 222 255);
+  --md-sys-color-on-tertiary-fixed: rgb(0 0 0);
+  --md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/dark-mc.css

@@ -0,0 +1,51 @@
+.dark-medium-contrast {
+  --md-sys-color-primary: rgb(173 204 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 22 49);
+  --md-sys-color-primary-container: rgb(112 146 198);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(159 208 255);
+  --md-sys-color-on-secondary: rgb(0 24 42);
+  --md-sys-color-secondary-container: rgb(100 149 193);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(222 193 231);
+  --md-sys-color-on-tertiary: rgb(33 14 42);
+  --md-sys-color-tertiary-container: rgb(162 136 171);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 186 177);
+  --md-sys-color-on-error: rgb(55 0 1);
+  --md-sys-color-error-container: rgb(255 84 73);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(251 250 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(200 202 212);
+  --md-sys-color-outline: rgb(160 163 171);
+  --md-sys-color-outline-variant: rgb(128 131 139);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(40 42 47);
+  --md-sys-color-inverse-primary: rgb(35 73 120);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 17 40);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 18 34);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(28 9 37);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/dark.css

@@ -0,0 +1,51 @@
+.dark {
+  --md-sys-color-primary: rgb(165 200 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 49 94);
+  --md-sys-color-primary-container: rgb(34 72 118);
+  --md-sys-color-on-primary-container: rgb(212 227 255);
+  --md-sys-color-secondary: rgb(154 203 250);
+  --md-sys-color-on-secondary: rgb(0 51 83);
+  --md-sys-color-secondary-container: rgb(12 74 114);
+  --md-sys-color-on-secondary-container: rgb(206 229 255);
+  --md-sys-color-tertiary: rgb(218 189 226);
+  --md-sys-color-on-tertiary: rgb(61 41 70);
+  --md-sys-color-tertiary-container: rgb(85 63 93);
+  --md-sys-color-on-tertiary-container: rgb(247 216 255);
+  --md-sys-color-error: rgb(255 180 171);
+  --md-sys-color-on-error: rgb(105 0 5);
+  --md-sys-color-error-container: rgb(147 0 10);
+  --md-sys-color-on-error-container: rgb(255 218 214);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(225 226 233);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(195 198 207);
+  --md-sys-color-outline: rgb(141 145 153);
+  --md-sys-color-outline-variant: rgb(67 71 78);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(46 48 53);
+  --md-sys-color-inverse-primary: rgb(60 96 144);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/light-hc.css

@@ -0,0 +1,51 @@
+.light-high-contrast {
+  --md-sys-color-primary: rgb(0 34 70);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(29 68 114);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(0 36 61);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(3 70 110);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(46 26 55);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(81 59 89);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(78 0 2);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(140 0 9);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(0 0 0);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(32 36 43);
+  --md-sys-color-outline: rgb(63 67 74);
+  --md-sys-color-outline-variant: rgb(63 67 74);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(255 255 255);
+  --md-sys-color-inverse-primary: rgb(228 236 255);
+  --md-sys-color-primary-fixed: rgb(29 68 114);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(0 45 88);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(3 70 110);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(0 47 77);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(81 59 89);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/light-mc.css

@@ -0,0 +1,51 @@
+.light-medium-contrast {
+  --md-sys-color-primary: rgb(29 68 114);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(83 118 167);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(3 70 110);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(71 121 163);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(81 59 89);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(133 108 141);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(140 0 9);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(218 52 46);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(63 67 74);
+  --md-sys-color-outline: rgb(91 95 103);
+  --md-sys-color-outline-variant: rgb(119 123 131);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(83 118 167);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(58 93 141);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(71 121 163);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(43 96 137);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(133 108 141);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/css/light.css

@@ -0,0 +1,51 @@
+.light {
+  --md-sys-color-primary: rgb(60 96 144);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(212 227 255);
+  --md-sys-color-on-primary-container: rgb(0 28 58);
+  --md-sys-color-secondary: rgb(46 98 140);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(206 229 255);
+  --md-sys-color-on-secondary-container: rgb(0 29 50);
+  --md-sys-color-tertiary: rgb(110 86 118);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(247 216 255);
+  --md-sys-color-on-tertiary-container: rgb(39 20 48);
+  --md-sys-color-error: rgb(186 26 26);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(255 218 214);
+  --md-sys-color-on-error-container: rgb(65 0 2);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(67 71 78);
+  --md-sys-color-outline: rgb(116 119 127);
+  --md-sys-color-outline-variant: rgb(195 198 207);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/index.css

@@ -0,0 +1,107 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+button#query-button {
+  height: 5em;
+  vertical-align: top;
+}
+
+button#play-button {
+  padding: '10px 20px';
+  font-size: '16px';
+  cursor: 'pointer';
+  background-color: '#28a745';
+  color: '#fff';
+  border: 'none';
+  border-radius: '5px';
+};
+
+button#pause-button {
+  background-color: #ffc107;
+}
+
+button#stop-button {
+  background-color: #dc3545;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+p#footer {
+  float: none;
+  clear: both;
+  vertical-align: bottom;
+}
+
+table#query-output {
+  float: none;
+  clear: both;
+}
+
+textarea#query-input {
+  height: 5em;
+  width: 50em;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/src/main.jsx

@@ -0,0 +1,7 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App';
+
+ReactDOM.createRoot(document.getElementById('root')).render(
+  <App />
+)

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -0,0 +1,69 @@
+import { fileURLToPath, URL } from 'node:url';
+import { defineConfig } from 'vite';
+import plugin from '@vitejs/plugin-react';
+import fs from 'fs';
+import path from 'path';
+import child_process from 'child_process';
+import process, { env } from 'process';
+
+const baseFolder =
+  process.env.APPDATA !== undefined && process.env.APPDATA !== ''
+    ? `${process.env.APPDATA}/ASP.NET/https`
+    : `${process.env.HOME}/.aspnet/https`;
+
+const certificateName = "bts.datastreamwebapp.client";
+const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
+const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
+
+if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
+  if (0 !== child_process.spawnSync('dotnet', [
+    'dev-certs',
+    'https',
+    '--export-path',
+    certFilePath,
+    '--format',
+    'Pem',
+    '--no-password',
+  ], { stdio: 'inherit', }).status) {
+    throw new Error("Could not create certificate.");
+  }
+}
+
+const target = env.ASPNETCORE_HTTPS_PORT
+  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
+  : env.ASPNETCORE_URLS
+    ? env.ASPNETCORE_URLS.split(';')[0]
+    : 'https://localhost:8443';
+
+export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
+  return {
+    plugins: [plugin()],
+    resolve: {
+      alias: {
+        '@': fileURLToPath(new URL('./src', import.meta.url))
+      }
+    },
+    build: {
+      minify: false,
+      sourcemap: true,
+      outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
+    },
+    server: {
+      proxy: {
+        '^/api': {
+          target,
+          secure: false
+        },
+        '^/ws': {
+          target,
+          secure: false
+        }
+      },
+      port: 5173,
+      https: {
+        key: fs.readFileSync(keyFilePath),
+        cert: fs.readFileSync(certFilePath),
+      }
+    }
+  };
+});

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Bts.DataStreamWebApp.Server.csproj

@@ -0,0 +1,40 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <InvariantGlobalization>true</InvariantGlobalization>
+    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
+    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
+    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+    </ProjectReference>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ConfigurationProfile Include="appsettings.Development.json" />
+    <ConfigurationProfile Include="appsettings.Production.json" />
+    <ConfigurationProfile Include="appsettings.ProductionAws.json" />
+    <ConfigurationProfile Include="appsettings.Staging.json" />
+    <ConfigurationProfile Include="appsettings.json" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/CannedQueriesConfig.cs

@@ -0,0 +1,5 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class CannedQueriesConfig {
+   public List<CannedQuery>? Items { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/CannedQuery.cs

@@ -0,0 +1,6 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class CannedQuery {
+   public string? Name { get; set; }
+   public List<string>? Query { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/CannedQueriesController.cs

@@ -0,0 +1,18 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using Microsoft.AspNetCore.Mvc;
+
+[ApiController]
+[Route("api/[controller]")]
+public class CannedQueriesController : ControllerBase {
+   readonly IConfiguration config;
+
+   public CannedQueriesController(IConfiguration config) =>
+      this.config = config;
+
+   [HttpGet(Name = "GetCannedQueries")]
+   public IEnumerable<CannedQuery>? Get() {
+      var cannedQueries = config.GetSection("CannedQueries").Get<List<CannedQuery>>()?.ToArray() ?? Array.Empty<CannedQuery>();
+      return cannedQueries;
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/ConfigurationController.cs

@@ -0,0 +1,22 @@
+// using Bts.DataStreamWebApp.Server;
+
+// using Microsoft.AspNetCore.Mvc;
+
+// [ApiController]
+// [Route("api/[controller]")]
+// public class ConfigController : ControllerBase {
+//    private readonly IConfiguration _configuration;
+
+//    public ConfigController(IConfiguration configuration) {
+//       _configuration = configuration;
+//    }
+
+//    [HttpGet("websocket")]
+//    public ActionResult<DataStreamWebSocketConfig> GetWebSocketConfig() {
+//       var endpoint = $"{_configuration["Kestrel:Endpoints:Http:Url"]}/ws";
+//       var webSocketConfig = new DataStreamWebSocketConfig {
+//          Endpoint = endpoint // or Https based on your setup
+//       };
+//       return Ok(webSocketConfig);
+//    }
+// }

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Controllers/DataStreamSubscriptionController.cs

@@ -0,0 +1,30 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using Microsoft.AspNetCore.Mvc;
+
+[Route("/ws")]
+public class DataStreamSubscriptionController : ControllerBase {
+   readonly ILogger logger;
+
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
+      this.logger = logger;
+      logger.LogInformation("Creating {ControllerType}", GetType().Name);
+   }
+
+   [Route("/ws")]
+   public async Task Get(CancellationToken cancellationToken) {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      logger.LogInformation("Establishing websocket connection...");
+      try {
+         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+         logger.LogInformation("Websocket connection established.");
+         using var server = new DataStreamWebSocketServer(webSocket, logger);
+         await server.Run(cancellationToken);
+      } catch (Exception ex) {
+         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
+      }
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscription.cs

@@ -0,0 +1,133 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System;
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamSubscription : IDisposable, IHostedService {
+   public readonly ViewStream ViewStream;
+   public DataPacket? LastFired;
+
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+
+   public DataStreamSubscription(string id, JsonRpcId requestId, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
+      ViewStream = viewStream;
+      RequestId = requestId;
+      Id = id;
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public string Id { get; }
+   public JsonRpcId RequestId { get; }
+   public SubscriptionStatus Status { get; private set; }
+
+   public void Start() {
+      if (Status == SubscriptionStatus.Subscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Subscribed;
+      ViewStream.onChange += onChange;
+   }
+
+   public Task StartAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Start();
+      return Task.CompletedTask;
+   }
+
+   public Task Clear() {
+      if (LastFired is null) {
+         return Task.CompletedTask;
+      }
+      logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
+      return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
+   }
+
+   public void Stop() {
+      if (Status == SubscriptionStatus.Unsubscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Unsubscribed;
+      ViewStream.onChange -= onChange;
+   }
+
+   public Task StopAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Stop();
+      return Task.CompletedTask;
+   }
+
+   public void Dispose() =>
+      Stop();
+
+   async void onChange(object newValue) {
+      if (newValue is not DataPacket dp) {
+         logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
+         return;
+      }
+      if (webSocket.State != WebSocketState.Open) {
+         logger.LogError(
+            "Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
+         Dispose();
+         return;
+      }
+      await sendDataPacket("subscribe", dp);
+   }
+
+   async Task sendDataPacket(string method, DataPacket dp) {
+      var serializerOpts = new JsonSerializerOptions {
+         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+      };
+      serializerOpts.Converters.Add(new DataPacketJsonConverter() {
+         IncludeCurrent = true,
+         UseObjectRows = true,
+      });
+      var res = JsonRpcRequest.CreateNotification(method, new DataStreamSubscriptionUpdate() {
+         SubscriptionId = Id,
+         Table = dp
+      });
+      var str = serialize(res, serializerOpts);
+      if (str is null) {
+         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
+         Dispose();
+         return;
+      }
+      logger.LogInformation(
+         "Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
+         dp.newRows.Count,
+         dp.schema.columnNames.Length,
+         str.Length);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      try {
+         await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
+         LastFired = dp;
+      } catch (WebSocketException ex) {
+         logger.LogError($"sendDataPacket failed on webSocket.SendAsync: {ex}");
+      }
+   }
+
+   string? serialize<T>(T message, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      try {
+         return JsonSerializer.Serialize(message, opts);
+      } catch (Exception ex) {
+         logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
+      }
+      return null;
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamSubscriptionUpdate.cs

@@ -0,0 +1,13 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using BCG;
+
+using System.Text.Json.Serialization;
+
+public class DataStreamSubscriptionUpdate {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   [JsonPropertyName("table")]
+   public DataPacket? Table { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebAppServerConfig.cs

@@ -0,0 +1,89 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using Bts.PubSub;
+
+using CommandLine;
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamWebAppServerConfig {
+   /// <summary>
+   /// The login user for sparkd
+   /// </summary>
+   [JsonPropertyName("user")]
+   [Option("user", HelpText = "The sparkd login user")]
+   public string? User { get; set; }
+
+   /// <summary>
+   /// The login password for sparkd
+   /// </summary>
+   [JsonPropertyName("password")]
+   [Option("pass", HelpText = "The sparkd login password")]
+   public string? Password { get; set; }
+
+   /// <summary>
+   /// The config server url for sparkd
+   /// </summary>
+   [JsonPropertyName("sparkd")]
+   [Option("sparkd", HelpText = "The sparkd config server")]
+   public string? Sparkd { get; set; }
+
+   [JsonPropertyName("pubsub")]
+   public PubSubConfig? PubSub { get; set; }
+
+   [Option("content-root-path")]
+   [JsonPropertyName("content_root_path")]
+   public string? ContentRootPath { get; set; }
+
+   [Option("web-root-path")]
+   [JsonPropertyName("web_root_path")]
+   public string? WebRootPath { get; set; }
+
+   [Option("environment-name")]
+   [JsonPropertyName("environment_name")]
+   public string? EnvironmentName { get; set; }
+
+   public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
+
+   [Option(
+      "config",
+      MetaValue = "PATH",
+      HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
+   [JsonIgnore]
+   public string? ConfigFile { get; set; }
+
+   public static DataStreamWebAppServerConfig Default() =>
+      new() {
+         User = "bts",
+         PubSub = PubSubConfig.Default(),
+      };
+
+   public void MergeWith(DataStreamWebAppServerConfig other) {
+      User ??= other.User;
+      Password ??= other.Password;
+      PubSub ??= other.PubSub;
+      Sparkd ??= other.Sparkd;
+      ContentRootPath ??= other.ContentRootPath;
+      WebRootPath ??= other.WebRootPath;
+      ConfigFile ??= other.ConfigFile;
+   }
+
+   /// <summary>
+   /// Merges options with those defined in `ConfigFile` (does not overwrite
+   /// non-null existing values).
+   /// </summary>
+   /// <exception cref="InvalidOperationException">
+   /// JsonSerializer.Deserialize returns null
+   /// </exception>
+   public void MergeWithConfigFile() {
+      if (ConfigFile is null) {
+         return;
+      }
+      var cfgString = File.ReadAllText(ConfigFile);
+      var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
+         throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
+      MergeWith(parsedCfg);
+      ConfigFile = null;
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketConfig.cs

@@ -0,0 +1,5 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class DataStreamWebSocketConfig {
+   public string? Endpoint { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/DataStreamWebSocketServer.cs

@@ -0,0 +1,148 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
+
+   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public async Task<int> Run(CancellationToken cancellationToken) {
+      logger.LogInformation("DataStreamWebSocketServer running");
+      await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger, cancellationToken)) {
+         await (req.Method switch {
+            "subscribe" => onSubscribe(req, cancellationToken),
+            "unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>(), cancellationToken),
+            _ => Task.CompletedTask,
+         });
+      }
+      logger.LogInformation("DataStreamWebSocketServer stopped");
+      return 0;
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions. To additionally notify the client of the subscription status
+   /// change(s), use <see cref="DisposeAsync"/>.
+   /// </summary>
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      disposeSubscriptions();
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions (like <see cref="Dispose"/>), then sends 'subscription status change'
+   /// notifications to the client (unlike <see cref="Dispose"/>).
+   /// </summary>
+   public async ValueTask DisposeAsync() {
+      GC.SuppressFinalize(this);
+      var finalSubs = subscriptions.Values.Select(s => (s.RequestId, s.Id)).ToList();
+      disposeSubscriptions();
+      // Send a 'subscription status change' msg to the client for each one.
+      foreach (var (requestId, id) in finalSubs) {
+         await sendSubscriptionStatus(requestId, id, "unsubscribe", SubscriptionStatus.Unsubscribed, CancellationToken.None);
+      }
+   }
+
+   async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      if (!SubscribeRequest.Parse(req, out var subReq, out var parseErr, out var errCode)) {
+         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
+         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
+            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
+         await send(rpcError, cancellationToken);
+         return;
+      }
+      var subId = subReq.SubscriptionId ?? throw new InvalidOperationException("SubscriptionId is null (parse false)");
+      disposeSubscription(subId);
+      if (!subReq.ParseQuery(out var vs, out parseErr, out errCode)) {
+         logger.LogError("Invalid 'subscribe' request: {Reason}", parseErr);
+         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
+            req.Id, new() { Code = (long)errCode, Message = "Failed to parse subscription request", Data = parseErr });
+         await send(rpcError, cancellationToken);
+         return;
+      }
+      if (subscriptions.TryGetValue(subId, out var oldSub)) {
+         logger.LogInformation("Disposing existing subscription with id {SubscriptionId}", subId);
+         oldSub.Dispose();
+      }
+      var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
+      subscriptions[subId] = sub;
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
+      vs.ErrorChanged += OnStreamError;
+      await sub.StartAsync(cancellationToken);
+
+      async void OnStreamError(Exception exception) {
+         logger.LogError("Error in ViewStream (disposing subscription): {Exception}", exception);
+         subscriptions.Remove(sub.Id);
+         sub.Dispose();
+         var error = new SubscriptionError() {
+            ErrorMessage = "Error in ViewStream (disposing subscription)"
+         };
+         if (exception is not null) {
+            error.ErrorMessage += ". " + exception.Message;
+            error.StackTrace = exception.StackTrace?.Split("\n")?.ToList();
+         }
+         var rpcError = JsonRpcResponse.CreateError<SubscriptionError>(
+            req.Id, new() { Code = (long)ErrorCode.ViewStreamError, Message = "Error in ViewStream (disposing subscription)", Data = error });
+         await send(rpcError, cancellationToken);
+      }
+   }
+
+   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      var subId = req.Params?.SubscriptionId ?? string.Empty;
+      if (!disposeSubscription(subId)) {
+         logger.LogInformation("Unable to dispose subscription. Subscription with id {Id} not found.", subId);
+      }
+      await sendSubscriptionStatus(req.Id.Value, subId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   bool disposeSubscription(string subId) {
+      if (!subscriptions.TryGetValue(subId, out var sub)) {
+         return false;
+      }
+      logger.LogInformation("Disposing subscription with id {Id}", subId);
+      sub.Dispose();
+      subscriptions.Remove(subId);
+      return true;
+   }
+
+   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
+      logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
+      var response = JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
+         SubscriptionId = subId,
+         Status = status,
+      });
+      return send(response, cancellationToken);
+   }
+
+   Task send<T>(T message, CancellationToken cancellationToken) {
+      if (message is null) {
+         return Task.CompletedTask;
+      }
+      var str = JsonSerializer.Serialize(message);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
+      return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken);
+   }
+
+   void disposeSubscriptions() {
+      logger.LogInformation("Disposing DataStreamWebSocketServer subscriptions. Count: {Count}", subscriptions.Count);
+      subscriptions.Values.DisposeAll();
+      subscriptions.Clear();
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile

@@ -0,0 +1,11 @@
+FROM bluetradingsystems/linux_dotnet7
+
+ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
+ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
+
+RUN dotnet dev-certs https --trust
+COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
+WORKDIR "/home/bts/bin"

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/CannedQueriesResponse.cs

@@ -0,0 +1,5 @@
+namespace Bts.DataStreamWebApp.Server;
+
+public class CannedQueriesResponse {
+   public List<CannedQuery>? Items { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/ErrorCode.cs

@@ -0,0 +1,7 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+enum ErrorCode {
+   None = 0,
+   InvalidSubscriptionRequest,
+   ViewStreamError,
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscribeRequest.cs

@@ -0,0 +1,72 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamExpressions;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+public class SubscribeRequest {
+   [JsonPropertyName("query")]
+   public string? Query { get; set; }
+
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   internal static bool Parse(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)]out SubscribeRequest? subscription, [NotNullWhen(false)]out SubscriptionError? error, out ErrorCode errorCode) {
+      subscription = null;
+      errorCode = ErrorCode.None;
+      var paramsJsonObj = req.Params?.AsObject();
+      error = null;
+      if (paramsJsonObj is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         error = new() { ErrorMessage = "'params' object not found" };
+         return false;
+      }
+      subscription = paramsJsonObj.Deserialize<SubscribeRequest>();
+      if (subscription is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         error = new() { ErrorMessage = "'params' object invalid" };
+         return false;
+      }
+      return true;
+   }
+
+   internal bool ParseQuery([NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out SubscriptionError? error, out ErrorCode errorCode) {
+      stream = null;
+      errorCode = ErrorCode.None;
+      error = null;
+      if (Query is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         error = new() { ErrorMessage = "'params.query' field not found (expected a string)" };
+         return false;
+      }
+      var exp = DataStreamExpressionsInterpreter.Evaluate(Query);
+      if (exp is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         error = new() { ErrorMessage = "'params.query' expression has invalid syntax" };
+         return false;
+      }
+      Exception? exception = null;
+      try {
+         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      } catch (Exception ex) {
+         exception = ex;
+      }
+      if (stream is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         error = new() { ErrorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)" };
+         if (exception is not null) {
+            error.ErrorMessage += ". " + exception.Message;
+            error.StackTrace = exception.StackTrace?.Split("\n")?.ToList();
+         }
+         return false;
+      }
+      return true;
+   }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscriptionError.cs

@@ -0,0 +1,11 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+public class SubscriptionError {
+   [JsonPropertyName("error_message")]
+   public string? ErrorMessage { get; set; }
+
+   [JsonPropertyName("stack_trace")]
+   public List<string>? StackTrace { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/SubscriptionStatusMessage.cs

@@ -0,0 +1,18 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+enum SubscriptionStatus {
+   None,
+   Subscribed,
+   Unsubscribed,
+}
+
+class SubscriptionStatusMessage {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   [JsonPropertyName("status")]
+   [JsonConverter(typeof(JsonStringEnumConverter))]
+   public SubscriptionStatus Status { get; set; } = SubscriptionStatus.None;
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Messages/UnsubscribeRequest.cs

@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+public class UnsubscribeRequest {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Program.cs

@@ -0,0 +1,82 @@
+using Bts.DataStreamCore.Services;
+using Bts.PubSub.Services;
+
+using Spark.Api.Services;
+using Spark.Core.Services;
+
+using System.Reactive.Linq;
+using System.Reflection;
+
+var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
+   Args = args,
+   ApplicationName = "Bts.DataStreamWebApp.Server",
+});
+ConfigureApp(builder.Configuration, builder.Environment.EnvironmentName);
+ConfigureServices(builder.Services, builder.Configuration);
+
+var app = builder.Build();
+var wsOpts = new WebSocketOptions {
+   KeepAliveInterval = TimeSpan.FromMinutes(2)
+};
+if (app.Environment.IsDevelopment()) {
+   app.UseDirectoryBrowser("/browse");
+   app.UseSwagger();
+   app.UseSwaggerUI();
+}
+app.UseHttpsRedirection();
+app.UseWebSockets(wsOpts);
+app.UseDefaultFiles();
+app.UseStaticFiles();
+app.UseAuthorization();
+app.MapControllers();
+try {
+   Console.Write($"Starting Application");
+   app.Run();
+   Console.Write($"Application exited gracefully.");
+} catch (Exception ex) {
+   Console.Error.Write($"Application crashed: {ex}");
+   return 1;
+}
+return 0;
+
+static void ConfigureApp(ConfigurationManager configBuilder, string environment) {
+   // We prefix appsettings with the exe name because multiple exes will usually be in the build dir
+   // (this way, each exe can still load its own appsettings files from the current directory).
+   var exeName = Assembly.GetCallingAssembly().GetName().Name;
+   AddJsonFile($"{exeName}.appsettings.json");
+   AddJsonFile($"{exeName}.appsettings.{environment}.json");
+
+   var cfg = configBuilder.GetSection("config")?.Value;
+   if (!SetBasePath(cfg)) {
+      AddJsonFile(cfg);
+   }
+
+   bool SetBasePath(string? path) {
+      if (!Directory.Exists(path)) {
+         return false;
+      }
+      configBuilder.SetBasePath(path);
+      return true;
+   }
+
+   bool AddJsonFile(string? path) {
+      if (!File.Exists(path)) {
+         return false;
+      }
+      configBuilder.AddJsonFile(path);
+      return true;
+   }
+}
+
+static void ConfigureServices(IServiceCollection services, IConfiguration config) {
+   services.AddDirectoryBrowser()
+      .AddControllers().Services
+      .AddEndpointsApiExplorer()
+      .AddSwaggerGen();
+   services.AddSpaStaticFiles();
+   services.AddSpark(config.GetSection("Spark"))
+      .AddSparkApi(config.GetSection("SparkApi"))
+      .AddMasterWorker(config.GetSection("MasterWorker"))
+      .AddViewStreamDatabase()
+      .AddPubSubClient(config.GetSection("PubSub"));
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/Properties/launchSettings.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "applicationUrl": "http://localhost:8080",
+      "environmentVariables": {
+        "ASPNETCORE_HTTP_PORT": "8080",
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:8080;https://localhost:8443",
+      "environmentVariables": {
+        "ASPNETCORE_HTTP_PORT": "8080",
+        "ASPNETCORE_HTTPS_PORT": "8443",
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+  },
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Development.json

@@ -0,0 +1,52 @@
+{
+  "PubSub": {
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://localhost:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://localhost:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    }
+  },
+  "CannedQueries": [
+    {
+      "Name": "BBO",
+      "Query": [
+        "bbo"
+      ]
+    },
+    {
+      "Name": "Trades",
+      "Query": [
+        "timesales"
+      ]
+    },
+    {
+      "Name": "Instruments",
+      "Query": [
+        "instruments"
+      ]
+    },
+    {
+      "Name": "Books",
+      "Query": [
+        "books"
+      ]
+    },
+    {
+      "Name": "Benzinga News",
+      "Query": [
+        "benzinga_news"
+      ]
+    }
+  ]
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Production.json

@@ -0,0 +1,8 @@
+{
+  "Spark": {
+    "ConfigServer": "http://sparkd:8000/config"
+  },
+  "SparkApi": {
+    "IsEnabled": true
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.ProductionAws.json

@@ -0,0 +1,115 @@
+{
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:00.1"
+  },
+  "Spark": {
+    "ConfigServer": "http://172.30.2.181:8000/config"
+  },
+  "SparkApi": {
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    },
+    "EnableAdmin": true,
+    "EnableOrders": true,
+    "EnablePosition": true
+  },
+  "PubSub": {
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "ServerPort": 22019,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://172.30.2.181:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://172.30.2.181:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  },
+  "Kestrel": {
+    "Endpoints": {
+      "Http": {
+        "Url": "http://172.30.2.181:8080"
+      },
+      "Https": {
+        "Url": "https://172.30.2.181:8443"
+      }
+    }
+  },
+  "CannedQueries": [
+    {
+      "Name": "BBO",
+      "Query": [
+        "bbo"
+      ]
+    },
+    {
+      "Name": "Trades",
+      "Query": [
+        "timesales"
+      ]
+    },
+    {
+      "Name": "Instruments",
+      "Query": [
+        "instruments"
+      ]
+    },
+    {
+      "Name": "Books",
+      "Query": [
+        "books"
+      ]
+    },
+    {
+      "Name": "Benzinga News",
+      "Query": [
+        "benzinga_news"
+      ]
+    }
+  ]
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.Staging.json

@@ -0,0 +1,44 @@
+{
+  "CannedQueries": [
+    {
+      "Name": "BBO",
+      "Query": [
+        "bbo"
+      ]
+    },
+    {
+      "Name": "Trades",
+      "Query": [
+        "timesales"
+      ]
+    },
+    {
+      "Name": "Instruments",
+      "Query": [
+        "instruments"
+      ]
+    },
+    {
+      "Name": "Books",
+      "Query": [
+        "books"
+      ]
+    },
+    {
+      "Name": "Benzinga News",
+      "Query": [
+        "benzinga_news"
+      ]
+    }
+  ],
+  "Spark": {
+    "ConfigServer": "http://sparkd:8000/config"
+  },
+  "SparkApi": {
+    "IsEnabled": true,
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.json

@@ -0,0 +1,85 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*",
+  "Kestrel": {
+    "Endpoints": {
+      "Http": {
+        "Url": "http://0.0.0.0:8080"
+      },
+      "Https": {
+        "Url": "https://0.0.0.0:8443"
+      }
+    }
+  },
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:00.1"
+  },
+  "SparkApi": {
+    "IsEnabled": false,
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    }
+  },
+  "PubSub": {
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "ServerPort": 22019,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://pubsub:22019",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://pubsub:22019",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+         "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf

@@ -0,0 +1,15 @@
+Logging {
+  value {
+    console {
+      default_level: "Info"
+    }
+    syslog {
+      default_level: "Info"
+    }
+    file {
+      default_level: "Info"
+    }
+    initial_queue_size: 120001
+    max_queue_growth: 4
+  }
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/alt-index.html

@@ -0,0 +1,168 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>ALT WebSocket Sample Application</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Message to send:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            var data = sendMessage.value;
+            socket.send(data);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/Bts.DataStreamWebApp.Server/wwwroot/index.html

@@ -0,0 +1,176 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>DataStreams via WebSocket</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Query:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            const rpcReq = {
+              jsonrpc: "2.0",
+              method: "subscribe",
+              params: {
+                query: sendMessage.value,
+              },
+              id: 1,
+            };
+            const rpcReqStr = JSON.stringify(rpcReq);
+            socket.send(rpcReqStr);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>

src/DataStreamSuite/Bts.Extensions.Hosting/Bts.Extensions.Hosting.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
+  </ItemGroup>
+
+</Project>

src/DataStreamSuite/Bts.Extensions.Hosting/HostBuilderExtensions.cs

@@ -0,0 +1,59 @@
+namespace Bts.Extensions.Hosting;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Hosting;
+
+using System.Diagnostics;
+
+public static class HostBuilderExtensions {
+   public static IHostBuilder CreateBtsBuilder(string[] args) {
+      // Obtain calling assembly name here, rather than in the lambda where it is actually needed,
+      // because want the name of the assembly that called this function, not the assembly that will eventually invoke
+      // the lambda...that will be the Microsoft.Extensions.Hosting assembly.
+      // Get the name of the assembly that called this function. That will act as the "appsettings"
+      // json file prefix
+      var exeName = GetCallingAssemblyName();
+      return Host.CreateDefaultBuilder(args)
+         .ConfigureAppConfiguration((context, configBuilder) =>
+            ConfigureApp(configBuilder, context.HostingEnvironment.EnvironmentName, exeName));
+   }
+
+   public static void ConfigureApp(IConfigurationBuilder configBuilder, string? environment = null, string? prefix = null) {
+      var baseFileName = prefix is null ? "appsettings" : $"{prefix}.appsettings";
+      AddJsonFile($"{baseFileName}.json");
+      if (environment is not null) {
+         AddJsonFile($"{baseFileName}.{environment}.json");
+      }
+
+      var cfg = configBuilder.Build().GetSection("config")?.Value;
+      if (!SetBasePath(cfg)) {
+         AddJsonFile(cfg);
+      }
+
+      bool SetBasePath(string? path) {
+         if (!Directory.Exists(path)) {
+            return false;
+         }
+         configBuilder.SetBasePath(path);
+         return true;
+      }
+
+      bool AddJsonFile(string? path) {
+         if (!File.Exists(path)) {
+            return false;
+         }
+         configBuilder.AddJsonFile(path);
+         return true;
+      }
+   }
+
+   public static string GetCallingAssemblyName() =>
+      new StackTrace()
+         .GetFrame(2)
+         ?.GetMethod()
+         ?.Module
+         ?.Assembly
+         ?.GetName()
+         ?.Name
+         ?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
+}

src/DataStreamSuite/Bts.Net/Bts.Net.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

src/DataStreamSuite/Bts.Net/HostComparer.cs

@@ -0,0 +1,28 @@
+namespace Bts.Net;
+
+using System.Linq;
+using System.Net;
+
+/// <summary>
+/// Provides facilities to compare hosts for equality.
+/// <para>
+/// This class doesn't implement IEqualityComparer because a consistent hash code cannot be
+/// guarenteed for equivalent host strings. The equals impl relies on Dns.GetHostAddresses, which
+/// is non-determinstic (implementations that depends on mutable properties face a similar problem).
+/// </para>
+/// </summary>
+public class HostComparer {
+   public static bool Equals(string? x, string? y) {
+      if (x is null) {
+         return y is null;
+      }
+      if (y is null) {
+         return false;
+      }
+      return Dns.GetHostAddresses(x).Intersect(
+         Dns.GetHostAddresses(y), IpAddressComparer.Instance).Any();
+   }
+
+   public static bool Equals(IPHostEntry x, IPHostEntry y) =>
+      x.AddressList.Intersect(y.AddressList, IpAddressComparer.Instance).Any();
+}

src/DataStreamSuite/Bts.Net/IpAddressComparer.cs

@@ -0,0 +1,30 @@
+namespace Bts.Net;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Net;
+
+/// <summary>
+/// Compares ip addresses. Unlike the default comparer, v4 and equivalent v6 addresses are
+/// considered equal.
+/// </summary>
+public class IpAddressComparer : IEqualityComparer<IPAddress> {
+   public static IEqualityComparer<IPAddress> Instance { get; } = new IpAddressComparer();
+
+   public bool Equals(IPAddress? x, IPAddress? y) {
+      if (x is null) {
+         return y is null;
+      }
+      if (y is null) {
+         return false;
+      }
+      return x.Equals(y) ||
+         (x.IsIPv4MappedToIPv6 && x.MapToIPv4().Equals(y)) ||
+         (y.IsIPv4MappedToIPv6 && y.MapToIPv4().Equals(x));
+   }
+
+   public int GetHashCode([DisallowNull] IPAddress address) =>
+      address.IsIPv4MappedToIPv6
+         // Return consistent hash code for v4 and equivalent v6
+         ? address.MapToIPv4().GetHashCode()
+         : address.GetHashCode();
+}

src/DataStreamSuite/Bts.Net/IpHostEntryJsonConverter.cs

@@ -0,0 +1,33 @@
+namespace Bts.Net;
+
+using System;
+using System.Net;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class IpHostEntryJsonConverter : JsonConverter<IPHostEntry> {
+   public override IPHostEntry? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      throw new NotImplementedException();
+   }
+
+   public override void Write(Utf8JsonWriter writer, IPHostEntry value, JsonSerializerOptions options) {
+      writer.WriteStartObject();
+      // HostName
+      writer.WriteString(nameof(value.HostName), value.HostName);
+      // Aliases
+      writer.WritePropertyName("Aliases");
+      writer.WriteStartArray();
+      foreach (var alias in value.Aliases) {
+         writer.WriteStringValue(alias);
+      }
+      writer.WriteEndArray();
+      // Addresses
+      writer.WritePropertyName("Addresses");
+      writer.WriteStartArray();
+      foreach (var address in value.AddressList) {
+         writer.WriteStringValue(address.ToString());
+      }
+      writer.WriteEndArray();
+      writer.WriteEndObject();
+   }
+}

src/DataStreamSuite/Bts.Net/IpHostEntryJsonSerializer.cs

@@ -0,0 +1,39 @@
+namespace Bts.Net;
+
+using System;
+using System.IO;
+using System.Net;
+using System.Text;
+using System.Text.Json;
+
+/// <summary>
+/// This class seriailzes IpHostEntry to json (string). A common buffer is used to reduce memory
+/// allocations on repeated calls to <see cref="Serialize"/>. For that reason, this class is NOT
+/// thread safe.
+/// </summary>
+public class IpHostEntryJsonSerializer {
+   readonly MemoryStream buffer = new();
+   readonly IpHostEntryJsonConverter converter;
+
+   public IpHostEntryJsonSerializer(IpHostEntryJsonConverter? converter = null) =>
+      this.converter = converter ?? new() { };
+
+   public string? Serialize(IPHostEntry dp, JsonSerializerOptions? opts = null) {
+      opts ??= JsonSerializerOptions.Default;
+      buffer.Position = 0;
+      buffer.SetLength(0);
+      using (var jsonWriter = new Utf8JsonWriter(buffer)) {
+         converter.Write(jsonWriter, dp, opts);
+      }
+      var bytes = buffer.GetBuffer();
+      var span = new ArraySegment<byte>(bytes, 0, (int)buffer.Length);
+      return Encoding.UTF8.GetString(span);
+   }
+
+   public void Serialize(IPHostEntry dp, Utf8JsonWriter jsonWriter, JsonSerializerOptions? opts = null) {
+      opts ??= JsonSerializerOptions.Default;
+      buffer.Position = 0;
+      buffer.SetLength(0);
+      converter.Write(jsonWriter, dp, opts);
+   }
+}

src/DataStreamSuite/Bts.Net/IpUtils.cs

@@ -0,0 +1,94 @@
+namespace Bts.Net;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Net;
+using System.Text.RegularExpressions;
+
+public static partial class IpUtils {
+   static readonly Regex ipv4Regex = new(string.Join('.', @"(((0|1)\d)|(2\d)|\d)?\d"));
+
+   // public static void ParseUri(string str, out string? scheme, out string? host, out string? port, out string? path, out string? queryParams) {
+   //    if (Uri.TryCreate(str, UriKind.Absolute, out var uri)) {
+   //    }
+   // }
+
+   // public static bool AreEndpointsEqual(Uri? x, Uri? y) {
+   //    if (x is null) {
+   //       return y is null;
+   //    }
+   //    if (y is null) {
+   //       return false;
+   //    }
+   //    if (x.IsAbsoluteUri != y.IsAbsoluteUri) {
+   //       return false;
+   //    }
+   //    if (x.IsAbsoluteUri) {
+   //       if (x.Scheme != y.Scheme) {
+   //          return false;
+   //       }
+   //    }
+   //    string? xHost = null;
+   // }
+
+   // static string ResolveHostIpv4(Uri uri) {
+   //    // switch (uri.HostNameType) {
+   //    //    case UriHostNameType.IPv4:
+   //    //       return uri.Host;
+   //    //    case UriHostNameType.Dns:
+   //    //       break;
+   //    //    case UriHostNameType.IPv6:
+   //    //       var addresses = Dns.GetHostAddresses(uri.Host);
+   //    //       foreach (var address in addresses) {
+   //    //          if (address.IsIPv4MappedToIPv6) {
+   //    //             return address.MapToIPv4().Address;
+   //    //          }
+   //    //       }
+   //    //       break;
+   //    // }
+   //    var hostAddresses = Dns.GetHostAddresses(uri.Host);
+   //    foreach (IPAddress address in hostAddresses) {
+   //       if (address.Equals(remoteEndPoint.Address) ||
+   //           (address.IsIPv4MappedToIPv6 && address.MapToIPv4().Equals(remoteEndPoint.Address)) ||
+   //           (remoteEndPoint.Address.IsIPv4MappedToIPv6 && remoteEndPoint.Address.MapToIPv4().Equals(address))) {
+   //          isMatch = true;
+   //          break;
+   //       }
+   //    }
+   // }
+
+   // public static bool AreEndpointsEqual(string? x, string? y) {
+   //    if (x == y) {
+   //       return true;
+   //    }
+   //    if (x is null) {
+   //       return y is null;
+   //    }
+   //    if (y is null) {
+   //       return false;
+   //    }
+   //    var xUri = new Uri(x);
+   //    var yUri = new Uri(y);
+   //    // return xUri.Equals(yUri);
+   // }
+
+   public static bool IsIpv4(string host) {
+      host = removeSchemaFromUrl(host);
+      host = removePortFromUrl(host);
+      return ipv4Regex.IsMatch(host);
+   }
+
+   static string removeSchemaFromUrl(string url) {
+      var i = url.IndexOf("://");
+      return i < 0 ? url : url.Substring(i + 3);
+   }
+
+   static string removePortFromUrl(string url) {
+      var match = MyRegex().Match(url);
+      return match.Success
+         ? string.Concat(url.AsSpan(0, match.Index), url.AsSpan(match.Index + match.Length))
+         : url;
+   }
+
+   [GeneratedRegex(":\\d{1,4}")]
+   private static partial Regex MyRegex();
+}

src/DataStreamSuite/ColumnMapExpressions/ColumnMapExpressions.csproj

@@ -7,14 +7,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
     <PackageReference Include="YaccLexTools" Version="1.2.2" />
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
   <ItemGroup Label="ColumnMapExpression">

src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs

@@ -18,10 +18,10 @@ namespace BCG {
          : this(new TableStream(tableName)) { }
 
       protected override void myOnFirstSubscriber() {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
+         myVS.onChange += myHandleChanges;
       }
       protected override void myOnLastSubscriber() {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
+         myVS.onChange -= myHandleChanges;
       }
 
 

src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs

@@ -78,11 +78,12 @@ namespace bts.BlueOrb {
       #endregion
 
       #region overridden methods
-      protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
+      protected override void myOnFirstSubscriber() =>
+         myParentStream.onChange += myHandleCallbacks;
 
       protected override void myOnLastSubscriber() {
          myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+         myParentStream.onChange -= myHandleCallbacks;
       }
 
       protected override void myFireEvent(object o) {

src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs

@@ -16,10 +16,10 @@ namespace bts.BlueOrb {
       }
 
       protected override void myOnFirstSubscriber() {
-         _stream.onChange += new ChangeDelegate(_onstream_change);
+         _stream.onChange += _onstream_change;
       }
       protected override void myOnLastSubscriber() {
-         _stream.onChange -= new ChangeDelegate(_onstream_change);
+         _stream.onChange -= _onstream_change;
       }
 
 

src/DataStreamSuite/DataStream/Misc/DSLStream.cs

@@ -12,8 +12,7 @@ namespace bts.SilverPine {
       public DSLStream(string dsl) {
          Log.Gen.info($"DSLStream.cons dsl: {dsl}");
          _dsl = dsl;
-         ViewStream rs = build_stream(dsl);
-         rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
+         ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
          mySetRealSource(rs);
       }
 
@@ -29,4 +28,4 @@ namespace bts.SilverPine {
          }
       }
    }
-}
+}

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -1,14 +1,24 @@
+using Bts.DataStream;
+using Bts.DataStreamCore;
+using Bts.DataStreamExpressions;
+using Bts.PubSub;
+
+using bts;
+
+using Microsoft.Extensions.Hosting;
+
+using Spark.Core.Disposables;
+
 using System;
 using System.Collections.Generic;
-
-using Bts.DataStreamExpressions;
-using BCG.Utilities.General;
-using bts;
+using System.Reactive.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace BCG {
    // listens to the requests posted on "cloud_streams" request table, and 
    // if unfulfilled starts a ViewPubsliher
-   public class RemoteQueryManager : IWorker {
+   public class RemoteQueryManager : IHostedService {
       public class StreamPubUnit {
          public string name;
          public string stream_expression;
@@ -39,23 +49,46 @@ namespace BCG {
       }
       private IDictionary<string, StreamPubUnit>
          _stream_pub_map = new Dictionary<string, StreamPubUnit>();
-      private ViewStream _stream = new TableStream("cloud_streams");
-      private PubTableBroker _broker = new PubTableBroker();
-      private ManagedSubscriber _sub = ManagedSubscriber.instance;
+      readonly IPubTableBroker _broker;
+      readonly IManagedSubscriber _sub;
+      readonly object syncRoot = new();
+      IDisposable? cloudStreamsSub;
+      IDisposable? remoteRequestsSub;
+      bool isDisposed;
 
 
-      public RemoteQueryManager() {
-         _stream.onChange += _on_str;
-         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
-            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-         });
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
+         _sub = managedSubscriber;
+         _broker = pubTableBroker;
       }
-      private void _on_str(object obj)
-      {
-         if (obj is not DataPacket dp) {
-            return;
+
+      public Task StartAsync(CancellationToken cancellationToken) {
+         lock (syncRoot) {
+            Log.assert(!isDisposed);
+            Log.isnull(cloudStreamsSub);
+            Log.isnull(remoteRequestsSub);
+            cloudStreamsSub = new TableStream("cloud_streams").AsObservable().Subscribe(_on_str);
+            remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
+               RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+               _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+            });
          }
+         return Task.CompletedTask;
+      }
+
+      public Task StopAsync(CancellationToken cancellationToken) {
+         lock (syncRoot) {
+            if (!isDisposed) {
+               isDisposed = true;
+               Disposable.dispose(ref cloudStreamsSub);
+               Disposable.dispose(ref remoteRequestsSub);
+            }
+         }
+         return Task.CompletedTask;
+      }
+
+      private void _on_str(DataPacket dp)
+      {
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");
@@ -82,9 +115,5 @@ namespace BCG {
       private void _expose_to_ws(string name, ViewStream stream) {
          // TODO: john
       }
-
-      public void doWork() {
-
-      }
    }
 }

src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs

@@ -25,12 +25,12 @@ namespace BCG {
          byte[] rem_req_bytes = rem_req.serialize();
          _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
          _source_str = new TableStream(_name);
-         _source_str.onChange += new ChangeDelegate(_handle_data);
+         _source_str.onChange += _handle_data;
       }
 
       protected override void myOnLastSubscriber() {
          //maybe there should be a way to turn remote requests off ?
-         _source_str.onChange -= new ChangeDelegate(_handle_data);
+         _source_str.onChange -= _handle_data;
       }
 
       static string get_expression(params string[] lines) {

src/DataStreamSuite/DataStream/Primitives/Count/CountStream.cs

@@ -18,13 +18,11 @@ namespace bts.BlueOrb {
       public CountStream(string tableName)
          : this(new TableStream(tableName)) { }
 
-      protected override void myOnFirstSubscriber() {
-         myVS.onChange += new ChangeDelegate(myHandleChanges);
-      }
-      protected override void myOnLastSubscriber() {
-         myVS.onChange -= new ChangeDelegate(myHandleChanges);
-      }
+      protected override void myOnFirstSubscriber () =>
+         myVS.onChange += myHandleChanges;
 
+      protected override void myOnLastSubscriber () =>
+         myVS.onChange -= myHandleChanges;
 
       private void myHandleChanges(object newValue) {
          myFireEvent(

src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingCollectorStream.cs

@@ -30,7 +30,7 @@ namespace BCG
 
         protected override void myOnFirstSubscriber()
         {
-            myToCrush.onChange += new ChangeDelegate(myToCrush_onChange);
+            myToCrush.onChange += myToCrush_onChange;
         }
 
         protected override void myOnLastSubscriber()

src/DataStreamSuite/DataStream/Primitives/Crushing/CrushingStream.cs

@@ -15,11 +15,11 @@ namespace BCG
             
             //set up an anonymous delegate for each of the streams
             //we do this so that we can unsubscribe
-            myDelegates = new ChangeDelegate[myToCrush.Length];
+            myDelegates = new DataStreamChangeDelegate[myToCrush.Length];
             for (int i = 0; i < myToCrush.Length; ++i)
             {
                 int tempI = i;
-                myDelegates[tempI] = new ChangeDelegate((o) => { myHandleChange(tempI, ((DataPacket)o).newRows); });
+                myDelegates[tempI] = o => myHandleChange(tempI, ((DataPacket)o).newRows);
             }
         }
 
@@ -27,7 +27,7 @@ namespace BCG
         private ViewStream[] myToCrush;
         private ITableCrusher myCrusher;
 
-        private ChangeDelegate[] myDelegates;
+        private DataStreamChangeDelegate[] myDelegates;
 
         protected override void myOnFirstSubscriber()
         {

src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs

@@ -28,7 +28,7 @@ namespace BCG
 
       protected override void myOnFirstSubscriber()
       {
-         myDataSource.onChange += new ChangeDelegate(myDataSource_onChange);
+         myDataSource.onChange += myDataSource_onChange;
       }
 
       protected override void myOnLastSubscriber()

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -18,7 +18,8 @@ namespace bts {
          _filename = filename;
       }
 
-      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
+      protected override void myOnFirstSubscriber() =>
+         _stream.onChange += _handle_changes;
 
       void _handle_changes(object dp_obj) {
          DataPacket dp = dp_obj as DataPacket;
@@ -26,7 +27,8 @@ namespace bts {
          myFireEvent(dp_obj);
       }
 
-      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
+      protected override void myOnLastSubscriber() =>
+         _stream.onChange -= _handle_changes;
 
       protected void _save(DataPacket dp) {
          StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);

src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs

@@ -1,4 +1,7 @@
+using bts;
+
 using Bts.DataStream;
+using Bts.DataStream.Primitives.Filter;
 
 using mbts.config;
 
@@ -21,46 +24,61 @@ namespace BCG.StreamWorld.Addition {
       public FilterStream(ViewStream vs, IStreamFilter filter) {
          myBaseStream = vs;
          myFilter = filter;
+         vs.ErrorChanged += OnError;
          if (filter is IDynamicStreamFilter)
             (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
       }
 
 
-      protected override void myOnFirstSubscriber() {
-         myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
-      }
+      protected override void myOnFirstSubscriber() =>
+         myBaseStream.onChange += myHandleChanges;
 
-      protected override void myOnLastSubscriber() {
-         myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
+      protected override void myOnLastSubscriber()
+      {
+         myBaseStream.onChange -= myHandleChanges;
          myBaseTable.clear();
          myStateTable.clear();
       }
 
+      bool filterRow(IBlueRow r) {
+         try {
+            return myFilter.filter(r);
+         } catch (Exception ex) {
+            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
+            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
+         }
+         return false;
+      }
+
       private void myHandleChanges(object newValue) {
-         DataPacket dp = newValue as DataPacket;
-         if (myStateTable == null) {
-            bts.Log.assert(myBaseTable == null);
-            myStateTable = new BlueTable(dp.schema);
-            myBaseTable = new BlueTable(dp.schema);
-         }
-         BlueTable dels = new BlueTable(dp.schema),
-                     adds = new BlueTable(dp.schema);
-         foreach (IBlueRow r in dp.deletedRows) {
-            myBaseTable.deleteRow(r);
-            if (myStateTable.containsID((int)r["id"])) {
-               dels.addRow(r);
-               myStateTable.deleteRow(r);
+         try {
+            DataPacket dp = newValue as DataPacket;
+            if (myStateTable == null) {
+               bts.Log.assert(myBaseTable == null);
+               myStateTable = new BlueTable(dp.schema);
+               myBaseTable = new BlueTable(dp.schema);
             }
-         }
-         foreach (IBlueRow r in dp.addedRows) {
-            myBaseTable.addRow(r);
-            if (myFilter.filter(r)) {
-               adds.addRow(r);
-               myStateTable.addRow(r);
+            BlueTable dels = new BlueTable(dp.schema),
+                        adds = new BlueTable(dp.schema);
+            foreach (IBlueRow r in dp.deletedRows) {
+               myBaseTable.deleteRow(r);
+               if (myStateTable.containsID((int)r["id"])) {
+                  dels.addRow(r);
+                  myStateTable.deleteRow(r);
+               }
             }
+            foreach (IBlueRow r in dp.addedRows) {
+               myBaseTable.addRow(r);
+               if (filterRow(r)) {
+                  adds.addRow(r);
+                  myStateTable.addRow(r);
+               }
+            }
+            if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
+               myFireEvent(new DataPacket(myStateTable, adds, dels));
+         } catch (Exception ex) {
+            OnError(ex);
          }
-         if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
-            myFireEvent(new DataPacket(myStateTable, adds, dels));
       }
 
       public IStreamFilter filter {
@@ -69,9 +87,20 @@ namespace BCG.StreamWorld.Addition {
             if (myBaseTable != null) {
                IBlueRowCollection oldState = new BlueTable(myStateTable);
                myStateTable.clear();
-               foreach (IBlueRow r in myBaseTable)
-                  if (myFilter.filter(r))
-                     myStateTable.addRow(r);
+               // If the filter throws an exception, we can rethrow it as a StreamFilterException
+               // and include the 'currentRow' (for which the filter failed) to provide context.
+               IBlueRow? currentRow = null;
+               foreach (IBlueRow r in myBaseTable) {
+                  try {
+                     currentRow = r;
+                     if (filterRow(r)) {
+                        myStateTable.addRow(r);
+                     }
+                  } catch (Exception ex) {
+                     Log.Gen.error($"FilterStream delegate threw exception: {ex}");
+                     OnError(new StreamFilterException(StreamId, StreamName, null, ex, currentRow));
+                  }
+               }
                IBlueRowCollection dels, adds;
                BlueDataUtility.computeDifferences(myStateTable, oldState, out dels, out adds);
                myFireEvent(new DataPacket(myStateTable, adds, dels));
@@ -184,13 +213,13 @@ namespace BCG
       #endregion
 
       #region overridden methods
-      protected override void myOnFirstSubscriber()
-      { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
+      protected override void myOnFirstSubscriber() =>
+         myParentStream.onChange += myHandleCallbacks;
 
       protected override void myOnLastSubscriber()
       {
          myLastFiredRows = null;
-         myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
+         myParentStream.onChange -= myHandleCallbacks;
       }
 
       protected override void myFireEvent(object o)
@@ -206,32 +235,36 @@ namespace BCG
          DataPacket retVal;
          DataPacket arrivedPacket = newValue as DataPacket;
          myLastReceivedRows = arrivedPacket.newRows;
-         if (myLastFiredRows != null)
-         {
-            retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
-               new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.deletedRows)
-               if (myLastFiredRows.containsID((int)r["id"]))
-               {
-                  myLastFiredRows.deleteRow(r);
-                  (retVal.deletedRows as BlueTable).addRow(r);
-               }
-            foreach (IBlueRow r in arrivedPacket.addedRows)
-               if (myFilter.filter(r))
-               {
-                  myLastFiredRows.addRow(r);
-                  (retVal.addedRows as BlueTable).addRow(r);
-               }
+         try {
+            if (myLastFiredRows != null)
+            {
+               retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
+                  new BlueTable(myLastFiredRows.schema));
+               foreach (IBlueRow r in arrivedPacket.deletedRows)
+                  if (myLastFiredRows.containsID((int)r["id"]))
+                  {
+                     myLastFiredRows.deleteRow(r);
+                     (retVal.deletedRows as BlueTable).addRow(r);
+                  }
+               foreach (IBlueRow r in arrivedPacket.addedRows)
+                  if (filterRow(r))
+                  {
+                     myLastFiredRows.addRow(r);
+                     (retVal.addedRows as BlueTable).addRow(r);
+                  }
+            }
+            else
+            {
+               myLastFiredRows = new BlueTable(arrivedPacket.schema);
+               retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
+               foreach (IBlueRow r in arrivedPacket.newRows)
+                  if (filterRow(r))
+                     myLastFiredRows.addRow(r);
+            }
+            myFireEvent(retVal);
+         } catch (Exception ex) {
+            OnError(ex);
          }
-         else
-         {
-            myLastFiredRows = new BlueTable(arrivedPacket.schema);
-            retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
-            foreach (IBlueRow r in arrivedPacket.newRows)
-               if (myFilter.filter(r))
-                  myLastFiredRows.addRow(r);
-         }
-         myFireEvent(retVal);
       }
 
       public IStreamFilter filter
@@ -249,16 +282,20 @@ namespace BCG
                {
                    if (myLastReceivedRows != null)
                    {
+                     try {
                        BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
                        foreach (IBlueRow r in myLastReceivedRows)
-                           if (myFilter.filter(r))
+                           if (filterRow(r))
                                nRows.addRow(r);
                        IBlueRowCollection dRows = new BlueTable(nRows.schema),
                           aRows = new BlueTable(nRows.schema);                      
                        BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
                        myLastFiredRows = nRows;
                        myFireEvent(new DataPacket(nRows, aRows, dRows));
-                   }
+                     } catch (Exception ex) {
+                        OnError(ex);
+                     }
+                  }
                }
             }
          }
@@ -282,5 +319,15 @@ namespace BCG
       {
          filter = myFilter;
       }
+
+      bool filterRow(IBlueRow r) {
+         try {
+            return myFilter.filter(r);
+         } catch (Exception ex) {
+            Log.Gen.error($"FilterStream delegate threw exception: {ex}");
+            OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
+         }
+         return false;
+      }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Filter/StreamFilterException.cs

@@ -0,0 +1,19 @@
+#nullable enable
+namespace Bts.DataStream.Primitives.Filter;
+
+using BCG;
+
+using System;
+
+public class StreamFilterException : Exception {
+   public StreamFilterException(Guid streamId, string streamName, string? message, Exception? innerException, IBlueRow? badRow = null)
+      : base(message, innerException) {
+      StreamId = streamId;
+      StreamName = streamName;
+      BadRow = badRow;
+   }
+
+   public Guid StreamId { get; }
+   public string StreamName { get; }
+   public IBlueRow? BadRow { get; }
+}