fix use dep injection in bts.datastreamwebapp.server

fix Add spark service modules designed for DependencyInjection
Fix mismatched types in PubsubHeartbeatStream schema
2024-07-11 11:06:29 -04:00 · 2024-07-11 11:05:23 -04:00 · 2024-07-11 10:59:41 -04:00 · 2024-07-11 10:43:24 -04:00 · 2024-07-11 10:38:30 -04:00 · 2024-07-11 10:37:29 -04:00
231 changed files with 13219 additions and 1963 deletions
@@ -66,6 +66,7 @@ RUN \

 # Dotnet
 RUN apt-get update && apt-get install -y dotnet7
+RUN apt-get update && apt-get install -y dotnet8

 # Node
 ARG node_major=18
@@ -19,6 +19,14 @@
    "--shm-size",
    "2gb",
    "-p",
+    "5000:5000",
+    "-p",
+    "5001:5001",
+    "-p",
+    "5173:5173",
+    "-p",
+    "5174:5174",
+    "-p",
    "8000:8000",
    "-p",
    "8001:8001",
@@ -47,4 +55,4 @@
      ]
    }
  }
-}
+}
@@ -207,17 +207,23 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private

+# local symbols
+dotnet_naming_symbols.local_symbols.applicable_kinds = local
+
+# parameter symbols
+dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
+
 # Naming styles

-dotnet_naming_style.snake_case.required_prefix =
-dotnet_naming_style.snake_case.required_suffix =
-dotnet_naming_style.snake_case.word_separator = _
-dotnet_naming_style.snake_case.capitalization = all_lower
+dotnet_naming_style.pascal_case.required_prefix =
+dotnet_naming_style.pascal_case.required_suffix =
+dotnet_naming_style.pascal_case.word_separator =
+dotnet_naming_style.pascal_case.capitalization = pascal_case

-dotnet_naming_style.private_snake_case.required_prefix = _
-dotnet_naming_style.private_snake_case.required_suffix =
-dotnet_naming_style.private_snake_case.word_separator = _
-dotnet_naming_style.private_snake_case.capitalization = all_lower
+dotnet_naming_style.camel_case.required_prefix =
+dotnet_naming_style.camel_case.required_suffix =
+dotnet_naming_style.camel_case.word_separator =
+dotnet_naming_style.camel_case.capitalization = camel_case

 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -229,6 +235,10 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case

+dotnet_naming_style.local.capitalization = camel_case
+
+dotnet_naming_style.parameter.capitalization = camel_case
+
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -238,14 +248,18 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion

-dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
-dotnet_naming_rule.public_members_snake_case.style = snake_case
-dotnet_naming_rule.public_members_snake_case.severity = suggestion
+dotnet_naming_rule.public_members.symbols = public_symbols
+dotnet_naming_rule.public_members.style = pascal_case
+dotnet_naming_rule.public_members.severity = suggestion

-dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
-dotnet_naming_rule.private_members_snake_case.style = private_snake_case
-dotnet_naming_rule.private_members_snake_case.severity = suggestion
+dotnet_naming_rule.private_members.symbols = private_symbols
+dotnet_naming_rule.private_members.style = camel_case
+dotnet_naming_rule.private_members.severity = suggestion

-dotnet_naming_rule.parameters.symbols = parameters
-dotnet_naming_rule.parameters.style = snake_case
+dotnet_naming_rule.variables_are_camel_case.severity = suggestion
+dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
+dotnet_naming_rule.variables_are_camel_case.style = local
+
+dotnet_naming_rule.parameters.symbols = parameter_symbols
+dotnet_naming_rule.parameters.style = parameter
 dotnet_naming_rule.parameters.severity = suggestion
@@ -1,3 +1,3 @@
-*.cmd eol=crlf
-*.cs eol=crlf
-*.sh eol=lf
+*.cmd text=auto eol=crlf
+*.cs text=auto eol=crlf
+*.sh text=auto eol=lf
@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = git@bitbucket.org:bluetradesys/externals.git
+	url = https://bitbucket.org/bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git
@@ -6,6 +6,35 @@
      "type": "coreclr",
      "request": "attach"
    },
+    {
+      "name": "Bts.DataStreamWebApp.Server",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream WebApp (Debug)",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    {
+      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
    {
      "name": "DataStreamPublisher",
      "type": "coreclr",
@@ -22,7 +51,26 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
      ],
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DataStreamPublisher (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "DataStreamPublisher",
+      },
+      "windows": {
+        "program": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -30,7 +78,7 @@
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
+      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
      "program": "",
      "windows": {
        "program": "DataStreamQueryAnalyzer.exe",
@@ -66,6 +114,25 @@
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
+    {
+      "name": "PubSubServer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "PubSubServer",
+      },
+      "windows": {
+        "program": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
    {
      "name": "Spark",
      "type": "coreclr",
@@ -121,6 +121,97 @@
        "color": "terminal.ansiCyan"
      },
    },
+    {
+      "label": "Build DataStreamQueryAnalyzer",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
    {
      "label": "Build Spark",
      "group": "build",
@@ -303,6 +394,22 @@
        "color": "terminal.ansiCyan"
      },
    },
+    {
+      "label": "Publish DataStreamWebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/publish_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/publish_data_stream_web_app.cmd",
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
    {
      "label": "Edit DataStream Config",
      "type": "process",
@@ -353,7 +460,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStream",
+      "dependsOn": "Build DataStreamQueryAnalyzer",
      "type": "process",
      "command": "",
      "windows": {
@@ -373,7 +480,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
      "type": "process",
      "command": "",
      "windows": {
@@ -393,7 +500,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStream (Release)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
      "type": "process",
      "command": "",
      "windows": {
@@ -526,6 +633,28 @@
        "color": "terminal.ansiGreen",
      },
    },
+    {
+      "label": "Run PubSubServer (Debug | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
    {
      "label": "Run PubSubServer (Release)",
      "dependsOn": "Build DataStream (Release)",
@@ -549,6 +678,28 @@
        "color": "terminal.ansiGreen",
      },
    },
+    {
+      "label": "Run PubSubServer (Release | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
    {
      "label": "Run Spark",
      "dependsOn": "Build Spark",
@@ -684,6 +835,36 @@
        "id": "wrench"
      },
    },
+    {
+      "label": "View PubSubServer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/logging/show_last_run.sh",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
+          "Starting PubSubServer"
+        ]
+      },
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "PubSubServer.exe",
+          "Starting PubSubServer"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
  ],
  "inputs": [
    {
@@ -31,8 +31,7 @@
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -222,6 +221,7 @@
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -258,7 +258,7 @@
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -8,7 +8,6 @@
   public int Integer;
   public double Real;
   public string Identifier;
-   public DateTime DateTime; 
   public ImmutableList<IArithmeticExpression> Arguments;
   public ImmutableList<Assignment> Assignments;
   public IArithmeticExpression Argument;
@@ -1,4 +1,6 @@
 using System;
+using System.Runtime.Versioning;
+
 using Microsoft.Win32;

 namespace BCG
@@ -10,6 +12,7 @@ namespace BCG
         rk.SetValue (name, value.ToString ());
      }

+      [SupportedOSPlatform("windows")]
      public static double getValue (RegistryKey rk, string name)
      {
         object v = rk.GetValue (name);
@@ -0,0 +1,20 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.3.1' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
+  <PropertyGroup>
+    <StartupCommand>npm run dev</StartupCommand>
+    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
+    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
+    <!-- Allows the build (or compile) script located on package.json to run on Build -->
+    <ShouldRunBuildScript>true</ShouldRunBuildScript>
+    <!-- Folder where production build objects will be placed -->
+    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
+  </PropertyGroup>
+</Project>
@@ -0,0 +1,8 @@
+# React + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/ico" href="/logo.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>BTS Cloud Streams</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>
@@ -0,0 +1,38 @@
+{
+  "name": "bts.datastreamwebapp.client",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@angular/material": "^18.0.4",
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@fontsource/noto-sans": "^5.0.22",
+    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
+    "@mui/icons-material": "^5.15.20",
+    "@mui/material": "^5.15.20",
+    "@mui/x-data-grid": "^7.7.1",
+    "bulma": "^1.0.1",
+    "react": "^18.3.1",
+    "react-data-table-component": "^7.6.2",
+    "react-dom": "^18.3.1",
+    "react-use-websocket": "^4.8.1",
+    "rpc-websockets": "^7.11.2"
+  },
+  "devDependencies": {
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^8.57.0",
+    "eslint-plugin-react": "^7.34.2",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-react-refresh": "^0.4.7",
+    "vite": "^5.2.13"
+  }
+}
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
@@ -0,0 +1,28 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+h1#main-header {
+  padding: 4px 8px 4px 8px;
+  text-align: center;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}
@@ -0,0 +1,24 @@
+import { createTheme, ThemeProvider, CssBaseline } from '@mui/material';
+import useMediaQuery from '@mui/material/useMediaQuery';
+import QueryControl from './QueryControl';
+import getTheme from './Theme';
+import '@fontsource/noto-sans';
+import './App.css';
+
+const App = () => {
+  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
+  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
+
+  return (
+    <ThemeProvider theme={theme}>
+      <CssBaseline />
+      <h1 id='main-header' style={{
+        color: theme.palette.text.primary,
+        fontFamily: 'Noto Sans',
+      }}>BTS Cloud Streams</h1>
+      <QueryControl websocketUrl="ws://localhost:5000/ws" theme={theme} />
+    </ThemeProvider>
+  );
+};
+
+export default App;
@@ -0,0 +1,32 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+th {
+  font-size: large;  
+  padding: 15px;
+}
+
+td {
+  padding: 15px;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}
@@ -0,0 +1,52 @@
+import PropTypes from 'prop-types';
+import { DataGrid } from '@mui/x-data-grid';
+
+function getGridColDef(col) {
+  return {
+    field: col.name,
+    headerName: col.name,
+  };
+}
+
+export default function DataTable({ columns, rows }) {
+  return (
+    <DataGrid
+      columns={columns.map(getGridColDef)}
+      rows={rows}
+      sx={{
+        borderColor: 'primary.main',
+        backgroundColor: 'primary.dark',
+        color: 'text.primary',
+        '& .MuiDataGrid-columnHeader': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-filler': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-cell': {
+          color: 'text.primary',
+        },
+        '& .MuiDataGrid-cell:hover': {
+          color: 'primary.main',
+        },
+        '& .MuiDataGrid-row': {
+          '--DataGrid-rowBorderColor': 'secondary.dark'
+        }
+      }}
+    />
+  );
+}
+
+DataTable.propTypes = {
+  columns: PropTypes.arrayOf(
+    PropTypes.shape({
+      name: PropTypes.string.isRequired,
+      type: PropTypes.string.isRequired,
+    })
+  ).isRequired,
+  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
+};
@@ -0,0 +1,100 @@
+import PropTypes from 'prop-types';
+import DataTable from './DataTable';
+import QueryInputForm from './QueryInputForm';
+import Container from '@mui/material/Container';
+import React from 'react';
+
+var rpcReqIdCounter = 1;
+
+function subscribeReq(query) {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "subscribe",
+    params: {
+      subscription_id: "1",
+      query: query
+    }
+  };
+}
+
+function unsubscribeReq() {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "unsubscribe",
+    params: {
+      subscription_id: "1",
+    }
+  };
+}
+
+class QueryControl extends React.Component {
+  constructor({ websocketUrl, theme }) {
+    super(websocketUrl);
+    console.log('Connecting to WebSocket:', websocketUrl);
+    this.state = {
+      rows: null,
+      ws: new WebSocket(websocketUrl),
+      columns: null,
+      isRunning: false,
+      theme: theme,
+    };
+    this.state.ws.onopen = () => {
+      console.info('WebSocket open');
+    };
+    this.state.ws.onclose = () => {
+      console.info('WebSocket closed');
+    };
+    this.state.ws.onmessage = e => {
+      console.info(`Received message: ${e.data}`);
+      const msg = JSON.parse(e.data);
+      if (msg.id != null) {
+        let newState = { ...this.state };
+        newState.isRunning = msg.result.status == 'Subscribed';
+        this.setState(newState);
+      } else {
+        let newState = { ...this.state };
+        newState.rows = msg.params.all;
+        console.info(`Received rows: ${newState.rows}`);
+        newState.columns = msg.params.columns;
+        this.setState(newState);
+      }
+    };
+
+    this.start = (query) => {
+      this.state.ws.send(JSON.stringify(subscribeReq(query)));
+    };
+
+    this.stop = () => {
+      this.state.ws.send(JSON.stringify(unsubscribeReq()));
+    };
+  }
+
+  componentWillUnmount() {
+    if (this.state.ws) {
+      console.warn('Closing websocket because component is unmounting');
+      this.state.ws.close();
+    }
+  }
+
+  render() {
+    console.info('Rendering');
+    return (
+      <Container>
+        <QueryInputForm isRunning={this.state.isRunning} onStart={this.start} onStop={this.stop} />
+        {
+          this.state.rows == null ? <p style={{ color: this.state.theme.palette.warning.main }}>No data received</p> :
+            <DataTable columns={this.state.columns} rows={this.state.rows} />
+        }
+      </Container>
+    );
+  }
+}
+
+QueryControl.propTypes = {
+  websocketUrl: PropTypes.string.isRequired,
+  theme: PropTypes.object.isRequired,
+};
+
+export default QueryControl;
@@ -0,0 +1,93 @@
+import { useRef } from 'react';
+import Box from '@mui/material/Box';
+import Button from '@mui/material/Button';
+import Grid from '@mui/material/Grid';
+import Paper from '@mui/material/Paper';
+import PlayArrowIcon from '@mui/icons-material/PlayArrow';
+import PropTypes from 'prop-types';
+import StopIcon from '@mui/icons-material/Stop';
+import TextField from '@mui/material/TextField';
+import Tooltip from '@mui/material/Tooltip';
+
+const QueryInputForm = ({ isRunning, onStart, onStop }) => {
+  let textInput = useRef(null);
+
+  const onClearClick = () => {
+    textInput.current.value = '';
+  };
+
+  const onRunClick = () => {
+    onStart(textInput.current.value);
+  };
+
+  const onStopClick = () => {
+    onStop();
+  };
+
+  const onKeyDown = (e) => {
+    // Detect keyboard shortcuts.
+    //   Ctrl+Enter : Run
+    if (e.ctrlKey && e.key === 'Enter') {
+      onRunClick();
+    }
+  };
+
+  return (
+    <Box sx={{ p: 2 }}>
+      <Paper elevation={3} sx={{ color: 'primary.main' }}>
+        <Box sx={{ p: 2 }}>
+          <TextField
+            fullWidth
+            multiline
+            rows={10}
+            variant="outlined"
+            onKeyDown={onKeyDown}
+            inputRef={textInput}
+            placeholder="Enter Query..."
+          />
+        </Box>
+        <Box sx={{ p: 2, borderTop: 1 }}>
+          <Grid container spacing={2} justifyContent="flex-start">
+            <Grid item xs>
+              <Tooltip title="Clear Query Text">
+                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
+                  Clear
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Run (Ctrl+Enter)">
+                <Button onClick={onRunClick} sx={{
+                  color: `query_start_button.${isRunning ? 'active' : 'inactive'}.color`,
+                  bgcolor: `query_start_button.${isRunning ? 'active' : 'inactive'}.background_color`,
+                  padding: '11px'
+                }}>
+                  <PlayArrowIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Stop">
+                <Button onClick={onStopClick} sx={{
+                  color: `query_stop_button.${isRunning ? 'inactive' : 'active'}.color`,
+                  bgcolor: `query_stop_button.${isRunning ? 'inactive' : 'active'}.background_color`,
+                  padding: '11px'
+                }}>
+                  <StopIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+          </Grid>
+        </Box>
+      </Paper>
+    </Box>
+  );
+};
+
+QueryInputForm.propTypes = {
+  isRunning: PropTypes.bool.isRequired,
+  onStart: PropTypes.func.isRequired,
+  onStop: PropTypes.func.isRequired,
+};
+
+export default QueryInputForm;
@@ -0,0 +1,88 @@
+import { createTheme } from '@mui/material/styles';
+import PropTypes from 'prop-types';
+import '@fontsource/noto-sans';
+import './App.css';
+
+const getLightPalette = () => {
+  return {
+    mode: 'light',
+    primary: {
+      light: '#6573c3',
+      main: '#3f51b5',
+      dark: '#2c387e',
+      contrastText: '#ffffff',
+    },
+    secondary: {
+      light: '#f73378',
+      main: '#f50057',
+      dark: '#ab003c',
+      contrastText: '#ffffff',
+    },
+    text: {
+      primary: '#000000c4',
+      secondary: '#00000086',
+      disabled: '#0000005b',
+      hint: '#0000005b',
+    }
+  };
+};
+
+const getDarkPalette = () => {
+  return {
+    mode: 'dark',
+    primary: {
+      light: '#bcc5ff',
+      main: '#33baf9',
+      dark: '#202334',
+      contrastText: '#78a2ac',
+    },
+    secondary: {
+      light: '#9fd0b8',
+      main: '#5eb289',
+      dark: '#ffffff12',
+      contrastText: '#c8d4f7',
+    },
+    text: {
+      primary: '#eaf8f8',
+      dark: '#eaf8f8a0'
+    },
+    query_start_button: {
+      active: {
+        color: '#33baf9',
+        background_color: '#ffffff40',
+      },
+      inactive: {
+        color: '#ffffff40',
+        background_color: '#ffffff00',
+      }
+    },
+    warning: {
+      main: '#c4823f'
+    },
+    query_stop_button: {
+      active: {
+        color: '#ff4444a0',
+        background_color: '#ffffff40',
+      },
+      inactive: {
+        color: '#ff444450',
+        background_color: '#ffffff00',
+      }
+    },
+  };
+};
+
+const getTheme = ({ mode }) => {
+  return createTheme({
+    typography: {
+      fontFamily: '"Helvetica"'
+    },
+    palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
+  });
+};
+
+export default getTheme;
+
+getTheme.propTypes = {
+  mode: PropTypes.string.isRequired,
+};
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>
@@ -0,0 +1,51 @@
+.dark-high-contrast {
+  --md-sys-color-primary: rgb(251 250 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 0 0);
+  --md-sys-color-primary-container: rgb(173 204 255);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(250 250 255);
+  --md-sys-color-on-secondary: rgb(0 0 0);
+  --md-sys-color-secondary-container: rgb(159 208 255);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(255 249 251);
+  --md-sys-color-on-tertiary: rgb(0 0 0);
+  --md-sys-color-tertiary-container: rgb(222 193 231);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 249 249);
+  --md-sys-color-on-error: rgb(0 0 0);
+  --md-sys-color-error-container: rgb(255 186 177);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(255 255 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(251 250 255);
+  --md-sys-color-outline: rgb(200 202 212);
+  --md-sys-color-outline-variant: rgb(200 202 212);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(0 0 0);
+  --md-sys-color-inverse-primary: rgb(0 42 83);
+  --md-sys-color-primary-fixed: rgb(219 231 255);
+  --md-sys-color-on-primary-fixed: rgb(0 0 0);
+  --md-sys-color-primary-fixed-dim: rgb(173 204 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
+  --md-sys-color-secondary-fixed: rgb(214 233 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 0 0);
+  --md-sys-color-secondary-fixed-dim: rgb(159 208 255);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
+  --md-sys-color-tertiary-fixed: rgb(249 222 255);
+  --md-sys-color-on-tertiary-fixed: rgb(0 0 0);
+  --md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}
@@ -0,0 +1,51 @@
+.dark-medium-contrast {
+  --md-sys-color-primary: rgb(173 204 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 22 49);
+  --md-sys-color-primary-container: rgb(112 146 198);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(159 208 255);
+  --md-sys-color-on-secondary: rgb(0 24 42);
+  --md-sys-color-secondary-container: rgb(100 149 193);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(222 193 231);
+  --md-sys-color-on-tertiary: rgb(33 14 42);
+  --md-sys-color-tertiary-container: rgb(162 136 171);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 186 177);
+  --md-sys-color-on-error: rgb(55 0 1);
+  --md-sys-color-error-container: rgb(255 84 73);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(251 250 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(200 202 212);
+  --md-sys-color-outline: rgb(160 163 171);
+  --md-sys-color-outline-variant: rgb(128 131 139);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(40 42 47);
+  --md-sys-color-inverse-primary: rgb(35 73 120);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 17 40);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 18 34);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(28 9 37);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}
@@ -0,0 +1,51 @@
+.dark {
+  --md-sys-color-primary: rgb(165 200 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 49 94);
+  --md-sys-color-primary-container: rgb(34 72 118);
+  --md-sys-color-on-primary-container: rgb(212 227 255);
+  --md-sys-color-secondary: rgb(154 203 250);
+  --md-sys-color-on-secondary: rgb(0 51 83);
+  --md-sys-color-secondary-container: rgb(12 74 114);
+  --md-sys-color-on-secondary-container: rgb(206 229 255);
+  --md-sys-color-tertiary: rgb(218 189 226);
+  --md-sys-color-on-tertiary: rgb(61 41 70);
+  --md-sys-color-tertiary-container: rgb(85 63 93);
+  --md-sys-color-on-tertiary-container: rgb(247 216 255);
+  --md-sys-color-error: rgb(255 180 171);
+  --md-sys-color-on-error: rgb(105 0 5);
+  --md-sys-color-error-container: rgb(147 0 10);
+  --md-sys-color-on-error-container: rgb(255 218 214);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(225 226 233);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(195 198 207);
+  --md-sys-color-outline: rgb(141 145 153);
+  --md-sys-color-outline-variant: rgb(67 71 78);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(46 48 53);
+  --md-sys-color-inverse-primary: rgb(60 96 144);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}
@@ -0,0 +1,51 @@
+.light-high-contrast {
+  --md-sys-color-primary: rgb(0 34 70);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(29 68 114);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(0 36 61);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(3 70 110);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(46 26 55);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(81 59 89);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(78 0 2);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(140 0 9);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(0 0 0);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(32 36 43);
+  --md-sys-color-outline: rgb(63 67 74);
+  --md-sys-color-outline-variant: rgb(63 67 74);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(255 255 255);
+  --md-sys-color-inverse-primary: rgb(228 236 255);
+  --md-sys-color-primary-fixed: rgb(29 68 114);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(0 45 88);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(3 70 110);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(0 47 77);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(81 59 89);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}
@@ -0,0 +1,51 @@
+.light-medium-contrast {
+  --md-sys-color-primary: rgb(29 68 114);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(83 118 167);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(3 70 110);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(71 121 163);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(81 59 89);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(133 108 141);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(140 0 9);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(218 52 46);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(63 67 74);
+  --md-sys-color-outline: rgb(91 95 103);
+  --md-sys-color-outline-variant: rgb(119 123 131);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(83 118 167);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(58 93 141);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(71 121 163);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(43 96 137);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(133 108 141);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}
@@ -0,0 +1,51 @@
+.light {
+  --md-sys-color-primary: rgb(60 96 144);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(212 227 255);
+  --md-sys-color-on-primary-container: rgb(0 28 58);
+  --md-sys-color-secondary: rgb(46 98 140);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(206 229 255);
+  --md-sys-color-on-secondary-container: rgb(0 29 50);
+  --md-sys-color-tertiary: rgb(110 86 118);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(247 216 255);
+  --md-sys-color-on-tertiary-container: rgb(39 20 48);
+  --md-sys-color-error: rgb(186 26 26);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(255 218 214);
+  --md-sys-color-on-error-container: rgb(65 0 2);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(67 71 78);
+  --md-sys-color-outline: rgb(116 119 127);
+  --md-sys-color-outline-variant: rgb(195 198 207);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}
@@ -0,0 +1,107 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+button#query-button {
+  height: 5em;
+  vertical-align: top;
+}
+
+button#play-button {
+  padding: '10px 20px';
+  font-size: '16px';
+  cursor: 'pointer';
+  background-color: '#28a745';
+  color: '#fff';
+  border: 'none';
+  border-radius: '5px';
+};
+
+button#pause-button {
+  background-color: #ffc107;
+}
+
+button#stop-button {
+  background-color: #dc3545;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+p#footer {
+  float: none;
+  clear: both;
+  vertical-align: bottom;
+}
+
+table#query-output {
+  float: none;
+  clear: both;
+}
+
+textarea#query-input {
+  height: 5em;
+  width: 50em;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}
@@ -0,0 +1,9 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App';
+
+ReactDOM.createRoot(document.getElementById('root')).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+)
@@ -0,0 +1,69 @@
+import { fileURLToPath, URL } from 'node:url';
+import { defineConfig } from 'vite';
+import plugin from '@vitejs/plugin-react';
+import fs from 'fs';
+import path from 'path';
+import child_process from 'child_process';
+import process, { env } from 'process';
+
+const baseFolder =
+  process.env.APPDATA !== undefined && process.env.APPDATA !== ''
+    ? `${process.env.APPDATA}/ASP.NET/https`
+    : `${process.env.HOME}/.aspnet/https`;
+
+const certificateName = "bts.datastreamwebapp.client";
+const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
+const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
+
+if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
+  if (0 !== child_process.spawnSync('dotnet', [
+    'dev-certs',
+    'https',
+    '--export-path',
+    certFilePath,
+    '--format',
+    'Pem',
+    '--no-password',
+  ], { stdio: 'inherit', }).status) {
+    throw new Error("Could not create certificate.");
+  }
+}
+
+const target = env.ASPNETCORE_HTTPS_PORT
+  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
+  : env.ASPNETCORE_URLS
+    ? env.ASPNETCORE_URLS.split(';')[0]
+    : 'https://localhost:5001';
+
+export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
+  return {
+    plugins: [plugin()],
+    resolve: {
+      alias: {
+        '@': fileURLToPath(new URL('./src', import.meta.url))
+      }
+    },
+    build: {
+      minify: false,
+      sourcemap: true,
+      outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
+    },
+    server: {
+      proxy: {
+        '^/api': {
+          target,
+          secure: false
+        },
+        '^/ws': {
+          target,
+          secure: false
+        }
+      },
+      port: 5173,
+      https: {
+        key: fs.readFileSync(keyFilePath),
+        cert: fs.readFileSync(certFilePath),
+      }
+    }
+  };
+});
@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <InvariantGlobalization>true</InvariantGlobalization>
+    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
+    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
+    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+    </ProjectReference>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+  </ItemGroup>
+
+  <!-- <ItemGroup>
+    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
+  </ItemGroup> -->
+</Project>
@@ -0,0 +1,106 @@
+{
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:00.1"
+  },
+  "Sparkd": {
+    "Config": {
+      "ConfigServer": "http://localhost:8000/config"
+    },
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    },
+    "EnableAdmin": true,
+    "EnableOrders": true,
+    "EnablePosition": true
+  },
+  "PubSub": {
+    "MillisecondsBetweenSubInfoUpdates": 1000,
+    "MillisecondsBetweenTimeSliceUpdates": 5000,
+    "MillisecondsBetweenNoBufferSpaceRetries": 100,
+    "RetriesOnNoBufferSpace": 1,
+    "DoItAllSecondsBetweenLogs": 0,
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "UseManagedSockets": true,
+    "StateUpdatePeriod": 5000,
+    "StatisticsUpdatePeriod": 5000,
+    "DisposedConnectionCheckPeriod": 1000,
+    "MessageProcessingWarningThreshold": 1000,
+    "ServerPort": 12345,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "MaxPacketSize": 500000000,
+    "BlueCommMustDie": true,
+    "CrushableTags": [],
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "ServerQueue": "PubSubQueue",
+    "PortRange": {
+      "First": 30400,
+      "Last": 31700
+    },
+    "BindToRandomPort": false,
+    "DebugTags": "^NOTAGS$",
+    "LatencyReportThresholdMs": 1500,
+    "UseSubstations": false,
+    "UseDynamicClients": false,
+    "IsSubstation": false,
+    "InternallyPublishSubInfo": true,
+    "UseManagedClientsInNativeCode": false,
+    "CompressedTags": "^NOTAGS$",
+    "TagsToReportCompressionRatio": "^NOTAGS$",
+    "DisableMsmq": false,
+    "UsePrepublishedServerState": false,
+    "IpTranslation": null,
+    "GcActivationThreshold": 0,
+    "WaitPeriodAfterGc": 1000,
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+         "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  }
+}
@@ -0,0 +1,4 @@
+{
+  "content_root_path": "build/Bts.DataStreamWebApp.Client/production",
+  "web_root_path": "dist"
+}
@@ -0,0 +1,30 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using Microsoft.AspNetCore.Mvc;
+
+[Route("/ws")]
+public class DataStreamSubscriptionController : ControllerBase {
+   readonly ILogger logger;
+
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
+      this.logger = logger;
+      logger.LogInformation("Creating {ControllerType}", GetType().Name);
+   }
+
+   [Route("/ws")]
+   public async Task Get(CancellationToken cancellationToken) {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      logger.LogInformation("Establishing websocket connection...");
+      try {
+         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+         logger.LogInformation("Websocket connection established.");
+         using var server = new DataStreamWebSocketServer(webSocket, logger);
+         await server.Run(cancellationToken);
+      } catch (Exception ex) {
+         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
+      }
+   }
+}
@@ -0,0 +1,125 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamSubscription : IDisposable, IHostedService {
+   public readonly ViewStream ViewStream;
+   public DataPacket? LastFired;
+
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+
+   public DataStreamSubscription(string id, JsonRpcId requestId, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
+      ViewStream = viewStream;
+      RequestId = requestId;
+      Id = id;
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public string Id { get; }
+   public JsonRpcId RequestId { get; }
+   public SubscriptionStatus Status { get; private set; }
+
+   public void Start() {
+      if (Status == SubscriptionStatus.Subscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Subscribed;
+      ViewStream.onChange += onChange;
+   }
+
+   public Task StartAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Start();
+      return Task.CompletedTask;
+   }
+
+   public Task Clear() {
+      if (LastFired is null) {
+         return Task.CompletedTask;
+      }
+      logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
+      return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
+   }
+
+   public void Stop() {
+      if (Status == SubscriptionStatus.Unsubscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Unsubscribed;
+      ViewStream.onChange -= onChange;
+   }
+
+   public Task StopAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Stop();
+      return Task.CompletedTask;
+   }
+
+   public void Dispose() =>
+      Stop();
+
+   async void onChange(object newValue) {
+      if (newValue is not DataPacket dp) {
+         logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
+         return;
+      }
+      if (webSocket.State != WebSocketState.Open) {
+         logger.LogError(
+            "Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
+         Dispose();
+         return;
+      }
+      await sendDataPacket("subscribe", dp);
+   }
+
+   async Task sendDataPacket(string method, DataPacket dp) {
+      var serializerOpts = new JsonSerializerOptions {
+         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+      };
+      serializerOpts.Converters.Add(new DataPacketJsonConverter() {
+         IncludeCurrent = true,
+         UseObjectRows = true,
+      });
+      var res = JsonRpcRequest.CreateNotification(method, dp);
+      var str = serialize(res, serializerOpts);
+      if (str is null) {
+         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
+         Dispose();
+         return;
+      }
+      logger.LogInformation(
+         "Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
+         dp.newRows.Count,
+         dp.schema.columnNames.Length,
+         str.Length);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
+      LastFired = dp;
+   }
+
+   string? serialize<T>(T message, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      try {
+         return JsonSerializer.Serialize(message, opts);
+      } catch (Exception ex) {
+         logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
+      }
+      return null;
+   }
+}
@@ -0,0 +1,89 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using Bts.PubSub;
+
+using CommandLine;
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamWebAppServerConfig {
+   /// <summary>
+   /// The login user for sparkd
+   /// </summary>
+   [JsonPropertyName("user")]
+   [Option("user", HelpText = "The sparkd login user")]
+   public string? User { get; set; }
+
+   /// <summary>
+   /// The login password for sparkd
+   /// </summary>
+   [JsonPropertyName("password")]
+   [Option("pass", HelpText = "The sparkd login password")]
+   public string? Password { get; set; }
+
+   /// <summary>
+   /// The config server url for sparkd
+   /// </summary>
+   [JsonPropertyName("sparkd")]
+   [Option("sparkd", HelpText = "The sparkd config server")]
+   public string? Sparkd { get; set; }
+
+   [JsonPropertyName("pubsub")]
+   public PubSubConfig? PubSub { get; set; }
+
+   [Option("content-root-path")]
+   [JsonPropertyName("content_root_path")]
+   public string? ContentRootPath { get; set; }
+
+   [Option("web-root-path")]
+   [JsonPropertyName("web_root_path")]
+   public string? WebRootPath { get; set; }
+
+   [Option("environment-name")]
+   [JsonPropertyName("environment_name")]
+   public string? EnvironmentName { get; set; }
+
+   public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
+
+   [Option(
+      "config",
+      MetaValue = "PATH",
+      HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
+   [JsonIgnore]
+   public string? ConfigFile { get; set; }
+
+   public static DataStreamWebAppServerConfig Default() =>
+      new() {
+         User = "bts",
+         PubSub = PubSubConfig.Default(),
+      };
+
+   public void MergeWith(DataStreamWebAppServerConfig other) {
+      User ??= other.User;
+      Password ??= other.Password;
+      PubSub ??= other.PubSub;
+      Sparkd ??= other.Sparkd;
+      ContentRootPath ??= other.ContentRootPath;
+      WebRootPath ??= other.WebRootPath;
+      ConfigFile ??= other.ConfigFile;
+   }
+
+   /// <summary>
+   /// Merges options with those defined in `ConfigFile` (does not overwrite
+   /// non-null existing values).
+   /// </summary>
+   /// <exception cref="InvalidOperationException">
+   /// JsonSerializer.Deserialize returns null
+   /// </exception>
+   public void MergeWithConfigFile() {
+      if (ConfigFile is null) {
+         return;
+      }
+      var cfgString = File.ReadAllText(ConfigFile);
+      var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
+         throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
+      MergeWith(parsedCfg);
+      ConfigFile = null;
+   }
+}
@@ -0,0 +1,121 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
+
+   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public async Task<int> Run(CancellationToken cancellationToken) {
+      logger.LogInformation("DataStreamWebSocketServer running");
+      await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger, cancellationToken)) {
+         await (req.Method switch {
+            "subscribe" => onSubscribe(req, cancellationToken),
+            "unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>(), cancellationToken),
+            _ => Task.CompletedTask,
+         });
+      }
+      logger.LogInformation("DataStreamWebSocketServer stopped");
+      return 0;
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions. To additionally notify the client of the subscription status
+   /// change(s), use <see cref="DisposeAsync"/>.
+   /// </summary>
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      disposeSubscriptions();
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions (like <see cref="Dispose"/>), then sends 'subscription status change'
+   /// notifications to the client (unlike <see cref="Dispose"/>).
+   /// </summary>
+   public async ValueTask DisposeAsync() {
+      GC.SuppressFinalize(this);
+      var finalSubs = subscriptions.Values.Select(s => (s.RequestId, s.Id)).ToList();
+      disposeSubscriptions();
+      // Send a 'subscription status change' msg to the client for each one.
+      foreach (var (requestId, id) in finalSubs) {
+         await sendSubscriptionStatus(requestId, id, "unsubscribe", SubscriptionStatus.Unsubscribed, CancellationToken.None);
+      }
+   }
+
+   async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      if (!SubscribeRequest.ParseQuery(req, out var vs, out var subId, out var errMsg, out var errCode)) {
+         logger.LogError("Invalid 'subscribe' request: {Reason}", errMsg);
+         var rpcError = new JsonRpcError<JsonNode>() {
+            Code = (long)errCode,
+            Message = $"{errCode}: {errMsg}",
+         };
+         await send(rpcError, cancellationToken);
+         return;
+      }
+      if (subscriptions.TryGetValue(subId, out var oldSub)) {
+         oldSub.Dispose();
+      }
+      var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
+      subscriptions[subId] = sub;
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
+      await sub.StartAsync(cancellationToken);
+   }
+
+   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      var subId = req.Params?.SubscriptionId ?? string.Empty;
+      if (!subscriptions.TryGetValue(subId, out var sub)) {
+         logger.LogInformation("Ignoring '{Method}' request because no subscription was found with id {Id}.", req.Method, subId);
+         return;
+      }
+      logger.LogInformation("Disposing subscription");
+      sub.Dispose();
+      subscriptions.Remove(subId);
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   async Task unsubscribe(JsonRpcId requestId, string subscriptionId, CancellationToken cancellationToken) {
+      await sendSubscriptionStatus(requestId, subscriptionId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
+      logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
+      var response = JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
+         SubscriptionId = subId,
+         Status = status,
+      });
+      return send(response, cancellationToken);
+   }
+
+   Task send<T>(T message, CancellationToken cancellationToken) {
+      if (message is null) {
+         return Task.CompletedTask;
+      }
+      var str = JsonSerializer.Serialize(message);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
+      return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken);
+   }
+
+   void disposeSubscriptions() {
+      logger.LogInformation("Disposing DataStreamWebSocketServer subscriptions. Count: {Count}", subscriptions.Count);
+      subscriptions.Values.DisposeAll();
+      subscriptions.Clear();
+   }
+}
@@ -0,0 +1,6 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+enum ErrorCode {
+   None = 0,
+   InvalidSubscriptionRequest,
+}
@@ -0,0 +1,67 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamExpressions;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+public class SubscribeRequest {
+   [JsonPropertyName("query")]
+   public string? Query { get; set; }
+
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   internal static bool ParseQuery(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(true)] out string? subscriptionId, [NotNullWhen(false)] out string? errorMessage, out ErrorCode errorCode) {
+      stream = null;
+      subscriptionId = null;
+      errorCode = ErrorCode.None;
+      var paramsJsonObj = req.Params?.AsObject();
+      if (paramsJsonObj is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params' object not found";
+         return false;
+      }
+      var subReq = paramsJsonObj.Deserialize<SubscribeRequest>();
+      if (subReq is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params' object invalid";
+         return false;
+      }
+      subscriptionId = subReq.SubscriptionId ?? string.Empty;
+      var query = subReq.Query;
+      if (query is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params.query' field not found (expected a string)";
+         return false;
+      }
+      errorMessage = null;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(query);
+      if (exp is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params.query' expression has invalid syntax";
+         return false;
+      }
+      string? exceptionMsg = null;
+      try {
+         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      } catch (Exception ex) {
+         exceptionMsg = ex.ToString();
+      }
+      if (stream is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
+         if (exceptionMsg is not null) {
+            errorMessage += ". Error" + exceptionMsg;
+         }
+         return false;
+      }
+      return true;
+   }
+}
@@ -0,0 +1,18 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+enum SubscriptionStatus {
+   None,
+   Subscribed,
+   Unsubscribed,
+}
+
+class SubscriptionStatusMessage {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   [JsonPropertyName("status")]
+   [JsonConverter(typeof(JsonStringEnumConverter))]
+   public SubscriptionStatus Status { get; set; } = SubscriptionStatus.None;
+}
@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+public class UnsubscribeRequest {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+}
@@ -0,0 +1,59 @@
+using Bts.DataStreamCore.Services;
+using Bts.PubSub.Services;
+
+using Spark.Api.Services;
+
+using System.Reactive.Linq;
+
+var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
+   Args = args,
+   ApplicationName = "Bts.DataStreamWebApp.Server",
+});
+ConfigureApp(builder.Configuration);
+ConfigureServices(builder.Services, builder.Configuration);
+
+var app = builder.Build();
+if (app.Environment.IsDevelopment()) {
+   app.UseDirectoryBrowser();
+}
+var wsOpts = new WebSocketOptions {
+   KeepAliveInterval = TimeSpan.FromMinutes(2)
+};
+app.UseWebSockets(wsOpts);
+app.UseDefaultFiles();
+app.UseStaticFiles();
+if (app.Environment.IsDevelopment()) {
+   app.UseSwagger();
+   app.UseSwaggerUI();
+}
+app.UseHttpsRedirection();
+app.UseAuthorization();
+app.MapControllers();
+try {
+   Console.Write($"Starting Application");
+   app.Run();
+   Console.Write($"Application exited gracefully.");
+} catch (Exception ex) {
+   Console.Error.Write($"Application crashed: {ex}");
+   return 1;
+}
+return 0;
+
+static void ConfigureApp(ConfigurationManager configBuilder) {
+   configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
+   var cfgRoot = configBuilder.GetSection("config");
+   if (cfgRoot?.Value is not null) {
+      configBuilder.AddJsonFile(cfgRoot.Value);
+   }
+}
+
+static void ConfigureServices(IServiceCollection services, IConfiguration config) {
+   services.AddDirectoryBrowser();
+   services.AddControllers();
+   services.AddEndpointsApiExplorer();
+   services.AddSwaggerGen();
+   services.AddMasterWorker(config.GetSection("MasterWorker"));
+   services.AddViewStreamDatabase(config.GetSection("ViewStreamDatabase"));
+   services.AddPubSubClient(config.GetSection("PubSub"));
+   services.AddSparkd(config.GetSection("Sparkd"));
+}
@@ -0,0 +1,26 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "applicationUrl": "http://localhost:5000",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:5000;https://localhost:5001",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+  },
+}
@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}
@@ -0,0 +1,168 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>ALT WebSocket Sample Application</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Message to send:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            var data = sendMessage.value;
+            socket.send(data);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>
@@ -0,0 +1,176 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>DataStreams via WebSocket</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Query:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            const rpcReq = {
+              jsonrpc: "2.0",
+              method: "subscribe",
+              params: {
+                query: sendMessage.value,
+              },
+              id: 1,
+            };
+            const rpcReqStr = JSON.stringify(rpcReq);
+            socket.send(rpcReqStr);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>
@@ -4,6 +4,8 @@ using System.Collections.Generic;
 using Bts.DataStreamExpressions;
 using BCG.Utilities.General;
 using bts;
+using Bts.PubSub;
+using Bts.DataStreamCore;

 namespace BCG {
   // listens to the requests posted on "cloud_streams" request table, and 
@@ -40,13 +42,15 @@ namespace BCG {
      private IDictionary<string, StreamPubUnit>
         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
      private ViewStream _stream = new TableStream("cloud_streams");
-      private PubTableBroker _broker = new PubTableBroker();
-      private ManagedSubscriber _sub = ManagedSubscriber.instance;
+      readonly IPubTableBroker _broker;
+      readonly IManagedSubscriber _sub;


-      public RemoteQueryManager() {
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
+         _sub = managedSubscriber;
+         _broker = pubTableBroker;
         _stream.onChange += _on_str;
-         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
+         _sub.Subscribe("RemoteRequest", (string tag, byte[] data) => {
            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
         });
@@ -1,9 +1,7 @@
-using System;
+#nullable disable
+using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using BCG.Utilities;

 namespace BCG
 {
@@ -1,6 +1,10 @@
+using bts;
+
 using Bts.DataStream;
+using Bts.DataStreamCore;

 using mbts.config;
+using mbts.utils;

 using System;

@@ -16,8 +20,8 @@ namespace BCG {
   [DataStreamUserRespresentation (false)]
   public class MapStream : MethodDrivenStream
   {
-      protected IStreamMapper myMapper;
-      protected IChangeableStreamMapper myChangeableMapper = null;
+      protected IStreamMapper? myMapper;
+      protected IChangeableStreamMapper? myChangeableMapper;
      private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;

      public class FuncWrapper : IStreamMapper {
@@ -72,7 +76,7 @@ namespace BCG {


      protected override void myHandleFirstCall() {
-
+         ArgumentNullException.ThrowIfNull(myMapper);
         IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
         myLastFiredPacket = new DataPacket();
         myLastFiredPacket.newRows = new BlueTable(outputSchema);
@@ -83,28 +87,34 @@ namespace BCG {
      }

      protected override void myHandleIncrementalAdd() {
-         //t.start ();
-         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
-            IBlueRow newRow = myMapper.map(row);
-            (myLastFiredPacket.newRows as BlueTable).addRow(newRow);
-            (myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
+         if (myLastArrivedPacket.addedRows.Count == 0) {
+            Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
+            return;
         }
-         //t.stop ();
-         //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
-         //System.Diagnostics.Trace.WriteLine (" ");
+         ArgumentNullException.ThrowIfNull(myMapper);
+         var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
+         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
+            var newRow = myMapper.map(row);
+            newRows.addRow(newRow);
+            addedRows.addRow(newRow);
+         }
+         myLastFiredPacket.addedRows = addedRows;
      }

      protected override void myHandleIncrementalDelete() {
-         //t.start ();
-         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
-            IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
-            (myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
-            (myLastFiredPacket.newRows as BlueTable).deleteRow(row);
+         if (myLastArrivedPacket.deletedRows.Count == 0) {
+            Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
+            return;
         }
-         //t.stop ();
-         //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
+         var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
+         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
+            var delRow = newRows[row.Id()];
+            deletedRows.addRow(delRow);
+            newRows.deleteRow(row);
+         }
+         myLastFiredPacket.deletedRows = deletedRows;
      }


@@ -1,11 +1,9 @@
+using Bts.DataStreamCore;
+
+using mbts.utils;
+
 using System;
 using System.Collections.Generic;
-using System.Text;
-
-using BCG.Utilities;
-using BCG.Utilities.General;
-
-using mbts;

 namespace BCG {
   /// <summary>
@@ -56,15 +54,15 @@ namespace BCG {
            return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
         }

-         public event MapperChangeDelegate onMapperChange;
-         private void _fake() { onMapperChange(); }
+         public event MapperChangeDelegate? onMapperChange;
      }

      //maps the id of the underlying stream to the
      //rows of the resulting stream
      private Dictionary<int, List<IBlueRow>> myMap = new Dictionary<int, List<IBlueRow>>();
      private int myID = 0;
-      private BlueTable myRows, myBaseRows;
+      private BlueTable? myRows;
+      private BlueTable? myBaseRows;
      private ViewStream myStream;
      private ISuperStreamMapper myMapper;
      public bool generate_ids = true;
@@ -92,49 +90,40 @@ namespace BCG {
         myRows = null;
         myBaseRows = null;
         myMap.Clear();
-         //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
      }

      private void myHandleData(object newValue) {
-         DataPacket idp = newValue as DataPacket;
-         //HighPerformanceTimer timer = new HighPerformanceTimer();
-         //timer.start();
-         DataPacket dp = newValue as DataPacket;
-         myRows = myRows ?? new BlueTable(myMapper.getOutputSchema(dp.schema));
-         myBaseRows = myBaseRows ?? new BlueTable(dp.schema);
+         var dp = ExceptionAssert.Cast(newValue).To<DataPacket>();
+         myRows ??= new BlueTable(myMapper.getOutputSchema(dp.schema));
+         myBaseRows ??= new BlueTable(dp.schema);
         BlueDataUtility.absorbChanges(myBaseRows, dp.deletedRows, dp.addedRows);
-         BlueTable dels = new BlueTable(myRows.schema),
-                     adds = new BlueTable(myRows.schema);
+         var dels = new BlueTable(myRows.schema);
+         var adds = new BlueTable(myRows.schema);

         foreach (IBlueRow r in dp.deletedRows) {
-            List<IBlueRow> mappedRows = myMap[(int)r["id"]];
+            var mappedRows = myMap[r.Id()];
            foreach (IBlueRow mr in mappedRows) {
               dels.addRow(mr);
               myRows.deleteRow(mr);
            }
-            myMap.Remove((int)r["id"]);
+            myMap.Remove(r.Id());
         }
         foreach (IBlueRow r in dp.addedRows) {
            IBlueRow[] mappedRows = myMapper.map(r);
            foreach (IBlueRow mr in mappedRows) {
               IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
-               if (!myMap.ContainsKey((int)r["id"])) {
-                  myMap.Add((int)r["id"], new List<IBlueRow>());
-               }
-               myMap[(int)r["id"]].Add(idChangedMapRow);
+               myMap.GetOrAdd(r.Id(), _ => new()).Add(idChangedMapRow);
               myRows.addRow(idChangedMapRow);
               adds.addRow(idChangedMapRow);
            }
         }
-         //timer.stop();
-         // if (timer.lastIntervalTime > 20)
-         //    {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
         myFireEvent(new DataPacket(myRows, adds, dels));
      }

      private void myHandleMapChange() {
-         if (myBaseRows != null)
+         if (myBaseRows is not null) {
            myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
+         }
      }
   }
 }
@@ -8,6 +8,7 @@ using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
 using bts.api;
+using Bts.PubSub.Routing;

 namespace bts {
   [DataStreamUserRespresentation("pss_echo")]
@@ -87,8 +88,10 @@ namespace bts {
      IOState _state = IOState.SEND;

      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd) {
-         var route = new PubSubRoute(server_url, ".*");
-         _router = new PubSubRouter(route);
+         _router = new PubSubRouter(new PubSubRouterConfig() {
+            PublisherRoutes = new() { new(server_url, ".*") },
+            SubscriberRoutes = new() { new(server_url, ".*") },
+         });
         _vsd = vsd;
         _payload_sz = payload_size;
         var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -196,4 +199,4 @@ namespace bts {
      }

   }
-}
+}
@@ -95,7 +95,7 @@ namespace BCG {
            sub_msg.serialize_into_str(str);
            str.Position = 0;
            bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
-            sub_msg_.consruct_deserialize(str);
+            sub_msg_.construct_deserialize(str);
            bool res = sub_msg.Equals(sub_msg_);
            return new TestResult {
                test_name = "SubscribeMsg",
@@ -117,7 +117,7 @@ namespace BCG {
            pub_msg.serialize_into_str(str);
            str.Position = 0;
            var pub_msg_ = new bts.PublishMsg();
-            pub_msg_.consruct_deserialize(str);
+            pub_msg_.construct_deserialize(str);
            bool res = pub_msg.Equals(pub_msg_);
            return new TestResult {
                test_name = "PublishMsg",
@@ -135,7 +135,7 @@ namespace BCG {
            msg.serialize_into_str(str);
            str.Position = 0;
            var msg_ = new bts.PubSubServerMessage();
-            msg_.consruct_deserialize(str);
+            msg_.construct_deserialize(str);
            bool res = msg.Equals(msg_);
            return new TestResult {
                test_name = "PubSubServerMessage",
@@ -156,7 +156,7 @@ namespace BCG {
            msg.serialize_into_str(str);
            str.Position = 0;
            var msg_ = new PubSubClientMessage();
-            msg_.consruct_deserialize(str);
+            msg_.construct_deserialize(str);
            var res = msg.Equals(msg_);
            return new TestResult {
                test_name = "PubSubClientMessage",
@@ -617,14 +617,15 @@ namespace BCG {


        private static TestResult _run_single_tag_data_test_clean() {
+            // var tagHandlerFactory
            // create message
-            var test_name = MethodBase.GetCurrentMethod().Name;
+            var test_name = MethodBase.GetCurrentMethod()?.Name;
            var schema = create_schema_with_all_types();
            var tag = "DS_" + schema.tableName;
            BlueTable table = new BlueTable(schema);
            IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
            IStreamMsgDeserilizer deser = new StreamIODeserializationController();
-            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
+            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType, new PubsubServerStreamTagHandler());
            // create with 100 rows
            int n_rows = 100;
            bool is_succ = true;
@@ -1,3 +1,5 @@
+using Bts.DataStreamCore;
+
 using System;

 namespace BCG {
@@ -7,14 +9,15 @@ namespace BCG {
   /// </summary>
 	public class CircularPublisher {
      private CircularRowCollection myBuffer;
-      private PubTableBroker myBroker;
+      private readonly IPubTableBroker myBroker;
      private int myBufferSize;
-      public CircularPublisher(int bufferSize) {
+      public CircularPublisher(int bufferSize, IPubTableBroker broker) {
         myBufferSize = bufferSize;
+         myBroker = broker;
      }

-      public CircularPublisher(int bufferSize, IBlueRowCollection rows)
-         : this(bufferSize) {
+      public CircularPublisher(int bufferSize, IPubTableBroker broker, IBlueRowCollection rows)
+         : this(bufferSize, broker) {
         publishRows(rows);
      }

@@ -23,7 +26,6 @@ namespace BCG {
            myBuffer = new CircularRowCollection(
               new BlueTableSchema(nr.schema.tableName, nr.schema.columnInfo),
               myBufferSize);
-            myBroker = new PubTableBroker();
            myBroker.create_table(myBuffer);
         }
         IBlueRow br = myBuffer.addRowWithReturn(nr);
@@ -38,7 +40,6 @@ namespace BCG {
               new BlueTableSchema(BlueDataUtility.extractRow(nrs, 0).schema.tableName,
               BlueDataUtility.extractRow(nrs, 0).schema.columnInfo),
               myBufferSize);
-            myBroker = new PubTableBroker();
            myBroker.create_table(myBuffer);
         }
         BlueTable deletes = new BlueTable(myBuffer.schema);
@@ -4,10 +4,12 @@ using System.Data;

 using BCG.Utilities;

+using Bts.DataStreamCore;
+
 namespace BCG {
   public class ViewPublisher : IDisposable {
      protected ViewPublisher() { }
-      private readonly PubTableBroker _broker;
+      private readonly IPubTableBroker _broker;
      protected readonly ViewStream _stream;
      private int _n_updates = 0;

@@ -15,7 +17,7 @@ namespace BCG {
      /// in the schema (be careful with that because this
      /// could lead to double publishes if the schema name
      /// is not uniquely chosen)
-      public ViewPublisher(string name, PubTableBroker broker, ViewStream vs) {
+      public ViewPublisher(string name, IPubTableBroker broker, ViewStream vs) {
         _stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
         _broker = broker;
         _stream.onChange += new DataStream.ChangeDelegate(_handle_update);
@@ -0,0 +1,11 @@
+namespace Bts.DataStream.Services;
+
+using BCG;
+using BCG.Utilities.General;
+
+using Microsoft.Extensions.DependencyInjection;
+
+public static class DataStreamServices {
+   public static IServiceCollection AddRemoteQueryManager(this IServiceCollection services) =>
+      services.AddSingleton<IWorker, RemoteQueryManager>();
+}
@@ -1,4 +1,5 @@
 using System;
+using System.Text.Json.Serialization;

 namespace BCG {
   /// <summary>
@@ -56,9 +57,16 @@ namespace BCG {
      private Type myColType;
      private object myColValue;

+      [JsonPropertyName("colNo")]
      public int colNo { get { return myColNo; } }
+
+      [JsonPropertyName("colName")]
      public string colName { get { return myColName; } }
+
+      [JsonPropertyName("colType")]
      public Type colType { get { return myColType; } }
+
+      [JsonIgnore]
      public object colValue {get{return myColValue;}}
   }
 }
@@ -0,0 +1,11 @@
+namespace Bts.DataStreamCore;
+
+public enum DataType {
+   Null,
+   String,
+   Float32,
+   Float64,
+   Int32,
+   Int64,
+   DateTime,
+}
@@ -0,0 +1,61 @@
+#nullable enable
+using BCG;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class BlueRowJsonConverter : JsonConverter<IBlueRow> {
+   IBlueTableSchema? schema = null;
+
+   public override IBlueRow Read(
+      ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (schema is null) {
+         throw new InvalidOperationException("Cannot read row from json because schema is null");
+      }
+      var mbr = new MutableBlueRow(schema);
+      int colNum = 0;
+      switch (reader.TokenType) {
+         case JsonTokenType.StartArray:
+            while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
+               mbr.setValue(colNum++, readCellValue(ref reader));
+               colNum++;
+            }
+            break;
+         case JsonTokenType.StartObject:
+            while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
+               if (reader.TokenType != JsonTokenType.PropertyName) {
+                  throw new InvalidOperationException(
+                     "Expected 'PropertyName' immediately after 'StartObject' token in row json");
+               }
+               var colName = reader.GetString();
+               if (!reader.Read()) {
+                  throw new InvalidOperationException(
+                     "Failed to read token immediately after 'PropertyName' token in row json");
+               }
+               mbr.setValue(colName, readCellValue(ref reader));
+            }
+            break;
+      }
+      if (mbr.Count != schema.columnInfo.Length) {
+         throw new InvalidOperationException(
+            $"Row json array length '{mbr.Count}' does not match the schema's " +
+            $"column length '{schema.columnInfo.Length}'.");
+      }
+      return mbr;
+
+      static object? readCellValue(ref Utf8JsonReader reader) =>
+         reader.TokenType switch {
+            JsonTokenType.String => reader.GetString(),
+            JsonTokenType.Number => reader.GetDouble(),
+            JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
+            _ => throw new InvalidOperationException("Failed to read cell value"),
+         };
+   }
+
+   public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
+      throw new NotImplementedException();
+   }
+}
@@ -0,0 +1,138 @@
+#nullable enable
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+public class DataPacketJsonConverter : JsonConverter<DataPacket> {
+   /// <summary>
+   /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
+   /// default, only incremental changes are included ('added' and 'deleted').
+   /// </summary>
+   public bool IncludeCurrent { get; set; }
+
+   /// <summary>
+   /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
+   /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
+   /// </summary>
+   public bool UseObjectRows { get; set; }
+
+   public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (!reader.ReadObject()) {
+         return null;
+      }
+      if (!reader.ReadPropertyName(out var prop)) {
+         return null;
+      }
+      if (!reader.Read()) {
+         return null;
+      }
+      if (prop == "added") {
+         var added = new List<MutableBlueRow>();
+         if (reader.TokenType != JsonTokenType.StartArray) {
+            return null;
+         }
+         while (true) {
+            if (!reader.Read()) {
+               break;
+            }
+            if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) {
+               return null;
+            }
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  // DAMNIT! Need the schema.
+                  break;
+            }
+         }
+      }
+      return null;
+   }
+
+   public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
+      using var _ = writer.WriteObject();
+      using (writer.WriteArray("columns")) {
+         foreach (var col in value.schema.columnInfo) {
+            using (writer.WriteObject()) {
+               writer.WriteString("name", col.colName);
+               writer.WriteNumber("index", col.colNo);
+               // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
+               writer.WriteString("type", col.type_code().ToString());
+            }
+         }
+      }
+
+      var columns = value.schema.columnInfo.Select(c => (Column: c, Write: GetCellValueWriter(c))).ToList();
+      WriteRows("added", value.addedRows);
+      WriteRows("deleted", value.deletedRows);
+      if (IncludeCurrent) {
+         WriteRows("all", value.newRows);
+      }
+
+      void WriteRows(string name, IBlueRowCollection rows) {
+         using (writer.WriteArray(name)) {
+            foreach (IBlueRow row in rows) {
+               WriteRow(row);
+            }
+         }
+      }
+
+      void WriteRow(IBlueRow row) {
+         using var rowCase = UseObjectRows ? writer.WriteObject() : writer.WriteArray();
+         for (int i = 0; i < columns.Count; i++) {
+            if (UseObjectRows) {
+               writer.WritePropertyName(columns[i].Column.colName);
+            }
+            columns[i].Write(row[i]);
+         }
+      }
+
+      Action<object?> GetCellValueWriter(ColumnInfo colInfo) {
+         var t = colInfo.colType;
+         if (t == typeof(int)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((int)v));
+         }
+         if (t == typeof(uint)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((uint)v));
+         }
+         if (t == typeof(long)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((long)v));
+         }
+         if (t == typeof(ulong)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
+         }
+         if (t == typeof(short)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((short)v));
+         }
+         if (t == typeof(ushort)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
+         }
+         if (t == typeof(double)) {
+            return NullCheckWrapper(v => JsonSerializer.Serialize(writer, (double)v, options));
+         }
+         if (t == typeof(float)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((float)v));
+         }
+         if (t == typeof(bool)) {
+            return NullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
+         }
+         return NullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
+
+         Action<object?> NullCheckWrapper(Action<object> writeNonNull) =>
+            v => {
+               if (v is null) {
+                  writer.WriteNullValue();
+               } else {
+                  writeNonNull(v);
+               }
+            };
+      }
+   }
+}
@@ -0,0 +1,49 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.IO;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+/// <summary>
+/// This class seriailzes DataPackets to json strings. A common buffer is used to reduce memory
+/// allocations on repeated calls to <see cref="Serialize"/>. For that reason, this class is NOT
+/// thread safe.
+/// </summary>
+public class DataPacketJsonSerializer {
+   readonly MemoryStream buffer = new();
+   readonly DataPacketJsonConverter converter;
+
+   public DataPacketJsonSerializer(DataPacketJsonConverter? converter = null) =>
+      this.converter = converter ?? new() {
+         IncludeCurrent = true,
+         UseObjectRows = true,
+      };
+
+   public string? Serialize(DataPacket dp) {
+      buffer.Position = 0;
+      buffer.SetLength(0);
+      using (var jsonWriter = new Utf8JsonWriter(buffer)) {
+         var opts = new JsonSerializerOptions() {
+            NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+         };
+         converter.Write(jsonWriter, dp, opts);
+      }
+      var bytes = buffer.GetBuffer();
+      var span = new ArraySegment<byte>(bytes, 0, (int)buffer.Length);
+      return Encoding.UTF8.GetString(span);
+   }
+
+   public void Serialize(DataPacket dp, Utf8JsonWriter jsonWriter) {
+      buffer.Position = 0;
+      buffer.SetLength(0);
+      var opts = new JsonSerializerOptions() {
+         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+      };
+      converter.Write(jsonWriter, dp, opts);
+   }
+}
@@ -8,6 +8,7 @@

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
  </ItemGroup>

  <ItemGroup>
@@ -0,0 +1,8 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+public interface IPubTableBroker {
+   void create_table(IBlueRowCollection table);
+   void make_batch_changes(IBlueRowCollection new_rows, IBlueRowCollection del_rows, IBlueRowCollection added_rows);
+}
@@ -8,12 +8,14 @@ using BCG.Utilities;

 using bts;

+using Bts.DataStreamCore;
+
 namespace BCG {
   /// <summary>
   /// This class will be used by the clients to publish tabular data to the pubsub
   /// </summary>
-   public class PubTableBroker {
-      protected PublisherCore _publisher_core;
+   public class PubTableBroker : IPubTableBroker {
+      protected IPublisherCore _publisher_core;
      protected IDictionary<string, SinglePubTableBroker> _publisher_map =
         new Dictionary<string, SinglePubTableBroker>();

@@ -22,7 +24,7 @@ namespace BCG {
      public PubTableBroker()
         : this(new PublisherCore()) { }

-      public PubTableBroker(PublisherCore publisher) {
+      public PubTableBroker(IPublisherCore publisher) {
         _publisher_core = publisher;
         Log.Gen.info("PubTableBroker constructed");
      }
@@ -14,9 +14,9 @@ namespace BCG {
   public class SinglePubTableBroker {
      private readonly string _tag;
      protected IStreamIOMsgSerializer _serializer;
-      protected PublisherCore _publisher_core;
+      protected IPublisherCore _publisher_core;

-      public SinglePubTableBroker(string tag, PublisherCore publisher_core) {
+      public SinglePubTableBroker(string tag, IPublisherCore publisher_core) {
         bts.Log.Gen.info($"SinglePubTableBroker.cons created for tag: {tag}");
         _tag = tag;
         _publisher_core = publisher_core;
@@ -51,4 +51,4 @@ namespace BCG {
      }

   }
-}
+}
@@ -13,6 +13,9 @@ namespace bts {
      public PubsubServerGenericDataHandler() {
         _bw = new BinaryWriter(_str);
      }
+
+      public string TagType => "*";
+
      public void update(string tag, byte[] msg) {
         _str.Position = 0;
         _bw.Write(msg);
@@ -25,9 +28,6 @@ namespace bts {
         Log.Gen.info($"PubsubServerGenericDataHandler.get_content tag: {tag} rv: {rv.Length}");
         return rv;
      }
-      public string[] supportedTagTypes { get; }
-      public void register(InternalPublishDelegate callback) { }
-      public void doWork() { }
   }

   // This class is created by the SingleTagData for stream tag_type. Every pub
@@ -40,6 +40,7 @@ namespace bts {
      // this is where the state of the table is kept.
      BlueTable _state = null;

+      public string TagType => DSTagManager.DataStreamTagType;

      public void update(string tag, byte[] msg) {
         StreamIOMessage str_msg = _deser.deserialize(msg);
@@ -74,15 +75,6 @@ namespace bts {
         Log.Gen.info($"PubsubServerSingleStreamTagHandler.get_content rv: {rv_txt}");
         return rv;
      }
-      public string[] supportedTagTypes {
-         get {
-            return new string[] { "DS" };
-         }
-      }
-      public void register(InternalPublishDelegate callback) {
-
-      }
-      public void doWork() { }
   }

   public class PubsubServerStreamTagHandler : IPubSubServerTagHandler {
@@ -92,6 +84,9 @@ namespace bts {

      public PubsubServerStreamTagHandler() {
      }
+
+      public string TagType => DSTagManager.DataStreamTagType;
+
      public void update(string tag, byte[] msg) {
         BCG.IPubSubServerTagHandler rv = _get_handler(tag);
         rv.update(tag, msg);
@@ -100,14 +95,6 @@ namespace bts {
         BCG.IPubSubServerTagHandler rv = _get_handler(tag);
         return rv.get_content(tag);
      }
-      public string[] supportedTagTypes {
-         get {
-            List<string> rv = new List<string>();
-            foreach (var h in _handler_map.Values)
-               rv.AddRange(h.supportedTagTypes);
-            return rv.ToArray();
-         }
-      }

      private BCG.IPubSubServerTagHandler _get_handler(string tag) {
         PubsubServerSingleStreamTagHandler rv = null;
@@ -117,13 +104,5 @@ namespace bts {
         }
         return rv;
      }
-      public void register(InternalPublishDelegate callback) {
-         foreach (var h in _handler_map.Values)
-            h.register(callback);
-      }
-      public void doWork() {
-         foreach (var h in _handler_map.Values)
-            h.doWork();
-      }
   }
-}
+}
@@ -0,0 +1,21 @@
+namespace Bts.DataStreamCore.Services;
+
+using BCG;
+using BCG.Utilities.General;
+
+using Bts.PubSub.Services;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+
+public static class DataStreamCoreServices {
+   public static IServiceCollection AddPubTableBroker(this IServiceCollection services) =>
+      services.AddSingleton<IPubTableBroker, PubTableBroker>();
+
+   public static IServiceCollection AddViewStreamDatabase(this IServiceCollection services, IConfiguration configurationSection) =>
+      services.Configure<WorkerServiceConfig>(configurationSection)
+         .AddSingleton<IWorker, ViewStreamDatabase>(sp => {
+            ViewStreamDatabase.SetGlobalInstance(new ViewStreamDatabase(sp.GetRequiredService<ISubscriberCore>()));
+            return ViewStreamDatabase.instance;
+         });
+}
@@ -9,6 +9,7 @@ using System.Threading;
 using mbts;
 using System.Linq;
 using System.Diagnostics;
+using bts;

 namespace BCG {
   public class ViewStreamDatabase : IWorker, IDisposable {
@@ -230,6 +231,11 @@ namespace BCG {
         }
      }

+      internal static void SetGlobalInstance(ViewStreamDatabase vsd) {
+         Log.isnull(mySelf);
+         mySelf = vsd;
+      }
+
      private List<PendingAddRemoveWorker> _pending_worker_ops = new List<PendingAddRemoveWorker>();
      private bool _is_worker_call_in_progress = false;
      class PendingAddRemoveWorker {
@@ -1,109 +1,120 @@
 {
-  "user": "bts",
-  "sparkd": "http://127.0.0.1:8000/config",
-  "pubsub": {
-    "milliseconds_between_sub_info_updates": 1000,
-    "milliseconds_between_time_slice_updates": 5000,
-    "milliseconds_between_no_buffer_space_retries": 100,
-    "retries_on_no_buffer_space": 1,
-    "do_it_all_seconds_between_logs": 0,
-    "heartbeat_interval": 15000,
-    "missing_heartbeats_before_gc": 50,
-    "use_managed_sockets": true,
-    "state_update_period": 5000,
-    "statistics_update_period": 5000,
-    "disposed_connection_check_period": 1000,
-    "message_processing_warning_threshold": 1000,
-    "server_port": 12345,
-    "publisher_routes": [
-      {
-        "name": "default",
-        "server_address": "bts-cloud://127.0.0.1:12345",
-        "tag": ".*"
-      }
-    ],
-    "subscriber_routes": [
-      {
-        "name": "default",
-        "server_address": "bts-cloud://127.0.0.1:12345",
-        "tag": ".*"
-      }
-    ],
-    "max_packet_size": 500000000,
-    "blue_comm_must_die": true,
-    "crushable_tags": [],
-    "tag_type_to_assembly_map": [
-      {
-        "handler_class": "BCG.PubSubServerTagHandler",
-        "assembly": "DataStreamCore.dll"
-      }
-    ],
-    "server_queue": "PubSubQueue",
-    "port_range": {
-      "first": 30400,
-      "last": 31700
+  "Sparkd": {
+    "Config": {
+      "ConfigServer": "http://localhost:8000/config"
    },
-    "bind_to_random_port": false,
-    "debug_tags": "^NOTAGS$",
-    "latency_report_threshold_ms": 1500,
-    "use_substations": false,
-    "use_dynamic_clients": false,
-    "is_substation": false,
-    "internally_publish_sub_info": true,
-    "use_managed_clients_in_native_code": false,
-    "compressed_tags": "^NOTAGS$",
-    "tags_to_report_compression_ratio": "^NOTAGS$",
-    "disable_msmq": false,
-    "use_prepublished_server_state": false,
-    "ip_translation": null,
-    "gc_activation_threshold": 0,
-    "wait_period_after_gc": 1000
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    },
+    "EnableAdmin": true,
+    "EnableOrders": true,
+    "EnablePosition": true
  },
-  "publishers": [
-    {
-      "name": "market_data",
-      "streams": [
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "Publisher": {
+    "Publishers": [
+      {
+        "Name": "test_data",
+        "Streams": [
+          {
+            "Name": "milliseconds_1000_100",
+            "Expression": "milliseconds(3000,5)",
+            "IsOnWebsocket": true
+          }
+        ]
+      }
+    ]
+  },
+  "PubSub": {
+    "MillisecondsBetweenSubInfoUpdates": 1000,
+    "MillisecondsBetweenTimeSliceUpdates": 5000,
+    "MillisecondsBetweenNoBufferSpaceRetries": 100,
+    "RetriesOnNoBufferSpace": 1,
+    "DoItAllSecondsBetweenLogs": 0,
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "UseManagedSockets": true,
+    "StateUpdatePeriod": 5000,
+    "StatisticsUpdatePeriod": 5000,
+    "DisposedConnectionCheckPeriod": 1000,
+    "MessageProcessingWarningThreshold": 1000,
+    "ServerPort": 12345,
+    "Routing": {
+      "PublisherRoutes": [
        {
-          "name": "bbo",
-          "expression": "bbo()",
-          "is_on_websocket": true
-        },
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
        {
-          "name": "instruments",
-          "expression": "instruments()",
-          "is_on_websocket": true
-        },
-        {
-          "name": "timesales",
-          "expression": "timesales(10000)",
-          "is_on_websocket": true
-        },
-        {
-          "name": "books",
-          "expression": "books()",
-          "is_on_websocket": true
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
        }
      ]
    },
-    {
-      "name": "test_data",
-      "streams": [
+    "MaxPacketSize": 500000000,
+    "BlueCommMustDie": true,
+    "CrushableTags": [],
+    "TagTypeToAssemblyMap": {
+      "Entries": [
        {
-          "name": "milliseconds_1000_100",
-          "expression": "milliseconds(1000,100)",
-          "is_on_websocket": true
-        },
-        {
-          "name": "test_literal",
-          "expression": "literal(\"x\", \"420.0\")",
-          "is_on_websocket": true
-        },
-        {
-          "name": "test_not_on_websocket",
-          "expression": "literal(\"x\", \"840.0\")",
-          "is_on_websocket": false
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
        }
      ]
-    }
-  ]
-}
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    },
+    "ServerQueue": "PubSubQueue",
+    "PortRange": {
+      "First": 30400,
+      "Last": 31700
+    },
+    "BindToRandomPort": false,
+    "DebugTags": "^NOTAGS$",
+    "LatencyReportThresholdMs": 1500,
+    "UseSubstations": false,
+    "UseDynamicClients": false,
+    "IsSubstation": false,
+    "InternallyPublishSubInfo": true,
+    "UseManagedClientsInNativeCode": false,
+    "CompressedTags": "^NOTAGS$",
+    "TagsToReportCompressionRatio": "^NOTAGS$",
+    "DisableMsmq": false,
+    "UsePrepublishedServerState": false,
+    "IpTranslation": null,
+    "GcActivationThreshold": 0,
+    "WaitPeriodAfterGc": 1000
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:01"
+  }
+}
@@ -17,14 +17,14 @@
    "publisher_routes": [
      {
        "name": "default",
-        "server_address": "bluto://localhost:12345",
+        "server_address": "bts-cloud://localhost:12345",
        "tag": ".*"
      }
    ],
    "subscriber_routes": [
      {
        "name": "default",
-        "server_address": "bluto://localhost:12345",
+        "server_address": "bts-cloud://localhost:12345",
        "tag": ".*"
      }
    ],
@@ -0,0 +1,6 @@
+namespace Bts.DataStreamPublisher;
+
+public class CsvPublisherConfig {
+   public string? Name { get; set; }
+   public string? CsvDir { get; set; }
+}
@@ -9,6 +9,7 @@

  <ItemGroup>
    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
  </ItemGroup>

  <ItemGroup>
@@ -1,7 +1,5 @@
 namespace Bts.DataStreamPublisher;

-using Bts.PubSub;
-
 using CommandLine;

 using System;
@@ -11,35 +9,12 @@ using System.Text.Json;
 using System.Text.Json.Serialization;

 public class DataStreamPublisherConfig {
-   /// <summary>
-   /// The login user for sparkd
-   /// </summary>
-   [JsonPropertyName("user")]
-   [Option("user", HelpText = "The sparkd login user")]
-   public string? User { get; set; }
-
-   /// <summary>
-   /// The login password for sparkd
-   /// </summary>
-   [JsonPropertyName("password")]
-   [Option("pass", HelpText = "The sparkd login password")]
-   public string? Password { get; set; }
-
-   /// <summary>
-   /// The config server url for sparkd
-   /// </summary>
-   [JsonPropertyName("sparkd")]
-   [Option("sparkd", HelpText = "The sparkd config server")]
-   public string? Sparkd { get; set; }
-
   /// <summary>
   /// The tables to publish on startup. Each key/value represents a table name/query to publish.
   /// </summary>
-   [JsonPropertyName("publishers")]
   public List<PublisherNode>? Publishers { get; set; }

-   [JsonPropertyName("pubsub")]
-   public PubSubConfig? PubSub { get; set; }
+   public List<string>? CsvDirs { get; set; }

   [Option(
      "config",
@@ -49,16 +24,10 @@ public class DataStreamPublisherConfig {
   public string? ConfigFile { get; set; }

   public static DataStreamPublisherConfig Default() =>
-      new() {
-         PubSub = PubSubConfig.Default(),
-      };
+      new() { };

   public void MergeWith(DataStreamPublisherConfig other) {
-      User ??= other.User;
-      Password ??= other.Password;
-      Sparkd ??= other.Sparkd;
      Publishers ??= other.Publishers;
-      PubSub ??= other.PubSub;
      ConfigFile ??= other.ConfigFile;
   }

@@ -1,140 +1,34 @@
-using BCG;
-using BCG.Utilities.General;
+using Bts.DataStreamPublisher.Services;

-using bts;
-using bts.api.core;
-using bts.utils;
-
-using Bts.DataStreamExpressions;
-using Bts.DataStreamPublisher;
-using Bts.PubSub;
-
-using CommandLine;
-
-using mbts.config;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;

 using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Text.Json;
-using System.Threading.Tasks;

-await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
-   .MapResult(RunAsync, errs => Task.FromResult(1));
+var host = Host.CreateDefaultBuilder(args)
+   .ConfigureAppConfiguration(ConfigureApp)
+   .ConfigureServices(ConfigureServices)
+   .Build();

-async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
-   try {
-      cfg.MergeWithConfigFile();
-      cfg.MergeWith(DataStreamPublisherConfig.Default());
-      PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
-      Configuration.initialize(cfg.Sparkd);
-      Log.Gen.info("Starting DataStreamPublisher");
-      var service_ctx = ServiceContext.create(cfg.Sparkd);
-      await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
-      _publish_streams(cfg.Publishers ?? new(0));
-   } catch (Exception ex) {
-      // Write the error to console first, in case the logging system is compromised.
-      var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
-      Console.Error.WriteLine(error_msg);
-      Console.Error.Flush();
-      bts.Log.Gen.error(error_msg);
-      return 1;
-   }
-   return 0;
+try {
+   Console.Write("Starting DataStreamPublisher");
+   host.Run();
+   Console.Write("DataStreamPublisher exited gracefully.");
+} catch (Exception ex) {
+   Console.Error.Write($"DataStreamPublisher crashed: {ex}");
+   return 1;
 }
+return 0;

-static ViewStream? build_stream(string? stream_expression) {
-   if (stream_expression is null) {
-      return null;
-   }
-   try {
-      bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
-      var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-      var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
-      return rv;
-
-   } catch (Exception ex) {
-      bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
-      return null;
+static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) {
+   configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
+   var cfgRoot = configBuilder.Build();
+   var cfg = cfgRoot["config"];
+   if (cfg is not null) {
+      configBuilder.AddJsonFile(cfg, false);
   }
 }

-void _publish_streams(IEnumerable<PublisherNode> publishers) {
-   PubTableBroker pb_broker = new PubTableBroker();
-   RemoteQueryManager csm = new RemoteQueryManager();
-   var csvPublishers = new List<CSVPublisher>();
-
-   foreach (var publisher in publishers) {
-      if (Directory.Exists(publisher.CsvDir)) {
-         csvPublishers.Add(new(publisher.CsvDir, pb_broker));
-      }
-   }
-
-   var viewPublishers = publishers
-      .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher))
-      .ToList();
-
-   while (true) {
-      System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
-      ViewStreamDatabase.instance.doWork();
-      csm.doWork();
-      MasterWorker.getInstance(false).doWork();
-      ManagedSubscriber.instance.doWork();
-   }
-
-   ViewPublisher? createViewPublisher(PublishedStream ps) {
-      Log.Gen.info(
-         $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
-      if (string.IsNullOrEmpty(ps.Name)) {
-         Log.Gen.warn(
-            $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
-         return null;
-      }
-      var vs = build_stream(ps.Expression);
-      if (vs is null) {
-         Log.Gen.warn(
-            $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
-         return null;
-      }
-      var vpub = new ViewPublisher(ps.Name, pb_broker, vs);
-      vs.onChange += on_change;
-      return vpub;
-   }
-}
-
-void on_change(object obj) {
-   bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
-}
-
-public class CSVPublisher : IWorker {
-   private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
-   private readonly string _csv_directory;
-   private PubTableBroker _pub_table_broker;
-
-   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
-      _csv_directory = csv_directory;
-      _pub_table_broker = pub_table_broker;
-      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-   }
-   public void doWork() {
-      var file_names = Directory.GetFiles(_csv_directory);
-      foreach (var fn in file_names) {
-         if (!_pubs.ContainsKey(fn)) {
-            _handle_new_file(fn);
-         }
-      }
-   }
-
-   private void _handle_new_file(string file_name) {
-      if (file_name.EndsWith(".csv")) {
-         ViewStream vs = new CSVReaderStream(file_name);
-         vs = new SchemaGuesserStream(vs);
-         var toks = file_name.Split('\\');
-         string table_name = toks.Last();
-         table_name = table_name.Remove(table_name.Length - ".csv".Length);
-         ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
-         _pubs.Add(file_name, vp);
-      }
-   }
-}
+static void ConfigureServices(HostBuilderContext context, IServiceCollection services) =>
+   services.AddDataStreamPublisher(context.Configuration);
@@ -9,7 +9,4 @@ public class PublisherNode {

   [JsonPropertyName("streams")]
   public List<PublishedStream>? Streams { get; set; }
-
-   [JsonPropertyName("csv_dir")]
-   public string? CsvDir { get; set; }
 }
@@ -0,0 +1,39 @@
+namespace Bts.DataStreamPublisher.Services;
+
+using BCG;
+using BCG.Utilities.General;
+
+using bts;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+public class CsvPublisher : IWorker, IDisposable {
+   readonly Dictionary<string, ViewPublisher> publishers = new();
+   readonly string directory;
+   readonly IPubTableBroker pubTableBroker;
+
+   public CsvPublisher(string directory, IPubTableBroker pubTableBroker) {
+      this.directory = directory;
+      this.pubTableBroker = pubTableBroker;
+   }
+
+   public void doWork() {
+      var newFiles = Directory.GetFiles(directory, "*.csv").WhereNot(publishers.ContainsKey);
+      foreach (var file in newFiles) {
+         ViewStream vs = new CSVReaderStream(file);
+         vs = new SchemaGuesserStream(vs);
+         var tableName = Path.GetFileNameWithoutExtension(file);
+         var vp = new ViewPublisher(tableName, pubTableBroker, vs);
+         publishers.Add(file, vp);
+      }
+   }
+
+   public void Dispose() {
+      publishers.Values.DisposeAll();
+      publishers.Clear();
+   }
+}
@@ -0,0 +1,105 @@
+namespace Bts.DataStreamPublisher.Services;
+
+using BCG;
+
+using bts;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamExpressions;
+using Bts.PubSub;
+
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Options;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+/// <summary>
+/// An <see cref="IHostedService"/> that publishes <see cref="ViewStream"/>s to the PubSub.
+/// It can be added to the application's services via <see cref="DataStreamPublisherServices.AddDataStreamPublisher"/>
+/// </summary>
+public class DataStreamPublisherService : IHostedService, IDisposable {
+   readonly DataStreamPublisherConfig config;
+   readonly IPubTableBroker pubTableBroker;
+   readonly IWorkScheduler workScheduler;
+   List<IDisposable>? resources = new();
+
+   public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, IOptions<DataStreamPublisherConfig> config)
+      : this(pubTableBroker, workScheduler, config.Value) { }
+
+   public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, DataStreamPublisherConfig config) {
+      this.pubTableBroker = pubTableBroker;
+      this.workScheduler = workScheduler;
+      this.config = config;
+   }
+
+   public Task StartAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      resources = new();
+      foreach (var publisherNode in config.Publishers.empty_if_null()) {
+         foreach (var publishedStream in publisherNode.Streams.empty_if_null()) {
+            if (string.IsNullOrEmpty(publishedStream.Name)) {
+               Log.Gen.warn("Skipping PublishedStream because it doesn't have a name (must define 'Name' in config json).");
+               continue;
+            }
+            var vs = CreateViewStream(publishedStream);
+            if (vs is null) {
+               continue;
+            }
+            resources.Add(new ViewPublisher(publishedStream.Name, pubTableBroker, vs));
+         }
+      }
+      var csvPublishers = config.CsvDirs.empty_if_null().Select(csvDir => new CsvPublisher(csvDir, pubTableBroker)).ToList();
+      resources.AddRange(csvPublishers);
+      resources.Add(workScheduler.Subscribe(new CompositeWorker(csvPublishers)));
+      return Task.CompletedTask;
+   }
+
+   public Task StopAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Dispose();
+      return Task.CompletedTask;
+   }
+
+   public void Dispose() {
+      resources?.DisposeAll();
+      resources?.Clear();
+   }
+
+   static ViewStream? CreateViewStream(PublishedStream ps) {
+      Log.Gen.info(
+         $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
+      if (string.IsNullOrEmpty(ps.Name)) {
+         Log.Gen.warn(
+            $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
+         return null;
+      }
+      var vs = buildStream(ps.Expression);
+      if (vs is null) {
+         Log.Gen.warn(
+            $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
+         return null;
+      }
+      return vs;
+   }
+
+   static ViewStream? buildStream(string? streamExpression) {
+      if (streamExpression is null) {
+         return null;
+      }
+      try {
+         Log.Gen.info($"Building ViewStream: {streamExpression}");
+         var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression);
+         return exp.getView(new Dictionary<string, StatementExpression>(0));
+      } catch (Exception ex) {
+         Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}");
+         return null;
+      }
+   }
+}
@@ -0,0 +1,62 @@
+namespace Bts.DataStreamPublisher.Services;
+
+using BCG;
+using BCG.Utilities.General;
+
+using bts;
+
+using Bts.DataStream.Services;
+using Bts.DataStreamCore;
+using Bts.DataStreamCore.Services;
+using Bts.DataStreamExpressions;
+using Bts.PubSub.Services;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+
+using Spark.Api.Services;
+using Spark.Core.Services;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+
+public static class DataStreamPublisherServices {
+   public static IServiceCollection AddDataStreamPublisher(this IServiceCollection services, IConfigurationSection configurationSection) =>
+      services.Configure<DataStreamPublisherConfig>(configurationSection)
+         .AddHostedService<DataStreamPublisherService>();
+
+   public static IServiceCollection AddCsvPublishers(this IServiceCollection services, IConfigurationSection configurationSection) {
+      foreach (var csvDir in configurationSection.Get<List<string>>().empty_if_null()) {
+         services.AddSingleton<IWorker, CsvPublisher>(sp => new(csvDir, sp.GetRequiredService<IPubTableBroker>()));
+      }
+      return services;
+   }
+
+   public static IServiceCollection AddDataStreamPublisher(this IServiceCollection services, IConfiguration configurationSection) {
+      services.AddMasterWorker(configurationSection.GetSection("MasterWorker"))
+         .AddViewStreamDatabase(configurationSection.GetSection("ViewStreamDatabase"))
+         .AddPubSubClient(configurationSection.GetSection("PubSub"))
+         .AddPubTableBroker()
+         .AddRemoteQueryManager()
+         .AddDataStreamPublisher(configurationSection.GetSection("DataStreamPublisher"))
+         .AddSpark(configurationSection.GetSection("Spark"))
+         .AddSparkd(configurationSection.GetSection("Sparkd"));
+      return services;
+   }
+
+   static ViewStream? buildStream(string? streamExpression) {
+      if (streamExpression is null) {
+         return null;
+      }
+      try {
+         Log.Gen.info($"build_stream stream_expression: {streamExpression}");
+         var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression);
+         return exp.getView(new Dictionary<string, StatementExpression>(0));
+      } catch (Exception ex) {
+         Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}");
+         return null;
+      }
+   }
+}
@@ -17,14 +17,14 @@
    "publisher_routes": [
      {
        "name": "default",
-        "server_address": "bluto://192.168.1.80:12345",
+        "server_address": "bts-cloud://192.168.1.80:12345",
        "tag": ".*"
      }
    ],
    "subscriber_routes": [
      {
        "name": "default",
-        "server_address": "bluto://192.168.1.80:12345",
+        "server_address": "bts-cloud://192.168.1.80:12345",
        "tag": ".*"
      }
    ],
@@ -58,4 +58,4 @@
    "gc_activation_threshold": 0,
    "wait_period_after_gc": 1000
  }
-}
+}
@@ -36,6 +36,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStreamWebApp.Server", "Bts.DataStreamWebApp.Server\Bts.DataStreamWebApp.Server.csproj", "{6DE5E4CE-423E-4571-81F7-539D268D6059}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_websocket", "..\client\spark_websocket\spark_websocket.csproj", "{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -113,5 +117,13 @@ Global
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Release|Any CPU.Build.0 = Release|Any CPU
+		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Debug|Any CPU.Build.0 = Debug|x64
+		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Release|Any CPU.ActiveCfg = Release|x64
+		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Release|Any CPU.Build.0 = Release|x64
 	EndGlobalSection
 EndGlobal
@@ -38,7 +38,7 @@ namespace bts {
            sub_msg.serialize_into_str(str);
            str.Position = 0;
            var sub_msg_desered = new SubscribeMsg();
-            sub_msg_desered.consruct_deserialize(str);
+            sub_msg_desered.construct_deserialize(str);
            Assert.That(sub_msg_desered.connection, Is.EqualTo(sub_msg.connection));
            Assert.That(sub_msg_desered.subscribing_app, Is.EqualTo(sub_msg.subscribing_app));
            compare_arrays(sub_msg.tags, sub_msg_desered.tags);
@@ -57,7 +57,7 @@ namespace bts {
            unsub_msg.serialize_into_str(str);
            str.Position = 0;
            var sub_msg_desered = new SubscribeMsg();
-            sub_msg_desered.consruct_deserialize(str);
+            sub_msg_desered.construct_deserialize(str);
            Assert.That(sub_msg_desered.connection, Is.EqualTo(unsub_msg.connection));
            Assert.That(sub_msg_desered.subscribing_app, Is.EqualTo(unsub_msg.subscribing_app));
            compare_arrays<string>(unsub_msg.tags, sub_msg_desered.tags);
@@ -80,7 +80,7 @@ namespace bts {
            pub_msg.serialize_into_str(str);
            str.Position = 0;
            PublishMsg pub_msg_desered = new PublishMsg();
-            pub_msg_desered.consruct_deserialize(str);
+            pub_msg_desered.construct_deserialize(str);
            Assert.That(pub_msg_desered.connection, Is.EqualTo(pub_msg.connection));
            Assert.That(pub_msg_desered.publishing_app, Is.EqualTo(pub_msg.publishing_app));
            Assert.That(pub_msg_desered.tag, Is.EqualTo(pub_msg.tag));
@@ -102,7 +102,7 @@ namespace bts {
            ps_server_msg.serialize_into_str(str);
            PubSubServerMessage ps_server_msg_desered = new PubSubServerMessage();
            str.Position = 0;
-            ps_server_msg_desered.consruct_deserialize(str);
+            ps_server_msg_desered.construct_deserialize(str);
            Assert.That(ps_server_msg_desered.tag, Is.EqualTo(ps_server_msg.tag));
            Assert.That(ps_server_msg_desered.tag_type, Is.EqualTo(ps_server_msg.tag_type));
            compare_arrays(ps_server_msg.data, ps_server_msg_desered.data);
@@ -125,7 +125,7 @@ namespace bts {
            msg.serialize_into_str(str);
            PubSubClientMessage msg_deser = new PubSubClientMessage();
            str.Position = 0;
-            msg_deser.consruct_deserialize(str);
+            msg_deser.construct_deserialize(str);
            Assert.That(msg_deser.msg_type, Is.EqualTo(msg.msg_type));
            Assert.That(msg_deser.sub_msg, Is.EqualTo(msg.sub_msg));
        }
@@ -30,15 +30,15 @@ namespace BCG {
   public class PubSubServerManagedTCPConnection : IPubSubServerConnection,// external facing interface to communicate msgs to Pubsubserver   
                                                   ITCPMessageHandler,     // to get messages from ITCPCOnnection
                                                   ITCPConnectionHandler { // to get connections from AsyncTCPConnector_APM
-      private ITCPConnection _connection;
+      private ITCPConnection? _connection;
      private readonly RemoteConnectionInfo _server_rc_info;
      private readonly bool _is_sub;
      // if _data_callback is not subscribed to , the user of this class can poll updates,
      // in which case this queue will be used
      private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>();
      private DateTime _last_hb = DateTime.Now;
-      private PubSubServerConnectionStatusChange _conn_callback;
-      private PubSubServerDataDelegate _data_callback;
+      private PubSubServerConnectionStatusChange? _conn_callback;
+      private PubSubServerDataDelegate? _data_callback;
      private readonly TimeSpan _connection_drop_threshold = PubSubConfig.Instance.ConnectionDropThreshold();
      private ThreadSafeQueue<ConnectionStatusChange> _conn_cb_queue = new ThreadSafeQueue<ConnectionStatusChange>();

@@ -51,10 +51,10 @@ namespace BCG {
            DISCONNECTED
         }
         public readonly Status connection_satus;
-         public readonly ITCPConnection new_connection = null;
-         public ConnectionStatusChange(Status connection_satus_, ITCPConnection new_connection_) {
-            connection_satus = connection_satus_;
-            new_connection = new_connection_;
+         public readonly ITCPConnection? new_connection;
+         public ConnectionStatusChange(Status connection_satus, ITCPConnection? new_connection) {
+            this.connection_satus = connection_satus;
+            this.new_connection = new_connection;
         }
         public override string ToString() {
            return connection_satus.ToString();
@@ -101,7 +101,7 @@ namespace BCG {
         var i = 0;
         foreach (var b in rv_raw) {
            PubSubServerMessage ps_msg = new PubSubServerMessage();
-            ps_msg.consruct_deserialize(b);
+            ps_msg.construct_deserialize(b);
            bts.Log.Gen.info($"PubSubServerManagedTCPConnection.get_updates downloaded: {ps_msg} msg_no: {i} / {rv_raw.Count}");
            rv.Add(ps_msg);
            i++;
@@ -162,10 +162,12 @@ namespace BCG {
         // TODO check for disconenction
         switch (message.messageType) {
            case RemoteMessageType.Data:
-               if (_data_callback != null)
+               ArgumentNullException.ThrowIfNull(message.message);
+               if (_data_callback != null) {
                  _data_callback(message.remoteConnectionInfo.ToString(), message.message);
-               else
+               } else {
                  _in_data_queue.enqueue(message.message);
+               }
               break;
            case RemoteMessageType.Heartbeat:
               _last_hb = DateTime.Now;
@@ -200,20 +202,6 @@ namespace BCG {

      #endregion

-      void _check_missing_hb() {
-         if (!_connection.isDisposed) {
-            TimeSpan hb_interval = DateTime.Now - _last_hb;
-            bool hb_missed = hb_interval > _connection_drop_threshold;
-            if (hb_missed) {
-               bts.Log.Gen.warn("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}");
-               ConnectionStatusChange ci = new ConnectionStatusChange(ConnectionStatusChange.Status.DISCONNECTED, null);
-               _conn_cb_queue.enqueue(ci);
-            }
-         } else {
-            // already disposed, nothing to do
-         }
-      }
-
      private static string _connection_txt(ITCPConnection conn) {
         if (conn != null) {
            return $"conn_type: {conn.GetType()} server: {conn.get_peer_address()} self: {conn.get_self_address()}";
@@ -233,22 +221,22 @@ namespace BCG {
         if (_conn_cb_queue.getCount() > 0) {
            var css = _conn_cb_queue.dequeueAll();
            foreach (var c in css) {
+               Log.notnull(c);
               Log.Gen.info($"PubSubServerManagedTCPConnection.doWork c: {c}");
               switch (c.connection_satus) {
                  case ConnectionStatusChange.Status.CONNECTED: {
-                        Log.assert(c.new_connection != null);
+                        Log.notnull(c.new_connection);
                        c.new_connection.register(this);
                        _connection = c.new_connection;
                        _last_hb = DateTime.Now;
-                        if (_conn_callback != null)
-                           _conn_callback(_server_rc_info.ToString(), true);
+                        _conn_callback?.Invoke(_server_rc_info.ToString(), true);
                        break;
                     }
                  case ConnectionStatusChange.Status.DISCONNECTED: {
+                        Log.notnull(_connection);
                        _connection.Dispose();
                        _connect_to_ps_server();
-                        if (_conn_callback != null)
-                           _conn_callback(_server_rc_info.ToString(), false);
+                        _conn_callback?.Invoke(_server_rc_info.ToString(), false);
                        break;
                     }
               }
@@ -257,6 +245,5 @@ namespace BCG {
      }

      #endregion
-
   }
 }
@@ -1,41 +1,23 @@
-using System;
-using System.IO;
-using BCG.Utilities.General;
-using System.Collections.Generic;
-using mbts;
-
 namespace BCG {
-   public delegate void InternalPublishDelegate(ICollection<Triplet<string, string, byte[]>> pubData);
   /// <summary>
   /// The Pubsub holds an instance of this for every tag
   /// type. this interface holds the state of each tag
   /// in the pubsub.
   /// </summary>
-   public interface IPubSubServerTagHandler : IWorker {
+   public interface IPubSubServerTagHandler {
+      string TagType { get; }
      /// <summary>
      /// This is called when a new data gets published, so IPubSubServerTagHandler can
      ///interpret the data and update its state (which it will be asked to return in get_content)
      /// </summary>
      /// <param name="tag"></param>
      /// <param name="messageBodies"></param>
-      void update(string tag, byte[] msg);
+      void update(string tag, byte[]? msg);
      /// <summary>
      /// gets the entire content of the tag
      /// </summary>
      /// <param name="tag"></param>
      /// <returns></returns>
      byte[] get_content(string tag);
-      /// <summary>
-      /// list of tagTypes this interface supports
-      /// </summary>
-      string[] supportedTagTypes { get; }
-      /// <summary>
-      /// registering a callback gives this interface
-      /// the extra flexibility to publish its own
-      /// data back to the pubsub. this function will
-      /// be ignored executed outside the pubsub server
-      /// </summary>
-      /// <param name="callback"></param>
-      void register(InternalPublishDelegate callback);
   }
 }
@@ -0,0 +1,7 @@
+namespace Bts.PubSub;
+
+using BCG;
+
+public interface IPubSubTagHandlerFactory {
+   IPubSubServerTagHandler? GetHandler(string tagType);
+}
@@ -1,364 +1,405 @@
+using BCG;
+
+using bts.collections;
+using bts.utils;
+
 using System;
-using System.IO;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;

-namespace bts {
-    using BCG;
-
-    using System.IO;
-    using System.Linq;
-
-
-    public interface IPubSubMessage {
-        void serialize_into_str(Stream str);
-        void consruct_deserialize(Stream str);
-    }
-    public static class PubSubMessageUtils {
-        public static bool compare_obj<T>(T t1, T t2) {
-            if (t1 == null) {
-                return t2 == null;
-            } else
-                return t1.Equals(t2);
-        }
-        public static bool compare_arrays<T>(IEnumerable<T> col1, IEnumerable<T> col2) {
-            if (col1.Count() != col2.Count())
-                return false;
-            int i = 0;
-            foreach (T t in col1) {
-                T t1 = t;
-                T t2 = col2.ElementAt(i++);
-                var eqs = compare_obj(t1, t2);
-                if (!eqs)
-                    return false;
-            }
-            return true;
-        }
-    }
-    // Subscribe //////////////////////////////////////////////////////////////
-    ///////////////////////////////////////////////////////////////////////////
-    public class SubscribeMsg : IPubSubMessage {
-
-        public string connection;
-        public string subscribing_app = ClientInfo.thisClient.app;
-        public string[] tags;
-        public void serialize_into_str(Stream str) {
-            BinaryWriter bw = new BinaryWriter(str);
-            bw.write_string_cpp(connection);
-            bw.write_string_cpp(subscribing_app);
-            bw.Write(tags.Length);
-            foreach (var t in tags)
-                bw.write_string_cpp(t);
-        }
-
-        public void consruct_deserialize(Stream str) {
-            BinaryReader br = new BinaryReader(str);
-            this.connection = br.read_string_cpp();
-            this.subscribing_app = br.read_string_cpp();
-            var n_tags = br.ReadInt32();
-            tags = new string[n_tags];
-            for (var i = 0; i < n_tags; i++)
-                tags[i] = br.read_string_cpp();
-        }
-        public override string ToString() {
-            var tgs_str = String.Empty;
-            foreach (var t in tags)
-                tgs_str += t;
-            return $"{GetType().Name} connection: {connection} subscribing_app: {subscribing_app} tags: {tgs_str}";
-        }
-        public override bool Equals(object? obj) {
-            if (obj != null && obj is SubscribeMsg) {
-                var obj_sub_msg = obj as SubscribeMsg;
-                return PubSubMessageUtils.compare_obj(this.connection, obj_sub_msg.connection) &&
-                PubSubMessageUtils.compare_obj(this.subscribing_app, obj_sub_msg.subscribing_app) &&
-                PubSubMessageUtils.compare_arrays(tags, obj_sub_msg.tags);
-                // TODO compare tags
-            }
-            return false;
-        }
-
-        public override int GetHashCode() {
-            var str = ToString();
-            return str.GetHashCode();
-        }
-    }
-
-    // Unsubscribe ////////////////////////////////////////////////////////////
-    ///////////////////////////////////////////////////////////////////////////
-    public class UnsubscribeMsg : SubscribeMsg {
-    }
-
-    // Tasnaction /////////////////////////////////////////////////////////////
-    ///////////////////////////////////////////////////////////////////////////
-    public class TransactionMsg : IPubSubMessage {
-        uint n_msgs;
-        public void serialize_into_str(Stream str) {
-            BinaryWriter bw = new BinaryWriter(str);
-            bw.Write(n_msgs);
-        }
-        public void consruct_deserialize(Stream str) {
-            BinaryReader br = new BinaryReader(str);
-            n_msgs = br.ReadUInt32();
-        }
-        public override string ToString() {
-            return $"TransactionMsg n_msgs: {n_msgs}";
-        }
-
-        public override bool Equals(object? obj) {
-            if (obj != null && (obj is TransactionMsg)) {
-                TransactionMsg other_ = obj as TransactionMsg;
-                return n_msgs == other_.n_msgs;
-            }
-            return false;
-        }
-    }
-
-    // Publish ////////////////////////////////////////////////////////////////
-    /// ///////////////////////////////////////////////////////////////////////
-    public class PublishMsg : IPubSubMessage {
-        public string connection;
-        public string publishing_app = ClientInfo.thisClient.app;
-        public string tag;
-        public string tag_type;
-        public bool is_compressed = false;
-        // this is the size that of the msg, regarless of it being serialized or not
-        public long publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-        public byte[] data_to_publish;
-
-        public void sanity_check() {
-            Log.assert(!String.IsNullOrEmpty(tag_type));
-            Log.assert(!String.IsNullOrEmpty(tag));
-            Log.assert(!String.IsNullOrEmpty(connection));
-        }
-
-        public void serialize_into_str(Stream str) {
-            sanity_check();
-            BinaryWriter bw = new BinaryWriter(str);
-            bw.write_string_cpp(connection);
-            bw.write_string_cpp(publishing_app);
-            bw.write_string_cpp(tag);
-            bw.write_string_cpp(tag_type);
-            bw.Write(is_compressed);
-            bw.Write(publish_time_epoch_ms);
-            var sz = data_to_publish.Length;
-            bw.Write(sz);
-            bw.Write(data_to_publish);
-        }
-        public void consruct_deserialize(Stream str) {
-            BinaryReader br = new BinaryReader(str);
-            connection = br.read_string_cpp();
-            publishing_app = br.read_string_cpp();
-            tag = br.read_string_cpp();
-            tag_type = br.read_string_cpp();
-            is_compressed = br.ReadBoolean();
-            publish_time_epoch_ms = br.ReadInt64();
-            var size = br.ReadInt32();
-            data_to_publish = br.ReadBytes(size);
-        }
-        public override string ToString() {
-            var data_txt = StreamUtilityMethods.first_n_bytes_to_string(data_to_publish, 4);
-            return $"PublishMsg connection: {connection} publishing_app: {publishing_app} " +
-                    $"tag: {tag} tag_type: {tag_type} is_compressed: {is_compressed} " +
-                    $"publish_time_epoch_ms: {publish_time_epoch_ms} data len: {data_to_publish.Length} " +
-                    $"data (4): {data_txt}";
-        }
-
-        public override bool Equals(object? obj) {
-            if (obj != null && obj is PublishMsg) {
-                PublishMsg pub_msg_ = obj as PublishMsg;
-                return connection.Equals(pub_msg_.connection) &&
-                publishing_app.Equals(pub_msg_.publishing_app) &&
-                tag.Equals(pub_msg_.tag) &&
-                tag_type.Equals(pub_msg_.tag_type) &&
-                is_compressed.Equals(pub_msg_.is_compressed) &&
-                publish_time_epoch_ms.Equals(pub_msg_.publish_time_epoch_ms) &&
-                PubSubMessageUtils.compare_arrays(data_to_publish, pub_msg_.data_to_publish);
-            }
-            return false;
-        }
-
-    }
-    public enum PubSubMessageType {
-        SUBSCRIBE = 0,
-        UNSUBSCRIIBE,
-        TRANSACTION,
-        PUBLISH
-    }
-
-    public class PubSubClientMessage : IPubSubMessage {
-        public PubSubMessageType msg_type;
-        // only one of these wil be non null
-        public SubscribeMsg sub_msg = null;
-        public UnsubscribeMsg unsub_msg = null;
-        public TransactionMsg transaction_msg = null;
-        public PublishMsg pub_msg = null;
-
-        public PubSubClientMessage() { }
-
-        public PubSubClientMessage(SubscribeMsg sub_msg_) {
-            sub_msg = sub_msg_;
-            msg_type = PubSubMessageType.SUBSCRIBE;
-        }
-
-        public PubSubClientMessage(UnsubscribeMsg unsub_msg_) {
-            unsub_msg = unsub_msg_;
-            msg_type = PubSubMessageType.UNSUBSCRIIBE;
-        }
-
-        public PubSubClientMessage(TransactionMsg transaction_msg_) {
-            transaction_msg = transaction_msg_;
-            msg_type = PubSubMessageType.TRANSACTION;
-        }
-
-        public PubSubClientMessage(PublishMsg pub_msg_) {
-            pub_msg = pub_msg_;
-            msg_type = PubSubMessageType.PUBLISH;
-        }
-
-        public void serialize_into_str(Stream str) {
-            BinaryWriter bw = new BinaryWriter(str);
-            bw.Write((int)msg_type);
-            switch (msg_type) {
-                case PubSubMessageType.SUBSCRIBE:
-                    sub_msg.serialize_into_str(str);
-                    break;
-                case PubSubMessageType.UNSUBSCRIIBE:
-                    unsub_msg.serialize_into_str(str);
-                    break;
-                case PubSubMessageType.TRANSACTION:
-                    transaction_msg.serialize_into_str(str);
-                    break;
-                case PubSubMessageType.PUBLISH:
-                    pub_msg.serialize_into_str(str);
-                    break;
-            }
-        }
-
-        public byte[] serialize_into_bytes() {
-            MemoryStream ms = new MemoryStream();
-            BinaryReader br = new BinaryReader(ms);
-            serialize_into_str(ms);
-            var len = Convert.ToInt32(ms.Position);
-            ms.Position = 0;
-            return br.ReadBytes(len);
-        }
-
-        public void consruct_deserialize(Stream str) {
-            BinaryReader br = new BinaryReader(str);
-            int type = br.ReadInt32();
-            msg_type = (PubSubMessageType)type;
-            switch (msg_type) {
-                case PubSubMessageType.SUBSCRIBE:
-                    sub_msg = new SubscribeMsg();
-                    sub_msg.consruct_deserialize(str);
-                    break;
-                case PubSubMessageType.UNSUBSCRIIBE:
-                    unsub_msg = new UnsubscribeMsg();
-                    unsub_msg.consruct_deserialize(str);
-                    break;
-                case PubSubMessageType.TRANSACTION:
-                    transaction_msg = new TransactionMsg();
-                    transaction_msg.consruct_deserialize(str);
-                    break;
-                case PubSubMessageType.PUBLISH:
-                    pub_msg = new PublishMsg();
-                    pub_msg.consruct_deserialize(str);
-                    break;
-            }
-        }
-
-        public void consruct_deserialize(byte[] msg) {
-            MemoryStream ms = new MemoryStream(msg);
-            ms.Position = 0;
-            consruct_deserialize(ms);
-        }
-
-        public override string ToString() {
-            var rv = $"type: {msg_type} ";
-            switch (msg_type) {
-                case PubSubMessageType.SUBSCRIBE:
-                    rv += $"{sub_msg}";
-                    break;
-                case PubSubMessageType.UNSUBSCRIIBE:
-                    rv += $"{unsub_msg}";
-                    break;
-                case PubSubMessageType.TRANSACTION:
-                    rv += $"{transaction_msg}";
-                    break;
-                case PubSubMessageType.PUBLISH:
-                    rv += $"{pub_msg}";
-                    break;
-            }
-            return rv;
-        }
-
-        public override bool Equals(object? obj) {
-            if (obj != null && obj is PubSubClientMessage) {
-                var msg_ = obj as PubSubClientMessage;
-                return PubSubMessageUtils.compare_obj(msg_type, msg_.msg_type) &&
-                PubSubMessageUtils.compare_obj(sub_msg, msg_.sub_msg) &&
-                PubSubMessageUtils.compare_obj(pub_msg, msg_.pub_msg) &&
-                PubSubMessageUtils.compare_obj(transaction_msg, msg_.transaction_msg) &&
-                PubSubMessageUtils.compare_obj(unsub_msg, msg_.unsub_msg);
-            }
-            return false;
-        }
-
-
-    }
-
-    // 
-    public class PubSubServerMessage : IPubSubMessage {
-        public string tag;
-        public string tag_type;
-        public byte[] data;
-        public void serialize_into_str(Stream str) {
-            BinaryWriter bw = new BinaryWriter(str);
-            bw.write_string_cpp(tag);
-            bw.write_string_cpp(tag_type);
-            bw.Write(data.Length);
-            bw.Write(data);
-        }
-        public byte[] serialize_into_bytes() {
-            MemoryStream ms = new MemoryStream();
-            serialize_into_str(ms);
-            var pos = Convert.ToInt32(ms.Position);
-            ms.Position = 0;
-            BinaryReader br = new BinaryReader(ms);
-            return br.ReadBytes(pos);
-        }
-        public void consruct_deserialize(Stream str) {
-            BinaryReader br = new BinaryReader(str);
-            tag = br.read_string_cpp();
-            tag_type = br.read_string_cpp();
-            var data_sz = br.ReadInt32();
-            data = br.ReadBytes(data_sz);
-        }
-
-        public void consruct_deserialize(byte[] msg) {
-            MemoryStream str = new MemoryStream(msg);
-            str.Position = 0;
-            consruct_deserialize(str);
-        }
-
-        public override string ToString() {
-            var data_txt = StreamUtilityMethods.first_n_bytes_to_string(data, 4);
-            return $"PubSubServerMessage tag: {tag} tag_type: {tag_type} data_len: {data.Length} " +
-                   $" data: {data_txt}";
-        }
-
-        public override bool Equals(object? obj) {
-            if (obj != null && obj is PubSubServerMessage) {
-                PubSubServerMessage msg_ = obj as PubSubServerMessage;
-                return PubSubMessageUtils.compare_obj(tag, msg_.tag) &&
-                PubSubMessageUtils.compare_obj(tag_type, msg_.tag_type) &&
-                PubSubMessageUtils.compare_arrays(data, msg_.data);
-            }
-            return false;
-        }
-
-    }
+namespace bts;

+public interface IPubSubMessage {
+   void serialize_into_str(Stream str);
+   void construct_deserialize(Stream str);
 }

+public static class PubSubMessageUtils {
+   public static bool compare_obj<T>(T t1, T t2) {
+      if (t1 == null) {
+         return t2 == null;
+      } else
+         return t1.Equals(t2);
+   }
+   public static bool compare_arrays<T>(IEnumerable<T> col1, IEnumerable<T> col2) {
+      if (col1.Count() != col2.Count())
+         return false;
+      int i = 0;
+      foreach (T t in col1) {
+         T t1 = t;
+         T t2 = col2.ElementAt(i++);
+         var eqs = compare_obj(t1, t2);
+         if (!eqs)
+            return false;
+      }
+      return true;
+   }
+}

+// Subscribe //////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
+public class SubscribeMsg : IPubSubMessage, IEquatable<SubscribeMsg> {
+   public string? connection;
+   public string subscribing_app = ClientInfo.thisClient.app;
+   public string[]? tags;

+   public void serialize_into_str(Stream str) {
+      if (tags is null) {
+         throw new InvalidOperationException("Unable to serialize SubscribeMsg because tags is null");
+      }
+      var bw = new BinaryWriter(str);
+      bw.write_string_cpp(connection);
+      bw.write_string_cpp(subscribing_app);
+      bw.Write(tags.Length);
+      foreach (var t in tags) {
+         bw.write_string_cpp(t);
+      }
+   }
+
+   public void construct_deserialize(Stream str) {
+      var br = new BinaryReader(str);
+      connection = br.read_string_cpp();
+      subscribing_app = br.read_string_cpp();
+      var n_tags = br.ReadInt32();
+      tags = new string[n_tags];
+      for (var i = 0; i < n_tags; i++) {
+         tags[i] = br.read_string_cpp();
+      }
+   }
+
+   public bool Equals(SubscribeMsg? other) =>
+      other is not null
+         && connection == other.connection
+         && subscribing_app == other.subscribing_app
+         && tags.safe_sequence_equals(other.tags);
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as SubscribeMsg);
+
+   public override int GetHashCode() =>
+      ToString().GetHashCode();
+
+   public override string ToString() =>
+      $"{GetType().Name} connection: {connection} subscribing_app: {subscribing_app} tags: {string.Concat(tags.empty_if_null())}";
+}
+
+// Unsubscribe ////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
+public class UnsubscribeMsg : SubscribeMsg, IEquatable<UnsubscribeMsg> {
+   public bool Equals(UnsubscribeMsg? other) =>
+      other is not null && base.Equals(other);
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as UnsubscribeMsg);
+
+   public override int GetHashCode() =>
+      base.GetHashCode();
+}
+
+// Trasnaction /////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
+public class TransactionMsg : IPubSubMessage, IEquatable<TransactionMsg> {
+   uint n_msgs;
+
+   public void serialize_into_str(Stream str) {
+      var bw = new BinaryWriter(str);
+      bw.Write(n_msgs);
+   }
+
+   public void construct_deserialize(Stream str) {
+      var br = new BinaryReader(str);
+      n_msgs = br.ReadUInt32();
+   }
+
+   public override string ToString() =>
+      $"TransactionMsg n_msgs: {n_msgs}";
+
+   public bool Equals(TransactionMsg? other) =>
+      n_msgs == other?.n_msgs;
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as TransactionMsg);
+
+   public override int GetHashCode() =>
+      n_msgs.GetHashCode();
+}
+
+// Publish ////////////////////////////////////////////////////////////////
+/// ///////////////////////////////////////////////////////////////////////
+public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
+   public string? connection;
+   public string publishing_app = ClientInfo.thisClient.app;
+   public string? tag;
+   public string? tag_type;
+   public bool is_compressed = false;
+   // this is the size that of the msg, regarless of it being serialized or not
+   public long publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+   public byte[]? data_to_publish;
+
+   [MemberNotNull(nameof(connection), nameof(tag), nameof(tag_type))]
+   public void sanity_check() {
+      Log.assert(!string.IsNullOrEmpty(tag_type));
+      Log.assert(!string.IsNullOrEmpty(tag));
+      Log.assert(!string.IsNullOrEmpty(connection));
+      // The 'ThrowIfNullOrEmpty' calls are redundant (given the assert statements above), but
+      // necessary for the compiler to recognize the strings won't be null when this method exits.
+      ArgumentException.ThrowIfNullOrEmpty(tag_type);
+      ArgumentException.ThrowIfNullOrEmpty(tag);
+      ArgumentException.ThrowIfNullOrEmpty(connection);
+   }
+
+   public void serialize_into_str(Stream str) {
+      sanity_check();
+      if (data_to_publish is null) {
+         throw new InvalidOperationException("Unable to serialize PublishMsg because data_to_publish is null");
+      }
+      var bw = new BinaryWriter(str);
+      bw.write_string_cpp(connection);
+      bw.write_string_cpp(publishing_app);
+      bw.write_string_cpp(tag);
+      bw.write_string_cpp(tag_type);
+      bw.Write(is_compressed);
+      bw.Write(publish_time_epoch_ms);
+      var sz = data_to_publish.Length;
+      bw.Write(sz);
+      bw.Write(data_to_publish);
+   }
+
+   public void construct_deserialize(Stream str) {
+      var br = new BinaryReader(str);
+      connection = br.read_string_cpp();
+      publishing_app = br.read_string_cpp();
+      tag = br.read_string_cpp();
+      tag_type = br.read_string_cpp();
+      is_compressed = br.ReadBoolean();
+      publish_time_epoch_ms = br.ReadInt64();
+      var size = br.ReadInt32();
+      data_to_publish = br.ReadBytes(size);
+   }
+
+   public bool Equals(PublishMsg? other) =>
+      other is not null 
+         && connection == other.connection
+         && publishing_app == other.publishing_app
+         && tag == other.tag
+         && tag_type == other.tag_type
+         && is_compressed == other.is_compressed
+         && publish_time_epoch_ms == other.publish_time_epoch_ms
+         && data_to_publish.safe_sequence_equals(other.data_to_publish);
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as PublishMsg);
+
+   public override int GetHashCode() =>
+      HashCode.Combine(
+         connection,
+         publishing_app,
+         tag,
+         is_compressed,
+         publish_time_epoch_ms,
+         data_to_publish?.Length);
+
+   public override string ToString() {
+      var data_txt = StreamUtilityMethods.first_n_bytes_to_string(data_to_publish ?? Array.Empty<byte>(), 4);
+      return $"PublishMsg connection: {connection} publishing_app: {publishing_app} "
+         + $"tag: {tag} tag_type: {tag_type} is_compressed: {is_compressed} "
+         + $"publish_time_epoch_ms: {publish_time_epoch_ms} data len: {data_to_publish?.Length ?? 0} "
+         + $"data (4): {data_txt}";
+   }
+}
+
+public enum PubSubMessageType {
+   SUBSCRIBE = 0,
+   UNSUBSCRIIBE,
+   TRANSACTION,
+   PUBLISH
+}
+
+public class PubSubClientMessage : IPubSubMessage {
+   public PubSubMessageType msg_type;
+   // only one of these wil be non null
+   public SubscribeMsg? sub_msg;
+   public UnsubscribeMsg? unsub_msg;
+   public TransactionMsg? transaction_msg;
+   public PublishMsg? pub_msg;
+
+   public PubSubClientMessage() { }
+
+   public PubSubClientMessage(SubscribeMsg sub_msg_) {
+      sub_msg = sub_msg_;
+      msg_type = PubSubMessageType.SUBSCRIBE;
+   }
+
+   public PubSubClientMessage(UnsubscribeMsg unsub_msg_) {
+      unsub_msg = unsub_msg_;
+      msg_type = PubSubMessageType.UNSUBSCRIIBE;
+   }
+
+   public PubSubClientMessage(TransactionMsg transaction_msg_) {
+      transaction_msg = transaction_msg_;
+      msg_type = PubSubMessageType.TRANSACTION;
+   }
+
+   public PubSubClientMessage(PublishMsg pub_msg_) {
+      pub_msg = pub_msg_;
+      msg_type = PubSubMessageType.PUBLISH;
+   }
+
+   public void serialize_into_str(Stream str) {
+      var bw = new BinaryWriter(str);
+      bw.Write((int)msg_type);
+      switch (msg_type) {
+         case PubSubMessageType.SUBSCRIBE:
+            ValidateMsg(sub_msg).serialize_into_str(str);
+            break;
+         case PubSubMessageType.UNSUBSCRIIBE:
+            ValidateMsg(unsub_msg).serialize_into_str(str);
+            break;
+         case PubSubMessageType.TRANSACTION:
+            ValidateMsg(transaction_msg).serialize_into_str(str);
+            break;
+         case PubSubMessageType.PUBLISH:
+            ValidateMsg(pub_msg).serialize_into_str(str);
+            break;
+
+         T ValidateMsg<T>([NotNull]T? msg, [CallerArgumentExpression(nameof(msg))]string name = "") {
+            if (msg is null) {
+               throw new InvalidOperationException($"msg type is {msg_type} but {name} is null");
+            }
+            return msg;
+         }
+      }
+   }
+
+   public byte[] serialize_into_bytes() {
+      var ms = new MemoryStream();
+      var br = new BinaryReader(ms);
+      serialize_into_str(ms);
+      var len = Convert.ToInt32(ms.Position);
+      ms.Position = 0;
+      return br.ReadBytes(len);
+   }
+
+   public void construct_deserialize(Stream str) {
+      var br = new BinaryReader(str);
+      int type = br.ReadInt32();
+      msg_type = (PubSubMessageType)type;
+      switch (msg_type) {
+         case PubSubMessageType.SUBSCRIBE:
+            sub_msg = new SubscribeMsg();
+            sub_msg.construct_deserialize(str);
+            break;
+         case PubSubMessageType.UNSUBSCRIIBE:
+            unsub_msg = new UnsubscribeMsg();
+            unsub_msg.construct_deserialize(str);
+            break;
+         case PubSubMessageType.TRANSACTION:
+            transaction_msg = new TransactionMsg();
+            transaction_msg.construct_deserialize(str);
+            break;
+         case PubSubMessageType.PUBLISH:
+            pub_msg = new PublishMsg();
+            pub_msg.construct_deserialize(str);
+            break;
+      }
+   }
+
+   public void consruct_deserialize(byte[] msg) =>
+      construct_deserialize(new MemoryStream(msg) {
+         Position = 0
+      });
+
+   public bool Equals(PubSubClientMessage? other) =>
+      other is not null
+         && msg_type == other.msg_type
+         && sub_msg.safe_equals(other.sub_msg)
+         && pub_msg.safe_equals(other.pub_msg)
+         && transaction_msg.safe_equals(other.transaction_msg)
+         && unsub_msg.safe_equals(other.unsub_msg);
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as PubSubClientMessage);
+
+   public override int GetHashCode() {
+      var msgHash = msg_type switch {
+         PubSubMessageType.SUBSCRIBE => sub_msg?.GetHashCode(),
+         PubSubMessageType.UNSUBSCRIIBE => unsub_msg?.GetHashCode(),
+         PubSubMessageType.TRANSACTION => transaction_msg?.GetHashCode(),
+         PubSubMessageType.PUBLISH => pub_msg?.GetHashCode(),
+         _ => 0,
+      };
+      return HashCode.Combine(msg_type, msgHash);
+   }
+
+   public override string ToString() {
+      var rv = $"type: {msg_type} ";
+      rv += msg_type switch {
+         PubSubMessageType.SUBSCRIBE => $"{sub_msg}",
+         PubSubMessageType.UNSUBSCRIIBE => $"{sub_msg}",
+         PubSubMessageType.TRANSACTION => $"{sub_msg}",
+         PubSubMessageType.PUBLISH => $"{sub_msg}",
+         _ => "Invalid msg type",
+      };
+      return rv;
+   }
+}
+
+public class PubSubServerMessage : IPubSubMessage, IEquatable<PubSubServerMessage> {
+   public string? tag;
+   public string? tag_type;
+   public byte[]? data;
+
+   public void serialize_into_str(Stream str) {
+      if (data is null) {
+         throw new InvalidOperationException("Unable to serialize message because data is null");
+      }
+      var bw = new BinaryWriter(str);
+      bw.write_string_cpp(tag);
+      bw.write_string_cpp(tag_type);
+      bw.Write(data.Length);
+      bw.Write(data);
+   }
+
+   public byte[] serialize_into_bytes() {
+      var ms = new MemoryStream();
+      serialize_into_str(ms);
+      var pos = Convert.ToInt32(ms.Position);
+      ms.Position = 0;
+      var br = new BinaryReader(ms);
+      return br.ReadBytes(pos);
+   }
+
+   [MemberNotNull(nameof(tag), nameof(tag_type))]
+   public void construct_deserialize(Stream str) {
+      var br = new BinaryReader(str);
+      tag = br.read_string_cpp();
+      tag_type = br.read_string_cpp();
+      var data_sz = br.ReadInt32();
+      data = br.ReadBytes(data_sz);
+   }
+
+   [MemberNotNull(nameof(tag), nameof(tag_type))]
+   public void construct_deserialize(byte[] msg) =>
+      construct_deserialize(new MemoryStream(msg) { Position = 0});
+
+   public bool Equals(PubSubServerMessage? other) =>
+      other is not null
+         && tag == other.tag
+         && tag_type == other.tag_type
+         && PubSubMessageUtils.compare_arrays(data.empty_if_null(), other.data.empty_if_null());
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as PubSubServerMessage);
+
+   public override int GetHashCode() =>
+      HashCode.Combine(tag, tag_type, data?.Length);
+
+   public override string ToString() {
+      var data_txt = StreamUtilityMethods.first_n_bytes_to_string(data ?? Array.Empty<byte>(), 4);
+      return $"PubSubServerMessage tag: {tag} tag_type: {tag_type} data_len: {data?.Length ?? 0} " +
+             $" data: {data_txt}";
+   }
+}
@@ -0,0 +1,42 @@
+namespace Bts.PubSub;
+
+using BCG;
+
+using System.Collections.Generic;
+
+public class PubSubTagTypeHandlerConfig {
+   public string? TagType { get; set; }
+   public string? Assembly { get; set; }
+   public string? Class { get; set; }
+}
+
+public class PubSubTagHandlerFactoryConfig {
+   public List<PubSubTagTypeHandlerConfig>? Handlers { get; set; }
+
+   public static PubSubTagHandlerFactoryConfig Default() {
+      return new() {
+         Handlers = new() {
+            new() {
+               TagType = string.Empty,
+               Assembly = "DataStreamCore",
+               Class = "bts.PubsubServerStreamTagHandler",
+            },
+            new() {
+               TagType = DSTagManager.DataStreamTagType,
+               Assembly = "DataStreamCore",
+               Class = "bts.PubsubServerStreamTagHandler",
+            },
+            new() {
+               TagType = "System_Sub",
+               Assembly = "PubSub",
+               Class = "BCG.SystemSubscriptionMapPublisher",
+            },
+            new() {
+               TagType = "System_Stats",
+               Assembly = "PubSub",
+               Class = "BCG.SystemStatsPublisher",
+            }
+         }
+      };
+   }
+}
@@ -1,76 +1,106 @@
 using System;
-using System.Collections;
 using System.Reflection;
-using BCG.Utilities;
 using System.Collections.Generic;
 using BCG;
-using System.Runtime.InteropServices;
 using System.IO;
-
-
+using System.Linq;
+using Bts.PubSub;
+using Microsoft.Extensions.Options;
+using System.Text.RegularExpressions;
+using System.Text.Json;
+using Microsoft.Extensions.DependencyInjection;

 namespace bts {
-   public static class PubsubTagHandlerFactory {
-      // each tag type is represented by this
-      public class TagHandlerEntry {
-         public readonly string tag_type;
-         public readonly string assembly_name;
-         public readonly string handler_class_name;
-         public TagHandlerEntry(string tag_type_, string assembly_name_, string handler_class_name_) {
-            tag_type = tag_type_;
-            assembly_name = assembly_name_;
-            handler_class_name = handler_class_name_;
-         }
+   public class PubsubTagHandlerFactory : IPubSubTagHandlerFactory {
+      readonly List<TagTypeHandlerData> handlers;
+      readonly IServiceProvider serviceProvider;
+
+      public PubsubTagHandlerFactory(IServiceProvider serviceProvider, IOptions<PubSubTagHandlerFactoryConfig> config)
+         : this(serviceProvider, config.Value) { }
+
+      public PubsubTagHandlerFactory(IServiceProvider serviceProvider, PubSubTagHandlerFactoryConfig config) {
+         this.serviceProvider = serviceProvider;
+         Log.notnull(config.Handlers);
+         handlers = config.Handlers.Select(h => new TagTypeHandlerData(h)).ToList();
      }
-      private static Dictionary<string, TagHandlerEntry> _config_map = new Dictionary<string, TagHandlerEntry>()
-      {
-         {String.Empty, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")},
-      {DSTagManager.DataStreamTagType, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")},
-      {"System_Sub", new TagHandlerEntry("System_Sub", "PubSub", "BCG.SystemSubscriptionMapPublisher")},
-      {"System_Stats", new TagHandlerEntry("System_Stats", "PubSub", "BCG.SystemStatsPublisher")}
+
+      public IPubSubServerTagHandler? GetHandler(string tag_type) {
+         if (_handler_map.TryGetValue(tag_type, out var handler)) {
+            return handler;
+         }
+         return get_handler(tag_type, true);
+      }
+
+      private static PubSubTagTypeHandlerConfig _wildcard_handler = new() {
+         TagType = "*",
+         Assembly = "DataStreamCore",
+         Class = "bts.PubsubServerGenericDataHandler",
      };
-      private static TagHandlerEntry _wildcard_handler = new TagHandlerEntry("*", "DataStreamCore", "bts.PubsubServerGenericDataHandler");
-      private static Dictionary<string, IPubSubServerTagHandler> _handler_map = new Dictionary<string, IPubSubServerTagHandler>();
+      private Dictionary<string, IPubSubServerTagHandler> _handler_map = new Dictionary<string, IPubSubServerTagHandler>();

-      public static string assembly_location => Assembly.GetExecutingAssembly().Location;
+      public string assembly_location => Assembly.GetExecutingAssembly().Location;

-      public static BCG.IPubSubServerTagHandler get_handler(string tag_type, bool on_pubsub_server) {
-         TagHandlerEntry handler_entry = _find_config_entry(tag_type);
-         Log.assert(handler_entry != null);
-         return _find_or_build_handler(handler_entry);
+      public BCG.IPubSubServerTagHandler get_handler(string tag_type, bool on_pubsub_server) {
+         var handler_entry = _find_config_entry(tag_type);
+         return _find_or_build_handler(handler_entry.Config);
      }

-      private static TagHandlerEntry _find_config_entry(string tag_type) {
-         TagHandlerEntry rv = null;
-         if (_config_map.TryGetValue(tag_type, out rv))
-            return rv;
-         else
-            return _wildcard_handler;
-      }
-
-      private static BCG.IPubSubServerTagHandler _find_or_build_handler(TagHandlerEntry entry) {
-         IPubSubServerTagHandler rv = null;
-         if (_handler_map.TryGetValue(entry.tag_type, out rv)) {
-            return rv;
+      private TagTypeHandlerData _find_config_entry(string tag_type) {
+         // Look for an exact match.
+         foreach (var handler in handlers) {
+            if (handler.TagType == tag_type) {
+               return handler;
+            }
         }
-         {
+         // If an exact match wasn't found, look for a regex match
+         foreach (var handler in handlers) {
+            if (handler.TagTypeRegex.IsMatch(tag_type)) {
+               return handler;
+            }
+         }
+         Log.assert(false);
+         throw new InvalidOperationException($"Failed to handler config for tag type '{tag_type}'");
+      }
+
+      private BCG.IPubSubServerTagHandler _find_or_build_handler(PubSubTagTypeHandlerConfig entry) {
+         ArgumentNullException.ThrowIfNull(entry.TagType);
+         lock (_handler_map) {
+            if (_handler_map.TryGetValue(entry.TagType, out var rv)) {
+               return rv;
+            }
            rv = _build_handler(entry);
-            _handler_map.Add(entry.tag_type, rv);
+            _handler_map.Add(entry.TagType, rv);
+            return rv;
         }
-         return rv;
      }

-      private static BCG.IPubSubServerTagHandler _build_handler(TagHandlerEntry entry) {
+      private BCG.IPubSubServerTagHandler _build_handler(PubSubTagTypeHandlerConfig entry) {
+         Log.notnull(entry.Assembly);
+         Log.notnull(entry.Class);
         var ass_loc = assembly_location;
         bts.Log.Gen.info($"PubsubTagHandlerFactory.load_handler entry: {entry} assembly_location: {ass_loc}");
-         var assem = Assembly.LoadFrom(Path.ChangeExtension(entry.assembly_name, "dll"))
+         var assem = Assembly.LoadFrom(Path.ChangeExtension(entry.Assembly, "dll"))
            ?? throw new ArgumentException("Failed to instantiate TagHandlerEntry");
         bts.Log.Gen.info($"PubsubTagHandlerFactory.load_handler successfully loaded assembly: {assem}");
-         Type t = assem.GetType(entry.handler_class_name, true, true);
-         object obj = Activator.CreateInstance(t);
-         BCG.IPubSubServerTagHandler tag_handler = obj as BCG.IPubSubServerTagHandler;
-         bts.Log.Gen.info($"PubsubTagHandlerFactory.load_handler successfully created obj: {obj} of type: {t} from assem: {assem}");
+         var t = assem.GetType(entry.Class, true, true);
+         Log.notnull(t);
+         var obj = ActivatorUtilities.GetServiceOrCreateInstance(serviceProvider, t);
+         var tag_handler = Log.notnull(obj as IPubSubServerTagHandler);
+         Log.Gen.info($"PubsubTagHandlerFactory created handler. {JsonSerializer.Serialize(entry)}");
         return tag_handler;
      }
+
+      readonly struct TagTypeHandlerData {
+         public TagTypeHandlerData(PubSubTagTypeHandlerConfig config) {
+            Config = config;
+            Log.notnull(config.TagType);
+            TagType = config.TagType;
+            TagTypeRegex = new(TagType);
+         }
+
+         public PubSubTagTypeHandlerConfig Config { get; }
+         public string TagType { get; }
+         public Regex TagTypeRegex { get; }
+      }
   }
 }
@@ -1,104 +1,67 @@
+using bts;
+
+using mbts.utils;
+
 using System;
-using BCG;
-//using BCG.Utilities;
-using System.Text.RegularExpressions;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Text.Json.Serialization;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-using System.Runtime.InteropServices;
+using System.Text.RegularExpressions;

-namespace BCG {
-   [JsonConverter(typeof(PubSubRouteJsonConverter))]
-   public class PubSubRoute : IComparable<PubSubRoute> {
-      public string server_address;
-      public string tag_reg_exp;
-      //route names are used in intermediary code (i.e. while substations are being deployed)
-      //so that MC Hammer can advertise them. 
-      public string name;
-      private Regex _regex = null;
+namespace BCG;

-      public PubSubRoute() { }
+public class PubSubRoute : IEquatable<PubSubRoute>, IComparable<PubSubRoute>, ICloneable<PubSubRoute> {
+   [JsonPropertyName("server_address")]
+   public string? ServerAddress { get; set; }

-      public PubSubRoute(string the_server_address, string the_tag_reg_exp) {
-         server_address = the_server_address;
-         tag_reg_exp = the_tag_reg_exp;
-      }
+   [JsonPropertyName("tag")]
+   public string? TagRegExp { get; set; }
+   //route names are used in intermediary code (i.e. while substations are being deployed)
+   //so that MC Hammer can advertise them. 
+   [JsonPropertyName("name")]
+   public string? Name { get; set; }

-      public string[] getAllPubSubs() {
-         return server_address.Split(';');
-      }
+   static readonly IComparer<PubSubRoute> comparer = new ComparerBuilder<PubSubRoute>() {
+      p => p.ServerAddress,
+      p => p.TagRegExp,
+   };

-      public Regex get_regex() {
-         return _regex ?? new Regex(tag_reg_exp);
-      }
+   Regex? myRegex;

-      #region IComparable<PubSubRoute> Members
+   public PubSubRoute() { }

-      public int CompareTo(PubSubRoute other) {
-         return server_address == other.server_address ? tag_reg_exp.CompareTo(other.tag_reg_exp)
-                                                      : server_address.CompareTo(other.server_address);
-      }
-
-      #endregion
-
-      public override bool Equals(object obj) {
-         if (!(obj is PubSubRoute) || obj == null)
-            return false;
-         else {
-            PubSubRoute other = obj as PubSubRoute;
-            return server_address.Equals(other.server_address) &&
-                     tag_reg_exp.Equals(other.tag_reg_exp);
-         }
-      }
-
-      public override int GetHashCode() {
-         return base.GetHashCode();
-      }
-
-      public static PubSubRoute get_default_route() {
-         return new PubSubRoute("bluto://127.0.0.1:12345", ".*");
-      }
-
-      public override string ToString() {
-         return $"tag_reg_exp: {tag_reg_exp} server_address: {server_address}";
-      }
+   public PubSubRoute(string theServerAddress, string theTagRegExp) {
+      ServerAddress = theServerAddress;
+      TagRegExp = theTagRegExp;
   }

-   public static class PubSubRouteExtensions {
-      public static bool IsMatch(this PubSubRoute route, string tag) =>
-         route.get_regex().IsMatch(tag);
+   [MemberNotNull(nameof(TagRegExp), nameof(myRegex))]
+   public bool IsMatch(string tag) {
+      if (TagRegExp is null) {
+         throw new InvalidOperationException();
+      }
+      myRegex ??= new Regex(TagRegExp);
+      return myRegex.IsMatch(tag);
   }

-   public class PubSubRouteJsonConverter : JsonConverter<PubSubRoute> {
-      public override PubSubRoute? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-         var json_obj = JsonSerializer.Deserialize<JsonObject>(ref reader);
-         if (json_obj is null) {
-            return null;
-         }
-         var rv = new PubSubRoute();
-         ReadString("name", ref rv.name);
-         ReadString("server_address", ref rv.server_address);
-         ReadString("tag", ref rv.tag_reg_exp);
-         return rv;
+   public PubSubRoute Clone() =>
+      new() {
+         ServerAddress = ServerAddress,
+         TagRegExp = TagRegExp,
+         Name = Name,
+      };

-         void ReadString(string key, ref string valueOut) {
-            if (json_obj.TryGetPropertyValue(key, out var valueJson)) {
-               var valueStr = valueJson?.GetValue<string>();
-               // Only (over)write valueOut if non-null
-               if (valueStr is not null) {
-                  valueOut = valueStr;
-               }
-            }
-         }
-      }
+   public int CompareTo(PubSubRoute? other) =>
+      other is null ? -1 : comparer.Compare(this, other);

-      public override void Write(Utf8JsonWriter writer, PubSubRoute value, JsonSerializerOptions options) {
-         var jsonObj = new JsonObject {
-            ["name"] = value.name,
-            ["server_address"] = value.server_address,
-            ["tag"] = value.tag_reg_exp,
-         };
-         JsonSerializer.Serialize(writer, jsonObj);
-      }
-   }
+   public bool Equals(PubSubRoute? other) =>
+      other is not null
+      && ServerAddress == other.ServerAddress
+      && TagRegExp == other.TagRegExp;
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as PubSubRoute);
+
+   public override int GetHashCode() =>
+      HashCode.Combine(ServerAddress, TagRegExp);
 }
Author	SHA1	Message	Date
John Colvin	ba664b95b8	fix use dep injection in bts.datastreamwebapp.server	2024-07-11 11:06:29 -04:00
John Colvin	d6f370f862	fix Add spark service modules designed for DependencyInjection	2024-07-11 11:05:23 -04:00
John Colvin	079826f4de	Fix mismatched types in PubsubHeartbeatStream schema	2024-07-11 10:59:41 -04:00
John Colvin	2eee3cdaab	Change DataStreamPublisher json fields to match cs class property names verbatim	2024-07-11 10:43:24 -04:00
John Colvin	5e3215ee52	fix: Always call 'handle_internal_pubs' in PubSubMessageReactor main loop	2024-07-11 10:38:30 -04:00
John Colvin	7739cc2f3d	Use Dependency Injection in Bts.DataStreamWebApp.Server	2024-07-11 10:37:29 -04:00
John Colvin	8347a7c126	refactor: Change Bts.DataStreamWebApp.Server framework from net8 -> net7 Although it would build on net8, all other cs code targets net7. Furthermore, VS Code intellisense seems less reliable/crashes frequently when the sln file targets both frameworks).	2024-07-11 10:37:28 -04:00
John Colvin	fbfc960df6	improvement: Make DataStreamSubscription an IHostedService To properly manage its life cycle	2024-07-11 10:37:28 -04:00
John Colvin	451f721030	improvement: Accept CancellationToken, catch exceptions when running ds websocket server	2024-07-11 10:37:28 -04:00
John Colvin	bb992c0410	Fix mismatched types in PubSubServerTagStatsStream schema	2024-07-11 10:37:28 -04:00
John Colvin	db18aebf92	Add DataStreamPublisher service modules designed for Dependency Injection	2024-07-11 10:37:28 -04:00
John Colvin	8127c9c9fe	refactor: Decouple ViewPublisher from IPubTableBroker impl	2024-07-11 10:37:28 -04:00
John Colvin	1825b339b1	Change PubSubServer json fields to match cs class property names verbatim	2024-07-11 10:37:28 -04:00
John Colvin	03ab3fbe90	Get rid of JsonPropertyName attributes in PubSubConfig The Dependency Injection framework (based around dotnet ApplicationBuilder) does not support this attribute, and maps property names verbatim to json fields.	2024-07-11 10:37:28 -04:00
John Colvin	b61bf16a42	Add PubSub service modules designed for Dependency Injection	2024-07-11 10:37:28 -04:00
John Colvin	d969489bf4	refactor: Decouple RemoteQueryManager from IManagedSubscriber, IPubTableBroker impl	2024-07-11 10:37:28 -04:00
John Colvin	9cd4974f49	refactor: Decouple ManagedSubscriber from ISubscriberCore impl	2024-07-11 10:37:28 -04:00
John Colvin	793634c8cb	refactor: Only support 1 tag type in IPubSubServerTagHandler	2024-07-11 10:37:28 -04:00
John Colvin	eae20b6c94	refactor: Decouple PubTableBroker from IPublisherCore impl Reference interface rather than the PublisherCore impl.	2024-07-11 10:37:28 -04:00
John Colvin	2848bb6a5d	improvement: Accept CancellationToken in JsonRpcRequestObservable	2024-07-11 10:37:16 -04:00
John Colvin	687324998b	Add spark service modules designed for Dependency Injection	2024-07-11 10:36:04 -04:00
John Colvin	97cebd62d4	fix: Dispose market data sub in various md streams	2024-07-11 10:36:04 -04:00
John Colvin	492ca0c28f	cleanup: Remove unused pubsub tag handler code * Removed 'IPubSubServerTagHandler.register' - All implementations of this function were empty * Removed 'IPubSubServerTagHandler.supportedTagTypes' - This property wasn't referenced anywhere * Remove IWorker inheritance from IPubSubServerTagHandler - No reason these interfaces have to be tied together	2024-07-10 13:35:18 -04:00
John Colvin	baa5019039	refactor: Rename CloudStreamManager.cs -> RemoteQueryManager.cs This matches the file name	2024-07-10 01:18:21 -04:00
John Colvin	ac2d31e1fc	fix build: Update vcpkg On some vs versions, vcpkg cmake scripts would fail w/ an error saying "Could not locate a complete Visual Studio instance"	2024-07-09 15:16:36 -04:00
John Colvin	9821134bc3	Change pub sub schema prefix 'bluto://' -> 'bts-cloud://' In config json files (not actual code change)	2024-07-09 15:14:30 -04:00
John Colvin	9cf892322d	fix build: Fix prop names after refactoring PubSubRoute * Change 'server_address' -> 'ServerAddress' * Change 'tag_reg_exp' -> 'TagRegExp'	2024-07-09 15:14:30 -04:00
John Colvin	77bf91be1e	build: Add scripts to publish data stream apps	2024-07-09 15:14:30 -04:00
John Colvin	63b064ac79	Fix typo in log msg	2024-07-09 15:14:30 -04:00
John Colvin	b617775c29	Fix net7 -> net8 typo in publish_data_stream_web_app.py The DataStream Web App is on net8	2024-07-09 15:14:30 -04:00
John Colvin	4f7d77420c	Fix build scripts so they propagate the exit code back to the shell I noticed this was broken because launch.json configurations would still start the debugger even when the preLaunchTask (build) failed!	2024-07-09 15:14:30 -04:00
John Colvin	3ba6a91ca7	vscode: Add DataStreamPublisher debug cfg to skip build	2024-07-09 15:14:30 -04:00
John Colvin	0abde9eb7c	build: Use separate cmd to build DataStreamSuite/QueryAnalyzer.sln The DataStreamSuite.sln builds the new webapp now, which will (eventually) replace DSQA. We'll keep it around for now, but with a clean separation from the main build.	2024-07-09 15:14:30 -04:00
John Colvin	1531a57bc5	feat: Add DataStream WebApp Included in DataStreamSuite.sln, but not DataStreamQueryAnalyzer.sln	2024-07-09 15:14:29 -04:00
John Colvin	a62ea64c5e	build: Copy libspark to all dotnet dirs in single cmake target This is cleaner than the previous impl, which used separate build targets to copy libspark to "net7.0", and "net7.0-windows" (dirs). Note: I decided to clean this up now because I'm about to upgrade the new DataStream WebApp to net8.0, and did not want to add yet another build target to copy libspark to "net8.0".	2024-07-09 15:14:29 -04:00
John Colvin	450773f588	fix: Parse ServerAddress as normal string The "JsonValueOrArrayConverter<T>" doesn't work the way I thought it would. When it is associated with a property via the JsonConverter attribute, dotnet json parsing code throws an exception if "T" isn't assignable to the property type.	2024-07-09 15:14:29 -04:00
John Colvin	f09d6badaf	fix build (api server): Change namespace to bts.websocket_core	2024-07-09 15:14:29 -04:00
John Colvin	6b054e0718	Remove docker submodules again	2024-07-09 15:14:29 -04:00
John Colvin	5e2789d16e	Change externals url from ssh to https style So we don't need ssh keys to clone it (the builtin credential manager makes https authentication safe/easy)	2024-07-09 15:14:29 -04:00
John Colvin	9ebdb2e460	cleanup: Fix benign typos in build scripts Change BUILD_SPARK_PY var name to BUILD_PY (because these scripts to actually build 'spark')	2024-07-09 15:14:29 -04:00
John Colvin	d7bf82722b	fix build (warn): Only use Registry on windows	2024-07-09 15:14:29 -04:00
John Colvin	e1526e25a7	cleanup: Fix numerous build warnings in DataStreamSuite Nearly all warnings were due to nullability issues, since the concept of nullable reference types did not exist when the DataStream code was originally written. * PubSubClientMessage - Normalize indentation/whitespace - Fix some typos - Address nullability issues to fix build warnings - Modernize code * Compression related files - ZipEntry - ZipFile - ZipInputStream - TarHeader * RemoteMessage - make 'message' nullable to resolve warnings elsewhere (in calling code) * PubSubSingleTagStateEvent.cs * PubSubMessageReactor * PubSubState * SystemHeartbeatPublisher * SystemStatsPublisher * SingleTagData * ManagedTCPMessageHub * PubsubDeserializationController * PublisherCore * InflaterHuffmanTree * AsyncTCPConnection_APM * ThreadSafeQueue * ManagedSubscriber * SubscriberCore * PubSubServerManagedTCPConnection * AsyncTCPConnection_APM * SuperMapStream * MapStream * TradeBarsStream	2024-07-09 15:14:26 -04:00
John Colvin	089ba9b369	cleanup: Normalize indentation in TradeBarsStream	2024-07-09 15:09:02 -04:00
John Colvin	567976043f	cleanup: Disable nullability warnings * BZip2InputStream * TarInputStream * Inflater * InflaterDynHeader * BZip2OutputStream * DeflaterHuffman * ParallelJoinTest	2024-07-09 15:07:24 -04:00
John Colvin	2fd3382939	cleanup: Modernize PubSubRoute code, json parsing	2024-07-09 15:06:34 -04:00
John Colvin	a96f1c8f17	cleanup: Remove unused DateTime grammar from exp parser	2024-07-09 15:03:30 -04:00
John Colvin	6f623f9a37	Add JsonRpcRequestObservable, AsyncEnumerable	2024-07-09 15:03:30 -04:00
John Colvin	827189cdb4	Add json facilities for DataStream websocket api	2024-07-09 15:03:30 -04:00
John Colvin	123f12ab88	refactor: Add handy WebsocketExtensions * IsSocketStateOpen * IsClosedProperly	2024-07-09 14:59:06 -04:00
John Colvin	f51ba8fb56	feat: Add JsonRpc types for websocket paradigm	2024-07-09 14:59:06 -04:00
John Colvin	2506fb4f2b	cleanup: Remove empty file (clearly added by accident)	2024-07-09 14:59:06 -04:00
John Colvin	f92964114e	fix: Fix improper 'OfType' linq cast Previously, animations were expressed through polymorphism (derived from the base class CellAnimation), but the new impl uses composition. This 'OfType' call was leftover from the polymorphism implementation, an attempt to linq cast elements of base class (CellAnimation) to BackgroundAnimation.	2024-07-09 14:59:06 -04:00
John Colvin	baa0f1430b	editorconfig: Use standard c# capitalization rules * Use 'PascalCase' for public members (instead of 'snake_case') * Use 'camelCase' for private members (instead of '_snake_case')	2024-07-09 14:59:06 -04:00
John Colvin	7104bd787a	refactor: Add WebsocketClient class Basically an ISubject (I/O) wrapper for IWebsocketConnectionGateway	2024-07-09 14:59:06 -04:00
John Colvin	5db92d1f7d	refactor: Change WebsocketServerConfig.port to uint32 It should always be a number between 1-65535	2024-07-09 14:59:06 -04:00
John Colvin	eb0fcf0b84	refactor: Rename WebsocketServerConfig proto pkg 'bts.websocket_core' This matches the namespace used in spark_websocket csproj. Eventually, I'd like to use conventional capitalization (TitleCase namespaces/types, etc.), but let's shelf that until we have enough time/energy to do a more comprehensive refactoring.	2024-07-09 14:59:06 -04:00
John Colvin	a52b187313	refactor: Remove nullability constraint in GetValueOrDefault	2024-07-09 14:59:06 -04:00
John Colvin	669b059eb2	refactor: Use events to avoid circular deps for ws sync/gateway	2024-07-09 14:59:06 -04:00
John Colvin	36ab8492d3	refactor: Let caller specify nullability of T in json parse fn	2024-07-09 14:48:59 -04:00
John Colvin	54f37b2401	Merge remote-tracking branch 'origin/development' into development.ps_linux	2024-07-09 14:46:18 -04:00
Taha	5cf7eacd82	Fixing the extra .0 in the default pubsub server routeplusfixing acorner cae issue of gettign a fast streaming data cakback after an unsubscribe	2024-06-24 00:28:07 -04:00
John Colvin	087d8ab8c9	Set text=auto in gitattributes Otherwise git can cause files to show as "permanently modified" when you apply changes from the index (e.g., by running 'git rebase', or 'git stash pop')	2024-06-11 17:39:12 +00:00
John Colvin	b69b7b3350	vscode: Use '\' dir separator in show_last_run.cmd This script only runs on Windows	2024-06-08 15:08:25 -05:00
John Colvin	5620f33524	vscode: Add task to view logs for last PubSubServer run	2024-06-08 14:42:13 +00:00
John Colvin	f3e33fa390	vscode: Add task to run PubSubServer without building	2024-06-07 16:41:42 +00:00
John Colvin	3b631e0d5f	Fix a couple more typos in logs	2024-06-07 16:35:06 +00:00
				`@@ -0,0 +1 @@`
				<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>