fix use dep injection in bts.datastreamwebapp.server

fix Add spark service modules designed for DependencyInjection
Fix mismatched types in PubsubHeartbeatStream schema
2024-07-11 11:06:29 -04:00 · 2024-07-11 11:05:23 -04:00 · 2024-07-11 10:59:41 -04:00 · 2024-07-11 10:43:24 -04:00 · 2024-07-11 10:38:30 -04:00 · 2024-07-11 10:37:29 -04:00
444 changed files with 17599 additions and 11849 deletions
@@ -66,6 +66,7 @@ RUN \

 # Dotnet
 RUN apt-get update && apt-get install -y dotnet7
+RUN apt-get update && apt-get install -y dotnet8

 # Node
 ARG node_major=18
@@ -87,9 +88,6 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev

-# Databento (deps)
-RUN apt-get update && apt-get -y install libzstd-dev
-
 # Set up default executables (update-alternatives)
 RUN \
   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
@@ -19,6 +19,14 @@
    "--shm-size",
    "2gb",
    "-p",
+    "5000:5000",
+    "-p",
+    "5001:5001",
+    "-p",
+    "5173:5173",
+    "-p",
+    "5174:5174",
+    "-p",
    "8000:8000",
    "-p",
    "8001:8001",
@@ -27,7 +35,7 @@
    "-p",
    "8003:8003",
    "-p",
-    "12345:12345"
+    "12345:12345",
  ],
  "mounts": [
    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
@@ -47,4 +55,4 @@
      ]
    }
  }
-}
+}
@@ -207,17 +207,23 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private

+# local symbols
+dotnet_naming_symbols.local_symbols.applicable_kinds = local
+
+# parameter symbols
+dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
+
 # Naming styles

-dotnet_naming_style.snake_case.required_prefix =
-dotnet_naming_style.snake_case.required_suffix =
-dotnet_naming_style.snake_case.word_separator = _
-dotnet_naming_style.snake_case.capitalization = all_lower
+dotnet_naming_style.pascal_case.required_prefix =
+dotnet_naming_style.pascal_case.required_suffix =
+dotnet_naming_style.pascal_case.word_separator =
+dotnet_naming_style.pascal_case.capitalization = pascal_case

-dotnet_naming_style.private_snake_case.required_prefix = _
-dotnet_naming_style.private_snake_case.required_suffix =
-dotnet_naming_style.private_snake_case.word_separator = _
-dotnet_naming_style.private_snake_case.capitalization = all_lower
+dotnet_naming_style.camel_case.required_prefix =
+dotnet_naming_style.camel_case.required_suffix =
+dotnet_naming_style.camel_case.word_separator =
+dotnet_naming_style.camel_case.capitalization = camel_case

 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -229,6 +235,10 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case

+dotnet_naming_style.local.capitalization = camel_case
+
+dotnet_naming_style.parameter.capitalization = camel_case
+
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -238,14 +248,18 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion

-dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
-dotnet_naming_rule.public_members_snake_case.style = snake_case
-dotnet_naming_rule.public_members_snake_case.severity = suggestion
+dotnet_naming_rule.public_members.symbols = public_symbols
+dotnet_naming_rule.public_members.style = pascal_case
+dotnet_naming_rule.public_members.severity = suggestion

-dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
-dotnet_naming_rule.private_members_snake_case.style = private_snake_case
-dotnet_naming_rule.private_members_snake_case.severity = suggestion
+dotnet_naming_rule.private_members.symbols = private_symbols
+dotnet_naming_rule.private_members.style = camel_case
+dotnet_naming_rule.private_members.severity = suggestion

-dotnet_naming_rule.parameters.symbols = parameters
-dotnet_naming_rule.parameters.style = snake_case
+dotnet_naming_rule.variables_are_camel_case.severity = suggestion
+dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
+dotnet_naming_rule.variables_are_camel_case.style = local
+
+dotnet_naming_rule.parameters.symbols = parameter_symbols
+dotnet_naming_rule.parameters.style = parameter
 dotnet_naming_rule.parameters.severity = suggestion
@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = git@bitbucket.org:bluetradesys/externals.git
+	url = https://bitbucket.org/bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git
@@ -6,6 +6,35 @@
      "type": "coreclr",
      "request": "attach"
    },
+    {
+      "name": "Bts.DataStreamWebApp.Server",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream WebApp (Debug)",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    {
+      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
    {
      "name": "DataStreamPublisher",
      "type": "coreclr",
@@ -22,27 +51,34 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
      ],
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
    {
-      "name": "(gdb) Launch",
-      "type": "cppdbg",
+      "name": "DataStreamPublisher (Skip Build)",
+      "type": "coreclr",
      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": ["run"],
+      "program": "",
+      "linux": {
+        "program": "DataStreamPublisher",
+      },
+      "windows": {
+        "program": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
+      "console": "integratedTerminal"
    },
    {
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
      "request": "launch",
-      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
      "program": "",
      "windows": {
        "program": "DataStreamQueryAnalyzer.exe",
@@ -78,6 +114,25 @@
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
+    {
+      "name": "PubSubServer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "PubSubServer",
+      },
+      "windows": {
+        "program": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
    {
      "name": "Spark",
      "type": "coreclr",
@@ -155,4 +210,4 @@
      "default": "http://localhost:8000/config",
    },
  ]
-}
+}
@@ -121,6 +121,97 @@
        "color": "terminal.ansiCyan"
      },
    },
+    {
+      "label": "Build DataStreamQueryAnalyzer",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
    {
      "label": "Build Spark",
      "group": "build",
@@ -303,6 +394,22 @@
        "color": "terminal.ansiCyan"
      },
    },
+    {
+      "label": "Publish DataStreamWebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/publish_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/publish_data_stream_web_app.cmd",
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
    {
      "label": "Edit DataStream Config",
      "type": "process",
@@ -353,7 +460,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStream",
+      "dependsOn": "Build DataStreamQueryAnalyzer",
      "type": "process",
      "command": "",
      "windows": {
@@ -373,7 +480,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
      "type": "process",
      "command": "",
      "windows": {
@@ -393,7 +500,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStream (Release)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
      "type": "process",
      "command": "",
      "windows": {
@@ -132,8 +132,6 @@ else()
      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
-      
      COMMAND
        ln ARGS -sf
        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
@@ -31,8 +31,7 @@
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -222,6 +221,7 @@
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -258,7 +258,7 @@
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -8,7 +8,6 @@
   public int Integer;
   public double Real;
   public string Identifier;
-   public DateTime DateTime; 
   public ImmutableList<IArithmeticExpression> Arguments;
   public ImmutableList<Assignment> Assignments;
   public IArithmeticExpression Argument;
@@ -1,4 +1,6 @@
 using System;
+using System.Runtime.Versioning;
+
 using Microsoft.Win32;

 namespace BCG
@@ -10,6 +12,7 @@ namespace BCG
         rk.SetValue (name, value.ToString ());
      }

+      [SupportedOSPlatform("windows")]
      public static double getValue (RegistryKey rk, string name)
      {
         object v = rk.GetValue (name);
@@ -0,0 +1,20 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.3.1' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
+  <PropertyGroup>
+    <StartupCommand>npm run dev</StartupCommand>
+    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
+    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
+    <!-- Allows the build (or compile) script located on package.json to run on Build -->
+    <ShouldRunBuildScript>true</ShouldRunBuildScript>
+    <!-- Folder where production build objects will be placed -->
+    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
+  </PropertyGroup>
+</Project>
@@ -0,0 +1,8 @@
+# React + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/ico" href="/logo.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>BTS Cloud Streams</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>
@@ -0,0 +1,38 @@
+{
+  "name": "bts.datastreamwebapp.client",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@angular/material": "^18.0.4",
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@fontsource/noto-sans": "^5.0.22",
+    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
+    "@mui/icons-material": "^5.15.20",
+    "@mui/material": "^5.15.20",
+    "@mui/x-data-grid": "^7.7.1",
+    "bulma": "^1.0.1",
+    "react": "^18.3.1",
+    "react-data-table-component": "^7.6.2",
+    "react-dom": "^18.3.1",
+    "react-use-websocket": "^4.8.1",
+    "rpc-websockets": "^7.11.2"
+  },
+  "devDependencies": {
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^8.57.0",
+    "eslint-plugin-react": "^7.34.2",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-react-refresh": "^0.4.7",
+    "vite": "^5.2.13"
+  }
+}
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
@@ -0,0 +1,28 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+h1#main-header {
+  padding: 4px 8px 4px 8px;
+  text-align: center;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}
@@ -0,0 +1,24 @@
+import { createTheme, ThemeProvider, CssBaseline } from '@mui/material';
+import useMediaQuery from '@mui/material/useMediaQuery';
+import QueryControl from './QueryControl';
+import getTheme from './Theme';
+import '@fontsource/noto-sans';
+import './App.css';
+
+const App = () => {
+  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
+  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
+
+  return (
+    <ThemeProvider theme={theme}>
+      <CssBaseline />
+      <h1 id='main-header' style={{
+        color: theme.palette.text.primary,
+        fontFamily: 'Noto Sans',
+      }}>BTS Cloud Streams</h1>
+      <QueryControl websocketUrl="ws://localhost:5000/ws" theme={theme} />
+    </ThemeProvider>
+  );
+};
+
+export default App;
@@ -0,0 +1,32 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+th {
+  font-size: large;  
+  padding: 15px;
+}
+
+td {
+  padding: 15px;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}
@@ -0,0 +1,52 @@
+import PropTypes from 'prop-types';
+import { DataGrid } from '@mui/x-data-grid';
+
+function getGridColDef(col) {
+  return {
+    field: col.name,
+    headerName: col.name,
+  };
+}
+
+export default function DataTable({ columns, rows }) {
+  return (
+    <DataGrid
+      columns={columns.map(getGridColDef)}
+      rows={rows}
+      sx={{
+        borderColor: 'primary.main',
+        backgroundColor: 'primary.dark',
+        color: 'text.primary',
+        '& .MuiDataGrid-columnHeader': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-filler': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-cell': {
+          color: 'text.primary',
+        },
+        '& .MuiDataGrid-cell:hover': {
+          color: 'primary.main',
+        },
+        '& .MuiDataGrid-row': {
+          '--DataGrid-rowBorderColor': 'secondary.dark'
+        }
+      }}
+    />
+  );
+}
+
+DataTable.propTypes = {
+  columns: PropTypes.arrayOf(
+    PropTypes.shape({
+      name: PropTypes.string.isRequired,
+      type: PropTypes.string.isRequired,
+    })
+  ).isRequired,
+  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
+};
@@ -0,0 +1,100 @@
+import PropTypes from 'prop-types';
+import DataTable from './DataTable';
+import QueryInputForm from './QueryInputForm';
+import Container from '@mui/material/Container';
+import React from 'react';
+
+var rpcReqIdCounter = 1;
+
+function subscribeReq(query) {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "subscribe",
+    params: {
+      subscription_id: "1",
+      query: query
+    }
+  };
+}
+
+function unsubscribeReq() {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "unsubscribe",
+    params: {
+      subscription_id: "1",
+    }
+  };
+}
+
+class QueryControl extends React.Component {
+  constructor({ websocketUrl, theme }) {
+    super(websocketUrl);
+    console.log('Connecting to WebSocket:', websocketUrl);
+    this.state = {
+      rows: null,
+      ws: new WebSocket(websocketUrl),
+      columns: null,
+      isRunning: false,
+      theme: theme,
+    };
+    this.state.ws.onopen = () => {
+      console.info('WebSocket open');
+    };
+    this.state.ws.onclose = () => {
+      console.info('WebSocket closed');
+    };
+    this.state.ws.onmessage = e => {
+      console.info(`Received message: ${e.data}`);
+      const msg = JSON.parse(e.data);
+      if (msg.id != null) {
+        let newState = { ...this.state };
+        newState.isRunning = msg.result.status == 'Subscribed';
+        this.setState(newState);
+      } else {
+        let newState = { ...this.state };
+        newState.rows = msg.params.all;
+        console.info(`Received rows: ${newState.rows}`);
+        newState.columns = msg.params.columns;
+        this.setState(newState);
+      }
+    };
+
+    this.start = (query) => {
+      this.state.ws.send(JSON.stringify(subscribeReq(query)));
+    };
+
+    this.stop = () => {
+      this.state.ws.send(JSON.stringify(unsubscribeReq()));
+    };
+  }
+
+  componentWillUnmount() {
+    if (this.state.ws) {
+      console.warn('Closing websocket because component is unmounting');
+      this.state.ws.close();
+    }
+  }
+
+  render() {
+    console.info('Rendering');
+    return (
+      <Container>
+        <QueryInputForm isRunning={this.state.isRunning} onStart={this.start} onStop={this.stop} />
+        {
+          this.state.rows == null ? <p style={{ color: this.state.theme.palette.warning.main }}>No data received</p> :
+            <DataTable columns={this.state.columns} rows={this.state.rows} />
+        }
+      </Container>
+    );
+  }
+}
+
+QueryControl.propTypes = {
+  websocketUrl: PropTypes.string.isRequired,
+  theme: PropTypes.object.isRequired,
+};
+
+export default QueryControl;
@@ -0,0 +1,93 @@
+import { useRef } from 'react';
+import Box from '@mui/material/Box';
+import Button from '@mui/material/Button';
+import Grid from '@mui/material/Grid';
+import Paper from '@mui/material/Paper';
+import PlayArrowIcon from '@mui/icons-material/PlayArrow';
+import PropTypes from 'prop-types';
+import StopIcon from '@mui/icons-material/Stop';
+import TextField from '@mui/material/TextField';
+import Tooltip from '@mui/material/Tooltip';
+
+const QueryInputForm = ({ isRunning, onStart, onStop }) => {
+  let textInput = useRef(null);
+
+  const onClearClick = () => {
+    textInput.current.value = '';
+  };
+
+  const onRunClick = () => {
+    onStart(textInput.current.value);
+  };
+
+  const onStopClick = () => {
+    onStop();
+  };
+
+  const onKeyDown = (e) => {
+    // Detect keyboard shortcuts.
+    //   Ctrl+Enter : Run
+    if (e.ctrlKey && e.key === 'Enter') {
+      onRunClick();
+    }
+  };
+
+  return (
+    <Box sx={{ p: 2 }}>
+      <Paper elevation={3} sx={{ color: 'primary.main' }}>
+        <Box sx={{ p: 2 }}>
+          <TextField
+            fullWidth
+            multiline
+            rows={10}
+            variant="outlined"
+            onKeyDown={onKeyDown}
+            inputRef={textInput}
+            placeholder="Enter Query..."
+          />
+        </Box>
+        <Box sx={{ p: 2, borderTop: 1 }}>
+          <Grid container spacing={2} justifyContent="flex-start">
+            <Grid item xs>
+              <Tooltip title="Clear Query Text">
+                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
+                  Clear
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Run (Ctrl+Enter)">
+                <Button onClick={onRunClick} sx={{
+                  color: `query_start_button.${isRunning ? 'active' : 'inactive'}.color`,
+                  bgcolor: `query_start_button.${isRunning ? 'active' : 'inactive'}.background_color`,
+                  padding: '11px'
+                }}>
+                  <PlayArrowIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Stop">
+                <Button onClick={onStopClick} sx={{
+                  color: `query_stop_button.${isRunning ? 'inactive' : 'active'}.color`,
+                  bgcolor: `query_stop_button.${isRunning ? 'inactive' : 'active'}.background_color`,
+                  padding: '11px'
+                }}>
+                  <StopIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+          </Grid>
+        </Box>
+      </Paper>
+    </Box>
+  );
+};
+
+QueryInputForm.propTypes = {
+  isRunning: PropTypes.bool.isRequired,
+  onStart: PropTypes.func.isRequired,
+  onStop: PropTypes.func.isRequired,
+};
+
+export default QueryInputForm;
@@ -0,0 +1,88 @@
+import { createTheme } from '@mui/material/styles';
+import PropTypes from 'prop-types';
+import '@fontsource/noto-sans';
+import './App.css';
+
+const getLightPalette = () => {
+  return {
+    mode: 'light',
+    primary: {
+      light: '#6573c3',
+      main: '#3f51b5',
+      dark: '#2c387e',
+      contrastText: '#ffffff',
+    },
+    secondary: {
+      light: '#f73378',
+      main: '#f50057',
+      dark: '#ab003c',
+      contrastText: '#ffffff',
+    },
+    text: {
+      primary: '#000000c4',
+      secondary: '#00000086',
+      disabled: '#0000005b',
+      hint: '#0000005b',
+    }
+  };
+};
+
+const getDarkPalette = () => {
+  return {
+    mode: 'dark',
+    primary: {
+      light: '#bcc5ff',
+      main: '#33baf9',
+      dark: '#202334',
+      contrastText: '#78a2ac',
+    },
+    secondary: {
+      light: '#9fd0b8',
+      main: '#5eb289',
+      dark: '#ffffff12',
+      contrastText: '#c8d4f7',
+    },
+    text: {
+      primary: '#eaf8f8',
+      dark: '#eaf8f8a0'
+    },
+    query_start_button: {
+      active: {
+        color: '#33baf9',
+        background_color: '#ffffff40',
+      },
+      inactive: {
+        color: '#ffffff40',
+        background_color: '#ffffff00',
+      }
+    },
+    warning: {
+      main: '#c4823f'
+    },
+    query_stop_button: {
+      active: {
+        color: '#ff4444a0',
+        background_color: '#ffffff40',
+      },
+      inactive: {
+        color: '#ff444450',
+        background_color: '#ffffff00',
+      }
+    },
+  };
+};
+
+const getTheme = ({ mode }) => {
+  return createTheme({
+    typography: {
+      fontFamily: '"Helvetica"'
+    },
+    palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
+  });
+};
+
+export default getTheme;
+
+getTheme.propTypes = {
+  mode: PropTypes.string.isRequired,
+};
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>
@@ -0,0 +1,51 @@
+.dark-high-contrast {
+  --md-sys-color-primary: rgb(251 250 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 0 0);
+  --md-sys-color-primary-container: rgb(173 204 255);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(250 250 255);
+  --md-sys-color-on-secondary: rgb(0 0 0);
+  --md-sys-color-secondary-container: rgb(159 208 255);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(255 249 251);
+  --md-sys-color-on-tertiary: rgb(0 0 0);
+  --md-sys-color-tertiary-container: rgb(222 193 231);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 249 249);
+  --md-sys-color-on-error: rgb(0 0 0);
+  --md-sys-color-error-container: rgb(255 186 177);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(255 255 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(251 250 255);
+  --md-sys-color-outline: rgb(200 202 212);
+  --md-sys-color-outline-variant: rgb(200 202 212);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(0 0 0);
+  --md-sys-color-inverse-primary: rgb(0 42 83);
+  --md-sys-color-primary-fixed: rgb(219 231 255);
+  --md-sys-color-on-primary-fixed: rgb(0 0 0);
+  --md-sys-color-primary-fixed-dim: rgb(173 204 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
+  --md-sys-color-secondary-fixed: rgb(214 233 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 0 0);
+  --md-sys-color-secondary-fixed-dim: rgb(159 208 255);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
+  --md-sys-color-tertiary-fixed: rgb(249 222 255);
+  --md-sys-color-on-tertiary-fixed: rgb(0 0 0);
+  --md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}
@@ -0,0 +1,51 @@
+.dark-medium-contrast {
+  --md-sys-color-primary: rgb(173 204 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 22 49);
+  --md-sys-color-primary-container: rgb(112 146 198);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(159 208 255);
+  --md-sys-color-on-secondary: rgb(0 24 42);
+  --md-sys-color-secondary-container: rgb(100 149 193);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(222 193 231);
+  --md-sys-color-on-tertiary: rgb(33 14 42);
+  --md-sys-color-tertiary-container: rgb(162 136 171);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 186 177);
+  --md-sys-color-on-error: rgb(55 0 1);
+  --md-sys-color-error-container: rgb(255 84 73);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(251 250 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(200 202 212);
+  --md-sys-color-outline: rgb(160 163 171);
+  --md-sys-color-outline-variant: rgb(128 131 139);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(40 42 47);
+  --md-sys-color-inverse-primary: rgb(35 73 120);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 17 40);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 18 34);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(28 9 37);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}
@@ -0,0 +1,51 @@
+.dark {
+  --md-sys-color-primary: rgb(165 200 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 49 94);
+  --md-sys-color-primary-container: rgb(34 72 118);
+  --md-sys-color-on-primary-container: rgb(212 227 255);
+  --md-sys-color-secondary: rgb(154 203 250);
+  --md-sys-color-on-secondary: rgb(0 51 83);
+  --md-sys-color-secondary-container: rgb(12 74 114);
+  --md-sys-color-on-secondary-container: rgb(206 229 255);
+  --md-sys-color-tertiary: rgb(218 189 226);
+  --md-sys-color-on-tertiary: rgb(61 41 70);
+  --md-sys-color-tertiary-container: rgb(85 63 93);
+  --md-sys-color-on-tertiary-container: rgb(247 216 255);
+  --md-sys-color-error: rgb(255 180 171);
+  --md-sys-color-on-error: rgb(105 0 5);
+  --md-sys-color-error-container: rgb(147 0 10);
+  --md-sys-color-on-error-container: rgb(255 218 214);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(225 226 233);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(195 198 207);
+  --md-sys-color-outline: rgb(141 145 153);
+  --md-sys-color-outline-variant: rgb(67 71 78);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(46 48 53);
+  --md-sys-color-inverse-primary: rgb(60 96 144);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}
@@ -0,0 +1,51 @@
+.light-high-contrast {
+  --md-sys-color-primary: rgb(0 34 70);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(29 68 114);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(0 36 61);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(3 70 110);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(46 26 55);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(81 59 89);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(78 0 2);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(140 0 9);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(0 0 0);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(32 36 43);
+  --md-sys-color-outline: rgb(63 67 74);
+  --md-sys-color-outline-variant: rgb(63 67 74);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(255 255 255);
+  --md-sys-color-inverse-primary: rgb(228 236 255);
+  --md-sys-color-primary-fixed: rgb(29 68 114);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(0 45 88);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(3 70 110);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(0 47 77);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(81 59 89);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}
@@ -0,0 +1,51 @@
+.light-medium-contrast {
+  --md-sys-color-primary: rgb(29 68 114);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(83 118 167);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(3 70 110);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(71 121 163);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(81 59 89);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(133 108 141);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(140 0 9);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(218 52 46);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(63 67 74);
+  --md-sys-color-outline: rgb(91 95 103);
+  --md-sys-color-outline-variant: rgb(119 123 131);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(83 118 167);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(58 93 141);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(71 121 163);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(43 96 137);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(133 108 141);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}
@@ -0,0 +1,51 @@
+.light {
+  --md-sys-color-primary: rgb(60 96 144);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(212 227 255);
+  --md-sys-color-on-primary-container: rgb(0 28 58);
+  --md-sys-color-secondary: rgb(46 98 140);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(206 229 255);
+  --md-sys-color-on-secondary-container: rgb(0 29 50);
+  --md-sys-color-tertiary: rgb(110 86 118);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(247 216 255);
+  --md-sys-color-on-tertiary-container: rgb(39 20 48);
+  --md-sys-color-error: rgb(186 26 26);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(255 218 214);
+  --md-sys-color-on-error-container: rgb(65 0 2);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(67 71 78);
+  --md-sys-color-outline: rgb(116 119 127);
+  --md-sys-color-outline-variant: rgb(195 198 207);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}
@@ -0,0 +1,107 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+button#query-button {
+  height: 5em;
+  vertical-align: top;
+}
+
+button#play-button {
+  padding: '10px 20px';
+  font-size: '16px';
+  cursor: 'pointer';
+  background-color: '#28a745';
+  color: '#fff';
+  border: 'none';
+  border-radius: '5px';
+};
+
+button#pause-button {
+  background-color: #ffc107;
+}
+
+button#stop-button {
+  background-color: #dc3545;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+p#footer {
+  float: none;
+  clear: both;
+  vertical-align: bottom;
+}
+
+table#query-output {
+  float: none;
+  clear: both;
+}
+
+textarea#query-input {
+  height: 5em;
+  width: 50em;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}
@@ -0,0 +1,9 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App';
+
+ReactDOM.createRoot(document.getElementById('root')).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+)
@@ -0,0 +1,69 @@
+import { fileURLToPath, URL } from 'node:url';
+import { defineConfig } from 'vite';
+import plugin from '@vitejs/plugin-react';
+import fs from 'fs';
+import path from 'path';
+import child_process from 'child_process';
+import process, { env } from 'process';
+
+const baseFolder =
+  process.env.APPDATA !== undefined && process.env.APPDATA !== ''
+    ? `${process.env.APPDATA}/ASP.NET/https`
+    : `${process.env.HOME}/.aspnet/https`;
+
+const certificateName = "bts.datastreamwebapp.client";
+const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
+const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
+
+if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
+  if (0 !== child_process.spawnSync('dotnet', [
+    'dev-certs',
+    'https',
+    '--export-path',
+    certFilePath,
+    '--format',
+    'Pem',
+    '--no-password',
+  ], { stdio: 'inherit', }).status) {
+    throw new Error("Could not create certificate.");
+  }
+}
+
+const target = env.ASPNETCORE_HTTPS_PORT
+  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
+  : env.ASPNETCORE_URLS
+    ? env.ASPNETCORE_URLS.split(';')[0]
+    : 'https://localhost:5001';
+
+export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
+  return {
+    plugins: [plugin()],
+    resolve: {
+      alias: {
+        '@': fileURLToPath(new URL('./src', import.meta.url))
+      }
+    },
+    build: {
+      minify: false,
+      sourcemap: true,
+      outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
+    },
+    server: {
+      proxy: {
+        '^/api': {
+          target,
+          secure: false
+        },
+        '^/ws': {
+          target,
+          secure: false
+        }
+      },
+      port: 5173,
+      https: {
+        key: fs.readFileSync(keyFilePath),
+        cert: fs.readFileSync(certFilePath),
+      }
+    }
+  };
+});
@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <InvariantGlobalization>true</InvariantGlobalization>
+    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
+    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
+    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+    </ProjectReference>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+  </ItemGroup>
+
+  <!-- <ItemGroup>
+    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
+  </ItemGroup> -->
+</Project>
@@ -0,0 +1,106 @@
+{
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:00.1"
+  },
+  "Sparkd": {
+    "Config": {
+      "ConfigServer": "http://localhost:8000/config"
+    },
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    },
+    "EnableAdmin": true,
+    "EnableOrders": true,
+    "EnablePosition": true
+  },
+  "PubSub": {
+    "MillisecondsBetweenSubInfoUpdates": 1000,
+    "MillisecondsBetweenTimeSliceUpdates": 5000,
+    "MillisecondsBetweenNoBufferSpaceRetries": 100,
+    "RetriesOnNoBufferSpace": 1,
+    "DoItAllSecondsBetweenLogs": 0,
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "UseManagedSockets": true,
+    "StateUpdatePeriod": 5000,
+    "StatisticsUpdatePeriod": 5000,
+    "DisposedConnectionCheckPeriod": 1000,
+    "MessageProcessingWarningThreshold": 1000,
+    "ServerPort": 12345,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "MaxPacketSize": 500000000,
+    "BlueCommMustDie": true,
+    "CrushableTags": [],
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "ServerQueue": "PubSubQueue",
+    "PortRange": {
+      "First": 30400,
+      "Last": 31700
+    },
+    "BindToRandomPort": false,
+    "DebugTags": "^NOTAGS$",
+    "LatencyReportThresholdMs": 1500,
+    "UseSubstations": false,
+    "UseDynamicClients": false,
+    "IsSubstation": false,
+    "InternallyPublishSubInfo": true,
+    "UseManagedClientsInNativeCode": false,
+    "CompressedTags": "^NOTAGS$",
+    "TagsToReportCompressionRatio": "^NOTAGS$",
+    "DisableMsmq": false,
+    "UsePrepublishedServerState": false,
+    "IpTranslation": null,
+    "GcActivationThreshold": 0,
+    "WaitPeriodAfterGc": 1000,
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+         "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  }
+}
@@ -0,0 +1,4 @@
+{
+  "content_root_path": "build/Bts.DataStreamWebApp.Client/production",
+  "web_root_path": "dist"
+}
@@ -0,0 +1,30 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using Microsoft.AspNetCore.Mvc;
+
+[Route("/ws")]
+public class DataStreamSubscriptionController : ControllerBase {
+   readonly ILogger logger;
+
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
+      this.logger = logger;
+      logger.LogInformation("Creating {ControllerType}", GetType().Name);
+   }
+
+   [Route("/ws")]
+   public async Task Get(CancellationToken cancellationToken) {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      logger.LogInformation("Establishing websocket connection...");
+      try {
+         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+         logger.LogInformation("Websocket connection established.");
+         using var server = new DataStreamWebSocketServer(webSocket, logger);
+         await server.Run(cancellationToken);
+      } catch (Exception ex) {
+         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
+      }
+   }
+}
@@ -0,0 +1,125 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamSubscription : IDisposable, IHostedService {
+   public readonly ViewStream ViewStream;
+   public DataPacket? LastFired;
+
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+
+   public DataStreamSubscription(string id, JsonRpcId requestId, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
+      ViewStream = viewStream;
+      RequestId = requestId;
+      Id = id;
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public string Id { get; }
+   public JsonRpcId RequestId { get; }
+   public SubscriptionStatus Status { get; private set; }
+
+   public void Start() {
+      if (Status == SubscriptionStatus.Subscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Subscribed;
+      ViewStream.onChange += onChange;
+   }
+
+   public Task StartAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Start();
+      return Task.CompletedTask;
+   }
+
+   public Task Clear() {
+      if (LastFired is null) {
+         return Task.CompletedTask;
+      }
+      logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
+      return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
+   }
+
+   public void Stop() {
+      if (Status == SubscriptionStatus.Unsubscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Unsubscribed;
+      ViewStream.onChange -= onChange;
+   }
+
+   public Task StopAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Stop();
+      return Task.CompletedTask;
+   }
+
+   public void Dispose() =>
+      Stop();
+
+   async void onChange(object newValue) {
+      if (newValue is not DataPacket dp) {
+         logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
+         return;
+      }
+      if (webSocket.State != WebSocketState.Open) {
+         logger.LogError(
+            "Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
+         Dispose();
+         return;
+      }
+      await sendDataPacket("subscribe", dp);
+   }
+
+   async Task sendDataPacket(string method, DataPacket dp) {
+      var serializerOpts = new JsonSerializerOptions {
+         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+      };
+      serializerOpts.Converters.Add(new DataPacketJsonConverter() {
+         IncludeCurrent = true,
+         UseObjectRows = true,
+      });
+      var res = JsonRpcRequest.CreateNotification(method, dp);
+      var str = serialize(res, serializerOpts);
+      if (str is null) {
+         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
+         Dispose();
+         return;
+      }
+      logger.LogInformation(
+         "Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
+         dp.newRows.Count,
+         dp.schema.columnNames.Length,
+         str.Length);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
+      LastFired = dp;
+   }
+
+   string? serialize<T>(T message, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      try {
+         return JsonSerializer.Serialize(message, opts);
+      } catch (Exception ex) {
+         logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
+      }
+      return null;
+   }
+}
@@ -0,0 +1,89 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using Bts.PubSub;
+
+using CommandLine;
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamWebAppServerConfig {
+   /// <summary>
+   /// The login user for sparkd
+   /// </summary>
+   [JsonPropertyName("user")]
+   [Option("user", HelpText = "The sparkd login user")]
+   public string? User { get; set; }
+
+   /// <summary>
+   /// The login password for sparkd
+   /// </summary>
+   [JsonPropertyName("password")]
+   [Option("pass", HelpText = "The sparkd login password")]
+   public string? Password { get; set; }
+
+   /// <summary>
+   /// The config server url for sparkd
+   /// </summary>
+   [JsonPropertyName("sparkd")]
+   [Option("sparkd", HelpText = "The sparkd config server")]
+   public string? Sparkd { get; set; }
+
+   [JsonPropertyName("pubsub")]
+   public PubSubConfig? PubSub { get; set; }
+
+   [Option("content-root-path")]
+   [JsonPropertyName("content_root_path")]
+   public string? ContentRootPath { get; set; }
+
+   [Option("web-root-path")]
+   [JsonPropertyName("web_root_path")]
+   public string? WebRootPath { get; set; }
+
+   [Option("environment-name")]
+   [JsonPropertyName("environment_name")]
+   public string? EnvironmentName { get; set; }
+
+   public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
+
+   [Option(
+      "config",
+      MetaValue = "PATH",
+      HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
+   [JsonIgnore]
+   public string? ConfigFile { get; set; }
+
+   public static DataStreamWebAppServerConfig Default() =>
+      new() {
+         User = "bts",
+         PubSub = PubSubConfig.Default(),
+      };
+
+   public void MergeWith(DataStreamWebAppServerConfig other) {
+      User ??= other.User;
+      Password ??= other.Password;
+      PubSub ??= other.PubSub;
+      Sparkd ??= other.Sparkd;
+      ContentRootPath ??= other.ContentRootPath;
+      WebRootPath ??= other.WebRootPath;
+      ConfigFile ??= other.ConfigFile;
+   }
+
+   /// <summary>
+   /// Merges options with those defined in `ConfigFile` (does not overwrite
+   /// non-null existing values).
+   /// </summary>
+   /// <exception cref="InvalidOperationException">
+   /// JsonSerializer.Deserialize returns null
+   /// </exception>
+   public void MergeWithConfigFile() {
+      if (ConfigFile is null) {
+         return;
+      }
+      var cfgString = File.ReadAllText(ConfigFile);
+      var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
+         throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
+      MergeWith(parsedCfg);
+      ConfigFile = null;
+   }
+}
@@ -0,0 +1,121 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
+
+   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public async Task<int> Run(CancellationToken cancellationToken) {
+      logger.LogInformation("DataStreamWebSocketServer running");
+      await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger, cancellationToken)) {
+         await (req.Method switch {
+            "subscribe" => onSubscribe(req, cancellationToken),
+            "unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>(), cancellationToken),
+            _ => Task.CompletedTask,
+         });
+      }
+      logger.LogInformation("DataStreamWebSocketServer stopped");
+      return 0;
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions. To additionally notify the client of the subscription status
+   /// change(s), use <see cref="DisposeAsync"/>.
+   /// </summary>
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      disposeSubscriptions();
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions (like <see cref="Dispose"/>), then sends 'subscription status change'
+   /// notifications to the client (unlike <see cref="Dispose"/>).
+   /// </summary>
+   public async ValueTask DisposeAsync() {
+      GC.SuppressFinalize(this);
+      var finalSubs = subscriptions.Values.Select(s => (s.RequestId, s.Id)).ToList();
+      disposeSubscriptions();
+      // Send a 'subscription status change' msg to the client for each one.
+      foreach (var (requestId, id) in finalSubs) {
+         await sendSubscriptionStatus(requestId, id, "unsubscribe", SubscriptionStatus.Unsubscribed, CancellationToken.None);
+      }
+   }
+
+   async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      if (!SubscribeRequest.ParseQuery(req, out var vs, out var subId, out var errMsg, out var errCode)) {
+         logger.LogError("Invalid 'subscribe' request: {Reason}", errMsg);
+         var rpcError = new JsonRpcError<JsonNode>() {
+            Code = (long)errCode,
+            Message = $"{errCode}: {errMsg}",
+         };
+         await send(rpcError, cancellationToken);
+         return;
+      }
+      if (subscriptions.TryGetValue(subId, out var oldSub)) {
+         oldSub.Dispose();
+      }
+      var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
+      subscriptions[subId] = sub;
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
+      await sub.StartAsync(cancellationToken);
+   }
+
+   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      var subId = req.Params?.SubscriptionId ?? string.Empty;
+      if (!subscriptions.TryGetValue(subId, out var sub)) {
+         logger.LogInformation("Ignoring '{Method}' request because no subscription was found with id {Id}.", req.Method, subId);
+         return;
+      }
+      logger.LogInformation("Disposing subscription");
+      sub.Dispose();
+      subscriptions.Remove(subId);
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   async Task unsubscribe(JsonRpcId requestId, string subscriptionId, CancellationToken cancellationToken) {
+      await sendSubscriptionStatus(requestId, subscriptionId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
+      logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
+      var response = JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
+         SubscriptionId = subId,
+         Status = status,
+      });
+      return send(response, cancellationToken);
+   }
+
+   Task send<T>(T message, CancellationToken cancellationToken) {
+      if (message is null) {
+         return Task.CompletedTask;
+      }
+      var str = JsonSerializer.Serialize(message);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
+      return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken);
+   }
+
+   void disposeSubscriptions() {
+      logger.LogInformation("Disposing DataStreamWebSocketServer subscriptions. Count: {Count}", subscriptions.Count);
+      subscriptions.Values.DisposeAll();
+      subscriptions.Clear();
+   }
+}
@@ -0,0 +1,6 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+enum ErrorCode {
+   None = 0,
+   InvalidSubscriptionRequest,
+}
@@ -0,0 +1,67 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamExpressions;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+public class SubscribeRequest {
+   [JsonPropertyName("query")]
+   public string? Query { get; set; }
+
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   internal static bool ParseQuery(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(true)] out string? subscriptionId, [NotNullWhen(false)] out string? errorMessage, out ErrorCode errorCode) {
+      stream = null;
+      subscriptionId = null;
+      errorCode = ErrorCode.None;
+      var paramsJsonObj = req.Params?.AsObject();
+      if (paramsJsonObj is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params' object not found";
+         return false;
+      }
+      var subReq = paramsJsonObj.Deserialize<SubscribeRequest>();
+      if (subReq is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params' object invalid";
+         return false;
+      }
+      subscriptionId = subReq.SubscriptionId ?? string.Empty;
+      var query = subReq.Query;
+      if (query is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params.query' field not found (expected a string)";
+         return false;
+      }
+      errorMessage = null;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(query);
+      if (exp is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params.query' expression has invalid syntax";
+         return false;
+      }
+      string? exceptionMsg = null;
+      try {
+         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      } catch (Exception ex) {
+         exceptionMsg = ex.ToString();
+      }
+      if (stream is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
+         if (exceptionMsg is not null) {
+            errorMessage += ". Error" + exceptionMsg;
+         }
+         return false;
+      }
+      return true;
+   }
+}
@@ -0,0 +1,18 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+enum SubscriptionStatus {
+   None,
+   Subscribed,
+   Unsubscribed,
+}
+
+class SubscriptionStatusMessage {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   [JsonPropertyName("status")]
+   [JsonConverter(typeof(JsonStringEnumConverter))]
+   public SubscriptionStatus Status { get; set; } = SubscriptionStatus.None;
+}
@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+public class UnsubscribeRequest {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+}
@@ -0,0 +1,59 @@
+using Bts.DataStreamCore.Services;
+using Bts.PubSub.Services;
+
+using Spark.Api.Services;
+
+using System.Reactive.Linq;
+
+var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
+   Args = args,
+   ApplicationName = "Bts.DataStreamWebApp.Server",
+});
+ConfigureApp(builder.Configuration);
+ConfigureServices(builder.Services, builder.Configuration);
+
+var app = builder.Build();
+if (app.Environment.IsDevelopment()) {
+   app.UseDirectoryBrowser();
+}
+var wsOpts = new WebSocketOptions {
+   KeepAliveInterval = TimeSpan.FromMinutes(2)
+};
+app.UseWebSockets(wsOpts);
+app.UseDefaultFiles();
+app.UseStaticFiles();
+if (app.Environment.IsDevelopment()) {
+   app.UseSwagger();
+   app.UseSwaggerUI();
+}
+app.UseHttpsRedirection();
+app.UseAuthorization();
+app.MapControllers();
+try {
+   Console.Write($"Starting Application");
+   app.Run();
+   Console.Write($"Application exited gracefully.");
+} catch (Exception ex) {
+   Console.Error.Write($"Application crashed: {ex}");
+   return 1;
+}
+return 0;
+
+static void ConfigureApp(ConfigurationManager configBuilder) {
+   configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
+   var cfgRoot = configBuilder.GetSection("config");
+   if (cfgRoot?.Value is not null) {
+      configBuilder.AddJsonFile(cfgRoot.Value);
+   }
+}
+
+static void ConfigureServices(IServiceCollection services, IConfiguration config) {
+   services.AddDirectoryBrowser();
+   services.AddControllers();
+   services.AddEndpointsApiExplorer();
+   services.AddSwaggerGen();
+   services.AddMasterWorker(config.GetSection("MasterWorker"));
+   services.AddViewStreamDatabase(config.GetSection("ViewStreamDatabase"));
+   services.AddPubSubClient(config.GetSection("PubSub"));
+   services.AddSparkd(config.GetSection("Sparkd"));
+}
@@ -0,0 +1,26 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "applicationUrl": "http://localhost:5000",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:5000;https://localhost:5001",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+  },
+}
@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}
@@ -0,0 +1,168 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>ALT WebSocket Sample Application</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Message to send:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            var data = sendMessage.value;
+            socket.send(data);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>
@@ -0,0 +1,176 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>DataStreams via WebSocket</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Query:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            const rpcReq = {
+              jsonrpc: "2.0",
+              method: "subscribe",
+              params: {
+                query: sendMessage.value,
+              },
+              id: 1,
+            };
+            const rpcReqStr = JSON.stringify(rpcReq);
+            socket.send(rpcReqStr);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>
@@ -4,6 +4,8 @@ using System.Collections.Generic;
 using Bts.DataStreamExpressions;
 using BCG.Utilities.General;
 using bts;
+using Bts.PubSub;
+using Bts.DataStreamCore;

 namespace BCG {
   // listens to the requests posted on "cloud_streams" request table, and 
@@ -40,13 +42,15 @@ namespace BCG {
      private IDictionary<string, StreamPubUnit>
         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
      private ViewStream _stream = new TableStream("cloud_streams");
-      private PubTableBroker _broker = new PubTableBroker();
-      private ManagedSubscriber _sub = ManagedSubscriber.instance;
+      readonly IPubTableBroker _broker;
+      readonly IManagedSubscriber _sub;


-      public RemoteQueryManager() {
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
+         _sub = managedSubscriber;
+         _broker = pubTableBroker;
         _stream.onChange += _on_str;
-         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
+         _sub.Subscribe("RemoteRequest", (string tag, byte[] data) => {
            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
         });
@@ -1,24 +1,23 @@
-using bts;
-
 using System;
 using System.Collections;

-namespace BCG {
+namespace BCG
+{
 	/// <summary>
 	/// Provides a stream that with distinct rows with respect to the columns
 	/// the user specifies in the argument
 	/// </summary>
-	[DataStreamUserRespresentation("distinct")]
-	public class DistinctRowStream : ViewStream {
-		public DistinctRowStream(ViewStream stream, params string[] cols) {
-			var g_by_eval = new DistinctRowEvaluator(cols);
-			ViewStream r = new GroupByStream(stream, g_by_eval);
-			var proj_exp = String.Empty;
-			foreach (var c in cols)
-				proj_exp += $"{c} ";
-			proj_exp = proj_exp.Trim();
-			r = new NewProjectStream(r, proj_exp);
-			mySetRealSource(r);
+	[DataStreamUserRespresentation ("distinct")]
+	public class DistinctRowStream : ViewStream
+	{
+		public DistinctRowStream (ViewStream stream, params string[] cols)
+		{
+         ArrayList groupByCols = new ArrayList ();
+         foreach (string col in cols)
+            foreach (string c in col.Split (' '))
+               groupByCols.Add (c);
+         mySetRealSource (new GroupByStream (stream, 
+            new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
 		}
 	}
 }
@@ -0,0 +1,49 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.IO;
+
+// using System.Runtime.Serialization.Formatters.Binary;
+
+// namespace BCG
+// {
+//    public class DumpLoaderStream : ViewStream, ISideEffectStream
+//    {
+//       //wtf
+//       //public class Exception : System.Exception
+//       //{public Exception(string msg) : base(msg) { }}
+
+//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
+//       {
+//          mySetRealSource(
+//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
+//                ViewStreamDatabase.instance));
+//       }
+
+//       public DumpLoaderStream(string fileName)
+//       {
+//         if (fileName == null)
+//            throw new LoaderStream.Exception("File name cannot be null");
+
+//         for (int i = 0; i < 5; i++)
+//            try
+//            {
+//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
+//               {
+//                  IClientDeserializer deser = new ClientUniversalDeserializer();
+//                  byte[] the_ops;
+//                  Array the_rows;
+//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
+
+//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
+//               }
+//               break;
+//            }
+//            catch (IOException)
+//            {
+//               System.Threading.Thread.Sleep(50);
+//            }
+
+//       }
+//    }
+// }
@@ -1,7 +1,6 @@
 using BCG;

 namespace bts {
-   // This is the other side of the SaverStream, its "loads" the stream and fires once
   [DataStreamUserRespresentation("loader")]
   public class LoaderStream : ViewStream {

@@ -5,11 +5,10 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using BCG;
+using bts;

-namespace bts {
-   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
-   // which just writes one update, as a create)
+namespace BCG {
+   // Writes all the udpates to a stream in a file.
   // File naming:
   // token_epoch_yyyyMMdd_HHmmss
   // File location: directory(passed)\token\
@@ -95,10 +94,10 @@ namespace bts {
      void _on_change(object new_value) {
         DataPacket dp = new_value as DataPacket;
         byte[] serialized_update = null;
-         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
+         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
         _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (first_write) {
-            // first file always starts with create
+         if (_write_create) {
+            // new files always starts with create
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
         } else
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -106,8 +105,8 @@ namespace bts {
         long utc_epoch = epoch_now();

         //time, length of data, data
-         //bw.Write(utc_epoch);
-         //bw.Write(serialized_update.Length);
+         bw.Write(utc_epoch);
+         bw.Write(serialized_update.Length);
         bw.Write(serialized_update);
         _out_fs.Flush();
         myFireEvent(dp);
@@ -0,0 +1,190 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+// using System.Runtime.Serialization.Formatters.Binary;
+// using BCG.Utilities.General;
+// using bts.collections;
+// using System.Reflection.Metadata;
+
+
+// namespace BCG {
+
+//    public class PlaybackStream : ViewStream, IWorker {
+//       private readonly PlaybackType _type;
+//       private BlueTable _current_table;
+//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+//       // these are the packets already downloaded from the current file
+//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
+//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
+//       private IEnumerable<string> _file_names;
+//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
+//       private DateTime _playback_t0, _playback_t1;
+//       private bool _all_files_read = false;
+//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
+
+//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
+//          _type = PlaybackType.Streaming;
+//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
+//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
+//          string token_directory_name = $"{directory_name}\\{token}";
+//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
+//          List<string> files_in_range = new List<string>();
+//          if (Directory.Exists(token_directory_name)) {
+//             _file_names =
+//          Directory.GetFiles(token_directory_name).Where(f => {
+//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
+//             return file_time > _playback_t0 && file_time < _playback_t1;
+//          });
+//          }
+//       }
+
+//       protected override void myOnFirstSubscriber() {
+//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
+//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
+//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
+//          if (snapshot_packet != null) {
+//             foreach (PlaybackPacket p in _snapshot_data.packets) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+//                for (int i = 0; i < ops.Length; i++) {
+//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//                   byte op = ops[i];
+//                   if (op == StreamOpCode.create) {
+//                      IBlueRowCollection full_table = rows_args[0];
+//                      _current_table = new BlueTable(full_table);
+//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                   } else {
+//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
+//                   }
+//                   if (p.idx == snapshot_packet.idx) {
+//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+//                      break;
+//                   }
+//                }
+//             }
+//          }
+//       }
+
+
+//       private void _load_packets_from_current_file() {
+//          int curr_idx = _current_fs.Item1;
+//          FileStream curr_fs = _current_fs.Item2;
+//          _packets = new List<PlaybackPacket>();
+//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
+//       }
+
+//       private bool current_file_all_replayed {
+//          get {
+//             bool rv = true;
+//             _packets.ForEach(p => rv = rv && p.replayed);
+//             return rv;
+//          }
+//       }
+
+//       protected override void myOnLastSubscriber() {
+//          var fs = _current_fs.Item2;
+//          if (fs != null) {
+//             fs.Close();
+//          }
+//          ViewStreamDatabase.instance.removeWorker(this);
+//       }
+
+//       // public event PlaybackStreamProgress onProgress;
+//       // public event Action onFileExhausted;
+//       private PlaybackPacket _locate_packet_to_replay() {
+//          foreach (var p in _packets) {
+//             // returns the first non replayed packet
+//             if (!p.replayed)
+//                return p;
+//          }
+//          //else
+//          {
+//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
+//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
+//             if (onProgress != null)
+//                onProgress(myFileReader.Position, myFileReader.Length);
+//          }
+//       }
+
+//       #region IWorker Members
+
+//       public void doWork() {
+//          if (_all_files_read)
+//             return;
+//          if (current_file_all_replayed) {
+//             _load_next_file();
+//          }
+//          PlaybackPacket curr_packet = _locate_packet_to_replay();
+//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
+//             byte[] ops;
+//             Array row_array;
+//             // what a stupid interface!
+//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+//             _fire_packet(ops, row_array);
+//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
+//          } else {
+//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
+//             // this is the time we need to fire (replay) this packet
+//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
+//             long now_epoch = PersistorStream.epoch_now();
+//             // fire when we are past the 
+//             if (now_epoch >= time_to_fire) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+//                _fire_packet(ops, row_array);
+//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
+//             }
+//          }
+//       }
+
+//       private void _fire_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//                //_current_table.addRows(full_table);
+//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+//                                           (myLastValue as DataPacket).newRows));
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                myFireEvent(new DataPacket(_current_table, adds, dels));
+//             } else {
+//                bts.Log.assert(false, $"Oprator: {op} is not expected");
+//             }
+
+//          }
+//       }
+
+//       private void _load_next_file() {
+//          var new_idx = _current_fs.Item1 + 1;
+//          if (new_idx >= _file_names.Count()) {
+//             _all_files_read = true;
+//             bts.Log.Gen.info("All files read");
+//             return;
+//          }
+//          var prev_fs = _current_fs.Item2;
+//          if (prev_fs != null)
+//             prev_fs.Close();
+//          var new_filename = _file_names.ElementAt(new_idx);
+//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
+//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
+//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
+//          _load_packets_from_current_file();
+//       }
+
+//       #endregion
+//    }
+// }
@@ -0,0 +1,68 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+
+// namespace BCG
+// {
+//    public class RecorderStream : ViewStream
+//    {
+//       private ViewStream mySource;
+//       private DateTime myStartTime;
+//       private FileStream myOutFile;
+//       private string myFileName;
+//       private IBlueDataSerializer mySerializer;
+
+//       public RecorderStream(ViewStream src, string recordFile)
+//       {
+//          mySource = src;
+//          myFileName = recordFile;
+
+//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
+//       }
+
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStartTime = DateTime.Now;         
+//          mySource.onChange += new ChangeDelegate(mySource_onChange);
+//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       {
+//          mySource.onChange -= mySource_onChange;
+//          myOutFile.Close();
+//       }
+
+//       void mySource_onChange(object newValue)
+//       {
+//          DataPacket dp = (DataPacket)newValue;         
+
+//          BlueTable toWrite = new BlueTable(dp.schema);
+
+//          if (mySerializer == null)
+//          {
+//             mySerializer = new UniversalSerializer(dp.schema);
+//          }
+//          foreach (IBlueRow r in dp.newRows)
+//             toWrite.addRow(r);
+
+//          byte[] result;
+
+//          if (myLastValue == null)
+//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
+//          else
+//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+
+//          BinaryWriter bw = new BinaryWriter(myOutFile);
+//          //write down what time this happened
+//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
+//          bw.Write(result.Length);
+//          bw.Write(result);         
+//          myOutFile.Flush();
+
+//          myFireEvent(dp);
+//       }
+//    }
+// }
@@ -5,8 +5,7 @@ using System.IO;
 using BCG;

 namespace bts {
-   // this saves the updates as a Create on the same file, each update calls -save, which
-   // reqrites the whole data to the packet. The other side of this stream is the Loader
+   // this saves the total state in the filr provided in the cons
   [DataStreamUserRespresentation("saver")]
   public class SaverStream : ViewStream {
      private ViewStream _stream;
@@ -32,6 +31,7 @@ namespace bts {
         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
         StreamSerializationUtils.write_to_file(str_bytes, _filename);
+
      }
   }
 }
@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
-using bts;


 namespace BCG {
@@ -1,9 +1,7 @@
-using System;
+#nullable disable
+using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using BCG.Utilities;

 namespace BCG
 {
@@ -1,6 +1,10 @@
+using bts;
+
 using Bts.DataStream;
+using Bts.DataStreamCore;

 using mbts.config;
+using mbts.utils;

 using System;

@@ -16,8 +20,8 @@ namespace BCG {
   [DataStreamUserRespresentation (false)]
   public class MapStream : MethodDrivenStream
   {
-      protected IStreamMapper myMapper;
-      protected IChangeableStreamMapper myChangeableMapper = null;
+      protected IStreamMapper? myMapper;
+      protected IChangeableStreamMapper? myChangeableMapper;
      private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;

      public class FuncWrapper : IStreamMapper {
@@ -72,7 +76,7 @@ namespace BCG {


      protected override void myHandleFirstCall() {
-
+         ArgumentNullException.ThrowIfNull(myMapper);
         IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
         myLastFiredPacket = new DataPacket();
         myLastFiredPacket.newRows = new BlueTable(outputSchema);
@@ -83,28 +87,34 @@ namespace BCG {
      }

      protected override void myHandleIncrementalAdd() {
-         //t.start ();
-         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
-            IBlueRow newRow = myMapper.map(row);
-            (myLastFiredPacket.newRows as BlueTable).addRow(newRow);
-            (myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
+         if (myLastArrivedPacket.addedRows.Count == 0) {
+            Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
+            return;
         }
-         //t.stop ();
-         //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
-         //System.Diagnostics.Trace.WriteLine (" ");
+         ArgumentNullException.ThrowIfNull(myMapper);
+         var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
+         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
+            var newRow = myMapper.map(row);
+            newRows.addRow(newRow);
+            addedRows.addRow(newRow);
+         }
+         myLastFiredPacket.addedRows = addedRows;
      }

      protected override void myHandleIncrementalDelete() {
-         //t.start ();
-         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
-            IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
-            (myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
-            (myLastFiredPacket.newRows as BlueTable).deleteRow(row);
+         if (myLastArrivedPacket.deletedRows.Count == 0) {
+            Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
+            return;
         }
-         //t.stop ();
-         //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
+         var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
+         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
+            var delRow = newRows[row.Id()];
+            deletedRows.addRow(delRow);
+            newRows.deleteRow(row);
+         }
+         myLastFiredPacket.deletedRows = deletedRows;
      }


@@ -1,11 +1,9 @@
+using Bts.DataStreamCore;
+
+using mbts.utils;
+
 using System;
 using System.Collections.Generic;
-using System.Text;
-
-using BCG.Utilities;
-using BCG.Utilities.General;
-
-using mbts;

 namespace BCG {
   /// <summary>
@@ -56,15 +54,15 @@ namespace BCG {
            return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
         }

-         public event MapperChangeDelegate onMapperChange;
-         private void _fake() { onMapperChange(); }
+         public event MapperChangeDelegate? onMapperChange;
      }

      //maps the id of the underlying stream to the
      //rows of the resulting stream
      private Dictionary<int, List<IBlueRow>> myMap = new Dictionary<int, List<IBlueRow>>();
      private int myID = 0;
-      private BlueTable myRows, myBaseRows;
+      private BlueTable? myRows;
+      private BlueTable? myBaseRows;
      private ViewStream myStream;
      private ISuperStreamMapper myMapper;
      public bool generate_ids = true;
@@ -92,49 +90,40 @@ namespace BCG {
         myRows = null;
         myBaseRows = null;
         myMap.Clear();
-         //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
      }

      private void myHandleData(object newValue) {
-         DataPacket idp = newValue as DataPacket;
-         //HighPerformanceTimer timer = new HighPerformanceTimer();
-         //timer.start();
-         DataPacket dp = newValue as DataPacket;
-         myRows = myRows ?? new BlueTable(myMapper.getOutputSchema(dp.schema));
-         myBaseRows = myBaseRows ?? new BlueTable(dp.schema);
+         var dp = ExceptionAssert.Cast(newValue).To<DataPacket>();
+         myRows ??= new BlueTable(myMapper.getOutputSchema(dp.schema));
+         myBaseRows ??= new BlueTable(dp.schema);
         BlueDataUtility.absorbChanges(myBaseRows, dp.deletedRows, dp.addedRows);
-         BlueTable dels = new BlueTable(myRows.schema),
-                     adds = new BlueTable(myRows.schema);
+         var dels = new BlueTable(myRows.schema);
+         var adds = new BlueTable(myRows.schema);

         foreach (IBlueRow r in dp.deletedRows) {
-            List<IBlueRow> mappedRows = myMap[(int)r["id"]];
+            var mappedRows = myMap[r.Id()];
            foreach (IBlueRow mr in mappedRows) {
               dels.addRow(mr);
               myRows.deleteRow(mr);
            }
-            myMap.Remove((int)r["id"]);
+            myMap.Remove(r.Id());
         }
         foreach (IBlueRow r in dp.addedRows) {
            IBlueRow[] mappedRows = myMapper.map(r);
            foreach (IBlueRow mr in mappedRows) {
               IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
-               if (!myMap.ContainsKey((int)r["id"])) {
-                  myMap.Add((int)r["id"], new List<IBlueRow>());
-               }
-               myMap[(int)r["id"]].Add(idChangedMapRow);
+               myMap.GetOrAdd(r.Id(), _ => new()).Add(idChangedMapRow);
               myRows.addRow(idChangedMapRow);
               adds.addRow(idChangedMapRow);
            }
         }
-         //timer.stop();
-         // if (timer.lastIntervalTime > 20)
-         //    {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
         myFireEvent(new DataPacket(myRows, adds, dels));
      }

      private void myHandleMapChange() {
-         if (myBaseRows != null)
+         if (myBaseRows is not null) {
            myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
+         }
      }
   }
 }
@@ -8,33 +8,26 @@ using System;
 using System.IO;
 using System.Linq;

-namespace bts
-{
-    public class PubSubTestingStream : ViewStream, IWorker
-    {
-        public class PayloadMessage
-        {
+namespace bts {
+    public class PubSubTestingStream : ViewStream, IWorker {
+        public class PayloadMessage {
            public int msg_id = 0;
            public string guid;
            public Int64 travel_epoch_ms = 0;
            public byte[] payload = null;
-            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
-            {
+            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
                msg_id = msg_id_args;
                travel_epoch_ms = send_epoch_args;
                payload = new byte[payload_size];
-                for (int i = 0; i < payload_size; i++)
-                {
+                for (int i = 0; i < payload_size; i++) {
                    payload[i] = payload_byte;
                }
            }
            private PayloadMessage() { }
            public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
-            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
-            { }
+            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }

-            public byte[] serialize()
-            {
+            public byte[] serialize() {
                var str = new MemoryStream();
                BinaryWriter bw = new BinaryWriter(str);
                bw.Write(msg_id);
@@ -47,8 +40,7 @@ namespace bts
                return bytes;
            }

-            public static PayloadMessage deserialize(byte[] ser_bytes)
-            {
+            public static PayloadMessage deserialize(byte[] ser_bytes) {
                var str = new MemoryStream(ser_bytes);
                BinaryReader br = new BinaryReader(str);
                int msg_id = br.ReadInt32();
@@ -56,8 +48,7 @@ namespace bts
                Int64 epoch = br.ReadInt64();
                int len = br.ReadInt32();
                var payload = br.ReadBytes(len);
-                return new PayloadMessage()
-                {
+                return new PayloadMessage() {
                    msg_id = msg_id,
                    guid = guid,
                    travel_epoch_ms = epoch,
@@ -82,8 +73,7 @@ namespace bts
        private readonly int _payload_sz;
        private const byte _payload_byte = (byte)'A';
        private readonly string _tag_name;
-        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
-        {
+        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
            _vsd = vsd;
            _payload_sz = payload_size;
            var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -106,53 +96,43 @@ namespace bts
        public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
        : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }

-        protected override void myOnFirstSubscriber()
-        {
+        protected override void myOnFirstSubscriber() {
            _vsd.addWorker(this);
            System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
            th.IsBackground = true;
            th.Start();
        }

-        protected override void myOnLastSubscriber()
-        {
+        protected override void myOnLastSubscriber() {
            _vsd.removeWorker(this);
            _continue_sub_thread = false;
        }

-        void _log_updates(IEnumerable<PubSubServerMessage> updates)
-        {
+        void _log_updates(IEnumerable<PubSubServerMessage> updates) {
            var idx = 0;
-            foreach (var msg in updates)
-            {
+            foreach (var msg in updates) {
                bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
                idx++;
            }
        }

-        private void _subscribe_thread_func()
-        {
+        private void _subscribe_thread_func() {
            // subscription is done in another thread
            SubscriberCore sub_core = new SubscriberCore();
            sub_core.subscribe(_tag_name);
-            while (_continue_sub_thread)
-            {
+            while (_continue_sub_thread) {
                // TODO recv_id
                var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
                if (updates == null || updates.Count() == 0)
                    continue;
                _log_updates(updates);
                _n_updates += updates.Count();
-                if (updates.Count() > 0)
-                {
+                if (updates.Count() > 0) {
                    var server_msg = updates.Last();
-                    if (server_msg.data != null && server_msg.data.Length > 0)
-                    {
+                    if (server_msg.data != null && server_msg.data.Length > 0) {
                        _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
                        _last_recvd_msg.travel_epoch_ms = epoch_now();
-                    }
-                    else
-                    {
+                    } else {
                        // why are we getting empty or null data ? (server is sending data regardless?)
                    }
                    _fire_dirty = true;
@@ -160,36 +140,29 @@ namespace bts
            }
        }

-        private void _publish_next_packet()
-        {
-            if (_last_sent_msg == null)
-            {
-                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
-                {
+        private void _publish_next_packet() {
+            if (_last_sent_msg == null) {
+                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
                    guid = Guid.NewGuid().ToString()
                };
-            }
-            else
-            {
+            } else {
                _last_sent_msg.msg_id++;
                _last_sent_msg.guid = Guid.NewGuid().ToString();
                _last_sent_msg.travel_epoch_ms = epoch_now();
            }
            var ser_bytes = _last_sent_msg.serialize();
            _n_pubs++;
-            _pub_core.publish("Blob", _tag_name, ser_bytes);
+            _pub_core.publish(String.Empty, _tag_name, ser_bytes);
            _fire_dirty = true;
        }

-        private void _fire_dp()
-        {
+        private void _fire_dp() {
            object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
            object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
            object echo_time = DBNull.Value;
            object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
            object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
-            if (_last_sent_msg != null && _last_recvd_msg != null)
-            {
+            if (_last_sent_msg != null && _last_recvd_msg != null) {
                echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
            }
            var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
@@ -201,25 +174,21 @@ namespace bts
            myFireEvent(dp);
        }

-        public void doWork()
-        {
+        public void doWork() {
            if (DateTime.Now - _last_publish_time > _publish_period)
-                if (_n_pubs <= _max_pubs)
-                {
+                if (_n_pubs <= _max_pubs) {
                    _publish_next_packet();
                    _last_publish_time = DateTime.Now;
                }
            // publish 
            // TODO: send id
-            if (_fire_dirty)
-            {
+            if (_fire_dirty) {
                _fire_dirty = false;
                _fire_dp();
            }
        }

-        public static long epoch_now()
-        {
+        public static long epoch_now() {
            return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
        }

@@ -8,23 +8,19 @@ using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
 using bts.api;
+using Bts.PubSub.Routing;

-namespace bts
-{
+namespace bts {
   [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker
-   {
-      public class PayloadMessage
-      {
+   public class PubsubEchoTestStream : ViewStream, IWorker {
+      public class PayloadMessage {
         public readonly string msg_id;
         public readonly Int64 timestamp = 0;
         public readonly byte[] payload = null;

-         public static byte[] create_data(int payload_size, byte payload_byte)
-         {
+         public static byte[] create_data(int payload_size, byte payload_byte) {
            var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++)
-            {
+            for (int i = 0; i < payload_size; i++) {
               rv[i] = payload_byte;
            }
            return rv;
@@ -40,16 +36,14 @@ namespace bts
         //       payload[i] = payload_byte;
         //    }
         // }
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
-         {
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
            msg_id = guid_;
            timestamp = timestamp_;
            payload = payload_;
         }

         private PayloadMessage() { }
-         public byte[] serialize()
-         {
+         public byte[] serialize() {
            var str = new MemoryStream();
            BinaryWriter bw = new BinaryWriter(str);
            bw.Write(msg_id);
@@ -61,8 +55,7 @@ namespace bts
            return bytes;
         }

-         public static PayloadMessage deserialize(byte[] ser_bytes)
-         {
+         public static PayloadMessage deserialize(byte[] ser_bytes) {
            var str = new MemoryStream(ser_bytes);
            BinaryReader br = new BinaryReader(str);
            string guid = br.ReadString();
@@ -88,17 +81,17 @@ namespace bts
      private readonly string _tag_name;
      private readonly int _payload_sz;
      private readonly IPubSubRouter _router;
-      private enum IOState
-      {
+      private enum IOState {
         SEND = 0,
         RECV = 1
      };
      IOState _state = IOState.SEND;

-      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
-      {
-         var route = new PubSubRoute(server_url, ".*");
-         _router = new PubSubRouter(route);
+      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+         _router = new PubSubRouter(new PubSubRouterConfig() {
+            PublisherRoutes = new() { new(server_url, ".*") },
+            SubscriberRoutes = new() { new(server_url, ".*") },
+         });
         _vsd = vsd;
         _payload_sz = payload_size;
         var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -117,8 +110,7 @@ namespace bts
      public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
      : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }

-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
         _pub_core = new PublisherCore(_router);
         _sub_core = new SubscriberCore(_router);
         _sub_core.subscribe(_tag_name);
@@ -126,14 +118,12 @@ namespace bts

      }

-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
         _vsd.removeWorker(this);
         _sub_core.unsubscribe(_tag_name);
      }

-      private void _fire_dp_after_send()
-      {
+      private void _fire_dp_after_send() {
         // as soon as the data is published, its reflected in the stream state
         Log.assert(_last_sent_msg != null);
         BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
@@ -143,8 +133,7 @@ namespace bts
         myFireEvent(dp_out);
      }

-      private void _fire_dp_after_recv()
-      {
+      private void _fire_dp_after_recv() {
         Log.assert(_last_recvd_msg != null);
         // update the last publish row
         int id = _table.Count - 1;
@@ -158,13 +147,11 @@ namespace bts
         myFireEvent(dp_out);
      }

-      private string _get_next_id()
-      {
+      private string _get_next_id() {
         return _table.Count.ToString();
      }

-      private void _publish_next_packet()
-      {
+      private void _publish_next_packet() {
         var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
         _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
         var ser_bytes = _last_sent_msg.serialize();
@@ -173,25 +160,19 @@ namespace bts
         _last_publish_time = DateTime.Now;
      }

-      public void doWork()
-      {
-         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
-         {
-            foreach (var u in updates)
-            {
+      public void doWork() {
+         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) => {
+            foreach (var u in updates) {
               var recv_msg = PayloadMessage.deserialize(u.data);
-               if (recv_msg.msg_id == _last_sent_msg.msg_id)
-               {
+               if (recv_msg.msg_id == _last_sent_msg.msg_id) {
                  return recv_msg;
               }
            }
            return null;
         };
-         switch (_state)
-         {
+         switch (_state) {
            case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period)
-               {
+               if (DateTime.Now - _last_publish_time >= _publish_period) {
                  _publish_next_packet();
                  _last_publish_time = DateTime.Now;
                  _fire_dp_after_send();
@@ -203,8 +184,7 @@ namespace bts
               var updates = _sub_core.get_updates(TimeSpan.Zero);
               updates = updates.Where(u => u.tag == _tag_name);
               var recv_msg = find_packet(updates);
-               if (recv_msg != null)
-               {
+               if (recv_msg != null) {
                  _last_recvd_msg = recv_msg;
                  _fire_dp_after_recv();
                  _state = IOState.SEND;
@@ -214,10 +194,9 @@ namespace bts
         }
      }

-      public static long epoch_now()
-      {
+      public static long epoch_now() {
         return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
      }

   }
-}
+}
@@ -95,7 +95,7 @@ namespace BCG {
            sub_msg.serialize_into_str(str);
            str.Position = 0;
            bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
-            sub_msg_.consruct_deserialize(str);
+            sub_msg_.construct_deserialize(str);
            bool res = sub_msg.Equals(sub_msg_);
            return new TestResult {
                test_name = "SubscribeMsg",
@@ -117,7 +117,7 @@ namespace BCG {
            pub_msg.serialize_into_str(str);
            str.Position = 0;
            var pub_msg_ = new bts.PublishMsg();
-            pub_msg_.consruct_deserialize(str);
+            pub_msg_.construct_deserialize(str);
            bool res = pub_msg.Equals(pub_msg_);
            return new TestResult {
                test_name = "PublishMsg",
@@ -135,7 +135,7 @@ namespace BCG {
            msg.serialize_into_str(str);
            str.Position = 0;
            var msg_ = new bts.PubSubServerMessage();
-            msg_.consruct_deserialize(str);
+            msg_.construct_deserialize(str);
            bool res = msg.Equals(msg_);
            return new TestResult {
                test_name = "PubSubServerMessage",
@@ -156,7 +156,7 @@ namespace BCG {
            msg.serialize_into_str(str);
            str.Position = 0;
            var msg_ = new PubSubClientMessage();
-            msg_.consruct_deserialize(str);
+            msg_.construct_deserialize(str);
            var res = msg.Equals(msg_);
            return new TestResult {
                test_name = "PubSubClientMessage",
@@ -617,14 +617,15 @@ namespace BCG {


        private static TestResult _run_single_tag_data_test_clean() {
+            // var tagHandlerFactory
            // create message
-            var test_name = MethodBase.GetCurrentMethod().Name;
+            var test_name = MethodBase.GetCurrentMethod()?.Name;
            var schema = create_schema_with_all_types();
            var tag = "DS_" + schema.tableName;
            BlueTable table = new BlueTable(schema);
            IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
            IStreamMsgDeserilizer deser = new StreamIODeserializationController();
-            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
+            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType, new PubsubServerStreamTagHandler());
            // create with 100 rows
            int n_rows = 100;
            bool is_succ = true;
@@ -7,45 +7,26 @@ namespace bts {
   using System.Text.Json;
   using System.Text.Json.Nodes;
   [DataStreamUserRespresentation("benzingaNews")]
-   public class BenzingaNewsStream : ViewStream {
+   public class BenzingaNewStream : ViewStream {
      private IBlueTableSchema _schema = null;
      private int _id = 0;
-
-
-      private static ViewStream _pre_process_raw_kvstream(ViewStream mongo_db_str) {
-         ViewStream r = new KeyValueToRelationalStream(mongo_db_str, "doc_id", "key", "value");
-         r = new DateStandardizationStream(r, "created_at", "ISO8601");
-         return r;
-      }
-
-      public BenzingaNewsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
-      : this(new MongoDBStream(connection_string, db_name, collection_name, poll_period), ttl) {
-      }
-
-      // public BenzingaNewsStream()
-      // : this(new MongoDBStream(DataStreamPub connection_string, db_name, collection_name, poll_period), ttl) {
-      // }
-
-      // assumes it gets a raw mongodb_raw_kv_stream with key value pairs
-      public BenzingaNewsStream(ViewStream mongodb_raw_kv_stream, string ttl) {
-         ViewStream output_stream = _pre_process_raw_kvstream(mongodb_raw_kv_stream);
+      // constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example
+      public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) {
         // Hiding body (with project, columns can be hidden, or renamed)
-         output_stream = new ProjectStream(output_stream, "!body");
+         ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body");
         // add column news_source = 'Benzinga', so we can append other news sources
         output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
         // now SuperMapStream is used to map one row into multiple, one per symbol,
         // (SuperMapStream takes 2 lambdas, one to get the enw schema and another to 
         // return an array of rows, per one row)
-         output_stream = new SuperMapStream(output_stream,
+         output_stream = new SuperMapStream(mongodb_raw_stream,

        (IBlueTableSchema s) => {
           // this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
           if (_schema == null) {
              var out_schema = s;
              out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
-              out_schema = BlueDataUtility.remove_columns(out_schema, "body");
-              out_schema = BlueDataUtility.remove_columns(out_schema, "tags");
-              out_schema = BlueDataUtility.add_column(out_schema, "doc_id_internal", typeof(string));
+              out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string));
              out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
              out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
              out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
@@ -63,13 +44,11 @@ namespace bts {
           }
           foreach (JsonNode jnode in json_objs) {
              if (jnode is JsonObject json_object) {
-                 string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
-                 string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
                 MutableBlueRow nr = new MutableBlueRow(_schema, r);
                 nr.setValue("id", _id++);
                 nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
-                 nr.setValue("symbol", symbol);
-                 nr.setValue("exchange", exchange);
+                 nr.setValue("symbol", jnode["symbol"]);
+                 nr.setValue("exchange", jnode["exchange"]);
                 nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
                 expanded_rows.Add(nr);
              }
@@ -7,127 +7,27 @@ using MongoDB.Driver;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Threading.Tasks;
-using System.Threading;

 namespace bts {
-    // [DataStreamUserRespresentation("mongodb_sql")]
-    // public class MongoDBSQLStream : ViewStream, IWorker {
-    //     private ViewStreamDatabase _vsd;
-    //     private readonly string _connection_string, _db_name, _collection_name;
-    //     private MongoClient _client;
-    //     // make this a sorted table
-    //     private BlueTable _table;
-
-
-    //     // Access the database
-    //     private IMongoDatabase _database;
-    //     private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-    //     // mongodb://192.168.1.80:27017
-
-    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-    //     : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
-
-    //     }
-    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-    //         bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-    //                                 $"db_name: {db_name}");
-    //         _connection_string = connection_string;
-    //         _db_name = db_name;
-    //         _collection_name = collection_name;
-    //         _vsd = vsd;
-    //         _client = new MongoClient(_connection_string);
-    //         _database = _client.GetDatabase(_db_name);
-    //     }
-
-    //     protected override void myOnFirstSubscriber() {
-    //         MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
-    //     }
-
-    //     protected override void myOnLastSubscriber() {
-    //         MasterWorker.getInstance(false).removeWorker(this);
-    //     }
-
-    //     public void doWork() {
-    //         _read_mogodb();
-    //     }
-
-    //     private void _read_mogodb() {
-    //         var timer = Stopwatch.StartNew();
-    //         // Access the collection
-    //         IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
-
-    //         // Fetch all documents from the collection
-    //         var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-    //         _table = _table == null ? null : new BlueTable(_table.schema);
-    //         // Iterate through the documents and print values
-    //         foreach (var document in documents.ToEnumerable()) {
-    //             bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-    //             if (_table == null) {
-    //                 _init_table(document);
-    //             }
-    //             _add_row(document);
-    //         }
-    //         timer.Stop();
-    //         Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
-    //         timer.Restart();
-    //         var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-    //         timer.Stop();
-    //         Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
-    //         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-    //             return;
-    //         myFireEvent(dp);
-    //     }
-
-    //     private void _init_table(BsonDocument doc_row) {
-    //         bts.Log.assert(_table == null);
-    //         List<ColumnInfo> cis = new List<ColumnInfo>();
-    //         cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-    //         foreach (var n in doc_row.Names) {
-    //             cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-    //         }
-    //         IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-    //         _table = new BlueTable(schema);
-    //     }
-
-    //     private void _add_row(BsonDocument doc) {
-    //         var key_val = doc["_id"];
-    //         var key = key_val.ToString();
-    //         int id;
-    //         if (!_id_map.TryGetValue(key, out id)) {
-    //             id = _id_map.Count;
-    //             _id_map.Add(key, id);
-    //         }
-    //         MutableBlueRow nr = new MutableBlueRow(_table.schema);
-    //         nr.setValue("id", id);
-    //         foreach (var n in doc.Names) {
-    //             var val = doc[n].ToString();
-    //             nr.setValue(n, val);
-    //         }
-    //         _table.addRow(nr);
-    //     }
-    // }
-
-    [DataStreamUserRespresentation("mongodb")]
-    public class MongoDBStream : ViewStream, IWorker {
+    [DataStreamUserRespresentation("mongodb_sql")]
+    public class MongoDBSQLStream : ViewStream, IWorker {
        private ViewStreamDatabase _vsd;
        private readonly string _connection_string, _db_name, _collection_name;
-        private readonly TimeSpan _poll_period;
        private MongoClient _client;
+        // make this a sorted table
        private BlueTable _table;
-        private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
-        private BlockingQueue _work_queue = new BlockingQueue();
+

        // Access the database
        private IMongoDatabase _database;
        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
        // mongodb://192.168.1.80:27017

-        public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
-        : this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
+        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {

        }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
+        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
                                    $"db_name: {db_name}");
            _connection_string = connection_string;
@@ -135,30 +35,114 @@ namespace bts {
            _collection_name = collection_name;
            _vsd = vsd;
            _client = new MongoClient(_connection_string);
-            _poll_period = poll_period;
            _database = _client.GetDatabase(_db_name);
        }

        protected override void myOnFirstSubscriber() {
-            _vsd.addWorker(this);
-            //MasterWorker.getInstance(false).addWorker(this, _poll_period);
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
        }

        protected override void myOnLastSubscriber() {
-            _vsd.removeWorker(this);
-            //MasterWorker.getInstance(false).removeWorker(this);
+            MasterWorker.getInstance(false).removeWorker(this);
        }

-        private DateTime _last_poll_time = DateTime.MinValue;
        public void doWork() {
-            var now = DateTime.Now;
-            if (now - _last_poll_time > _poll_period) {
-                ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
-                _last_poll_time = DateTime.Now;
+            _read_mogodb();
+        }
+
+        private void _read_mogodb() {
+            var timer = Stopwatch.StartNew();
+            // Access the collection
+            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+
+            // Fetch all documents from the collection
+            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+            _table = _table == null ? null : new BlueTable(_table.schema);
+            // Iterate through the documents and print values
+            foreach (var document in documents.ToEnumerable()) {
+                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+                if (_table == null) {
+                    _init_table(document);
+                }
+                _add_row(document);
            }
-            var dp = _table_queue.dequeue(0);
-            if (dp != null)
-                myFireEvent(dp);
+            timer.Stop();
+            Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
+            timer.Restart();
+            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+            timer.Stop();
+            Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
+            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+                return;
+            myFireEvent(dp);
+        }
+
+        private void _init_table(BsonDocument doc_row) {
+            bts.Log.assert(_table == null);
+            List<ColumnInfo> cis = new List<ColumnInfo>();
+            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+            foreach (var n in doc_row.Names) {
+                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+            }
+            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+            _table = new BlueTable(schema);
+        }
+
+        private void _add_row(BsonDocument doc) {
+            var key_val = doc["_id"];
+            var key = key_val.ToString();
+            int id;
+            if (!_id_map.TryGetValue(key, out id)) {
+                id = _id_map.Count;
+                _id_map.Add(key, id);
+            }
+            MutableBlueRow nr = new MutableBlueRow(_table.schema);
+            nr.setValue("id", id);
+            foreach (var n in doc.Names) {
+                var val = doc[n].ToString();
+                nr.setValue(n, val);
+            }
+            _table.addRow(nr);
+        }
+    }
+
+    [DataStreamUserRespresentation("mongodb")]
+    public class MongoDBStream : ViewStream, IWorker {
+        private ViewStreamDatabase _vsd;
+        private readonly string _connection_string, _db_name, _collection_name;
+        private MongoClient _client;
+        private BlueTable _table;
+
+        // Access the database
+        private IMongoDatabase _database;
+        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+        // mongodb://192.168.1.80:27017
+
+        public MongoDBStream(string connection_string, string db_name, string collection_name)
+        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+
+        }
+        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+                                    $"db_name: {db_name}");
+            _connection_string = connection_string;
+            _db_name = db_name;
+            _collection_name = collection_name;
+            _vsd = vsd;
+            _client = new MongoClient(_connection_string);
+            _database = _client.GetDatabase(_db_name);
+        }
+
+        protected override void myOnFirstSubscriber() {
+            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+        }
+
+        protected override void myOnLastSubscriber() {
+            MasterWorker.getInstance(false).removeWorker(this);
+        }
+
+        public void doWork() {
+            _read_mogodb();
        }

        private void _read_mogodb() {
@@ -182,11 +166,10 @@ namespace bts {
            timer.Restart();
            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
+            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
                return;
-            _table_queue.enqueue(dp);
-            //myFireEvent(dp);
+            myFireEvent(dp);
        }

        private void _init_table(BsonDocument doc_row) {
@@ -225,46 +208,46 @@ namespace bts {
        }
    }

-    // [DataStreamUserRespresentation("mongodb_flat")]
-    // public class MongoDBFlatStream : ViewStream {
-    //     public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
-    //     :
-    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
+    [DataStreamUserRespresentation("mongodb_flat")]
+    public class MongoDBFlatStream : ViewStream {
+        public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
+        :
+        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }

-    //     public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
-    //         ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-    //         rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
-    //         rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
-    //         //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
-    //         mySetRealSource(rs);
-    //     }
-    // }
+        public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
+            ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+            rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
+            rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
+            //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
+            mySetRealSource(rs);
+        }
+    }



-    // [DataStreamUserRespresentation("newsreplay")]
-    // public class NewsReplayStream : ViewStream {
-    //     public NewsReplayStream(string connection_string, string db_name, string collection_name,
-    //                             int buffer_sz, string fire_period) :
-    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
+    [DataStreamUserRespresentation("newsreplay")]
+    public class NewsReplayStream : ViewStream {
+        public NewsReplayStream(string connection_string, string db_name, string collection_name,
+                                int buffer_sz, string fire_period) :
+        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }

-    //     public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
-    //                             int buffer_sz, TimeSpan fire_period) {
-    //         ViewStream r;
+        public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
+                                int buffer_sz, TimeSpan fire_period) {
+            ViewStream r;

-    //         r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-    //         r = new KeyValueToRelationalStream(r,
-    //                                             "doc_id",
-    //                                             "key",
-    //                                             "value");
-    //         r = new DateStandardizationStream(r,
-    //                                            "created_at",
-    //                                            "ISO8601");
-    //         r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
+            r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+            r = new KeyValueToRelationalStream(r,
+                                                "doc_id",
+                                                "key",
+                                                "value");
+            r = new DateStandardizationStream(r,
+                                               "created_at",
+                                               "ISO8601");
+            r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);

-    //         mySetRealSource(r);
-    //     }
-    // }
+            mySetRealSource(r);
+        }
+    }


 }
@@ -6,7 +6,6 @@ namespace bts {
   using System;
   using System.Collections;
   using System.Collections.Generic;
-   using System.Linq;

   public class ProjectData {
      // schema from the base stream
@@ -148,23 +147,12 @@ namespace bts {
               mapped_to_orig_map.Add(mapped_col, orig_col);
               orig_to_mapped.Add(orig_col, mapped_col);
            } else {
-               if (input_schema.columnNames.Contains(p_tok)) {
-                  string orig_col = p_tok;
-                  string mapped_col = orig_col;
-                  orig_to_mapped.Add(orig_col, mapped_col);
-               }
+               Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
            }
         }
         // make the new schema
         List<ColumnInfo> new_cols = new List<ColumnInfo>();
         foreach (ColumnInfo ci in input_schema.columnInfo) {
-            if (ci.colName == "id") {
-               // if there's a hidden col, other columns will show by default, but if cols are being
-               // projected epxlicitely, than others wil be hidden
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
-               continue;
-            }
            if (removed_cols.Contains(ci.colName)) {
               continue;
            } else if (orig_to_mapped.ContainsKey(ci.colName)) {
@@ -172,12 +160,8 @@ namespace bts {
               ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
               new_cols.Add(nc);
            } else {
-               if (ci.colName == "id" || removed_cols.Count > 0) {
-                  // if there's a hidden col, other columns will show by default, but if cols are being
-                  // projected epxlicitely, than others wil be hidden
-                  ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-                  new_cols.Add(nc);
-               }
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
            }
         }
         IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
@@ -1,3 +1,5 @@
+using Bts.DataStreamCore;
+
 using System;

 namespace BCG {
@@ -7,14 +9,15 @@ namespace BCG {
   /// </summary>
 	public class CircularPublisher {
      private CircularRowCollection myBuffer;
-      private PubTableBroker myBroker;
+      private readonly IPubTableBroker myBroker;
      private int myBufferSize;
-      public CircularPublisher(int bufferSize) {
+      public CircularPublisher(int bufferSize, IPubTableBroker broker) {
         myBufferSize = bufferSize;
+         myBroker = broker;
      }

-      public CircularPublisher(int bufferSize, IBlueRowCollection rows)
-         : this(bufferSize) {
+      public CircularPublisher(int bufferSize, IPubTableBroker broker, IBlueRowCollection rows)
+         : this(bufferSize, broker) {
         publishRows(rows);
      }

@@ -23,7 +26,6 @@ namespace BCG {
            myBuffer = new CircularRowCollection(
               new BlueTableSchema(nr.schema.tableName, nr.schema.columnInfo),
               myBufferSize);
-            myBroker = new PubTableBroker();
            myBroker.create_table(myBuffer);
         }
         IBlueRow br = myBuffer.addRowWithReturn(nr);
@@ -38,7 +40,6 @@ namespace BCG {
               new BlueTableSchema(BlueDataUtility.extractRow(nrs, 0).schema.tableName,
               BlueDataUtility.extractRow(nrs, 0).schema.columnInfo),
               myBufferSize);
-            myBroker = new PubTableBroker();
            myBroker.create_table(myBuffer);
         }
         BlueTable deletes = new BlueTable(myBuffer.schema);
@@ -4,10 +4,12 @@ using System.Data;

 using BCG.Utilities;

+using Bts.DataStreamCore;
+
 namespace BCG {
   public class ViewPublisher : IDisposable {
      protected ViewPublisher() { }
-      private readonly PubTableBroker _broker;
+      private readonly IPubTableBroker _broker;
      protected readonly ViewStream _stream;
      private int _n_updates = 0;

@@ -15,7 +17,7 @@ namespace BCG {
      /// in the schema (be careful with that because this
      /// could lead to double publishes if the schema name
      /// is not uniquely chosen)
-      public ViewPublisher(string name, PubTableBroker broker, ViewStream vs) {
+      public ViewPublisher(string name, IPubTableBroker broker, ViewStream vs) {
         _stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
         _broker = broker;
         _stream.onChange += new DataStream.ChangeDelegate(_handle_update);
@@ -0,0 +1,11 @@
+namespace Bts.DataStream.Services;
+
+using BCG;
+using BCG.Utilities.General;
+
+using Microsoft.Extensions.DependencyInjection;
+
+public static class DataStreamServices {
+   public static IServiceCollection AddRemoteQueryManager(this IServiceCollection services) =>
+      services.AddSingleton<IWorker, RemoteQueryManager>();
+}
@@ -1783,18 +1783,6 @@ namespace BCG {
         #endregion
      }

-      public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
-         if (rows == null || rows.Count() == 0)
-            return null;
-         IBlueRow max_row = rows.ElementAt(0);
-         foreach (IBlueRow r in rows) {
-            int c = compare_rows(r, max_row, cols_to_compare);
-            if (c > 0)
-               max_row = r;
-         }
-         return max_row;
-      }
-
      public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
         foreach (KeyValuePair<string, bool> p in sort_map) {
            int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
@@ -1,4 +1,5 @@
 using System;
+using System.Text.Json.Serialization;

 namespace BCG {
   /// <summary>
@@ -56,9 +57,16 @@ namespace BCG {
      private Type myColType;
      private object myColValue;

+      [JsonPropertyName("colNo")]
      public int colNo { get { return myColNo; } }
+
+      [JsonPropertyName("colName")]
      public string colName { get { return myColName; } }
+
+      [JsonPropertyName("colType")]
      public Type colType { get { return myColType; } }
+
+      [JsonIgnore]
      public object colValue {get{return myColValue;}}
   }
 }
@@ -0,0 +1,11 @@
+namespace Bts.DataStreamCore;
+
+public enum DataType {
+   Null,
+   String,
+   Float32,
+   Float64,
+   Int32,
+   Int64,
+   DateTime,
+}
@@ -0,0 +1,61 @@
+#nullable enable
+using BCG;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class BlueRowJsonConverter : JsonConverter<IBlueRow> {
+   IBlueTableSchema? schema = null;
+
+   public override IBlueRow Read(
+      ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (schema is null) {
+         throw new InvalidOperationException("Cannot read row from json because schema is null");
+      }
+      var mbr = new MutableBlueRow(schema);
+      int colNum = 0;
+      switch (reader.TokenType) {
+         case JsonTokenType.StartArray:
+            while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
+               mbr.setValue(colNum++, readCellValue(ref reader));
+               colNum++;
+            }
+            break;
+         case JsonTokenType.StartObject:
+            while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
+               if (reader.TokenType != JsonTokenType.PropertyName) {
+                  throw new InvalidOperationException(
+                     "Expected 'PropertyName' immediately after 'StartObject' token in row json");
+               }
+               var colName = reader.GetString();
+               if (!reader.Read()) {
+                  throw new InvalidOperationException(
+                     "Failed to read token immediately after 'PropertyName' token in row json");
+               }
+               mbr.setValue(colName, readCellValue(ref reader));
+            }
+            break;
+      }
+      if (mbr.Count != schema.columnInfo.Length) {
+         throw new InvalidOperationException(
+            $"Row json array length '{mbr.Count}' does not match the schema's " +
+            $"column length '{schema.columnInfo.Length}'.");
+      }
+      return mbr;
+
+      static object? readCellValue(ref Utf8JsonReader reader) =>
+         reader.TokenType switch {
+            JsonTokenType.String => reader.GetString(),
+            JsonTokenType.Number => reader.GetDouble(),
+            JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
+            _ => throw new InvalidOperationException("Failed to read cell value"),
+         };
+   }
+
+   public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
+      throw new NotImplementedException();
+   }
+}
@@ -0,0 +1,138 @@
+#nullable enable
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+public class DataPacketJsonConverter : JsonConverter<DataPacket> {
+   /// <summary>
+   /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
+   /// default, only incremental changes are included ('added' and 'deleted').
+   /// </summary>
+   public bool IncludeCurrent { get; set; }
+
+   /// <summary>
+   /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
+   /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
+   /// </summary>
+   public bool UseObjectRows { get; set; }
+
+   public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      if (!reader.ReadObject()) {
+         return null;
+      }
+      if (!reader.ReadPropertyName(out var prop)) {
+         return null;
+      }
+      if (!reader.Read()) {
+         return null;
+      }
+      if (prop == "added") {
+         var added = new List<MutableBlueRow>();
+         if (reader.TokenType != JsonTokenType.StartArray) {
+            return null;
+         }
+         while (true) {
+            if (!reader.Read()) {
+               break;
+            }
+            if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) {
+               return null;
+            }
+            switch (reader.TokenType) {
+               case JsonTokenType.String:
+                  // DAMNIT! Need the schema.
+                  break;
+            }
+         }
+      }
+      return null;
+   }
+
+   public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
+      using var _ = writer.WriteObject();
+      using (writer.WriteArray("columns")) {
+         foreach (var col in value.schema.columnInfo) {
+            using (writer.WriteObject()) {
+               writer.WriteString("name", col.colName);
+               writer.WriteNumber("index", col.colNo);
+               // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
+               writer.WriteString("type", col.type_code().ToString());
+            }
+         }
+      }
+
+      var columns = value.schema.columnInfo.Select(c => (Column: c, Write: GetCellValueWriter(c))).ToList();
+      WriteRows("added", value.addedRows);
+      WriteRows("deleted", value.deletedRows);
+      if (IncludeCurrent) {
+         WriteRows("all", value.newRows);
+      }
+
+      void WriteRows(string name, IBlueRowCollection rows) {
+         using (writer.WriteArray(name)) {
+            foreach (IBlueRow row in rows) {
+               WriteRow(row);
+            }
+         }
+      }
+
+      void WriteRow(IBlueRow row) {
+         using var rowCase = UseObjectRows ? writer.WriteObject() : writer.WriteArray();
+         for (int i = 0; i < columns.Count; i++) {
+            if (UseObjectRows) {
+               writer.WritePropertyName(columns[i].Column.colName);
+            }
+            columns[i].Write(row[i]);
+         }
+      }
+
+      Action<object?> GetCellValueWriter(ColumnInfo colInfo) {
+         var t = colInfo.colType;
+         if (t == typeof(int)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((int)v));
+         }
+         if (t == typeof(uint)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((uint)v));
+         }
+         if (t == typeof(long)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((long)v));
+         }
+         if (t == typeof(ulong)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
+         }
+         if (t == typeof(short)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((short)v));
+         }
+         if (t == typeof(ushort)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
+         }
+         if (t == typeof(double)) {
+            return NullCheckWrapper(v => JsonSerializer.Serialize(writer, (double)v, options));
+         }
+         if (t == typeof(float)) {
+            return NullCheckWrapper(v => writer.WriteNumberValue((float)v));
+         }
+         if (t == typeof(bool)) {
+            return NullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
+         }
+         return NullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
+
+         Action<object?> NullCheckWrapper(Action<object> writeNonNull) =>
+            v => {
+               if (v is null) {
+                  writer.WriteNullValue();
+               } else {
+                  writeNonNull(v);
+               }
+            };
+      }
+   }
+}
@@ -0,0 +1,49 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.IO;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+/// <summary>
+/// This class seriailzes DataPackets to json strings. A common buffer is used to reduce memory
+/// allocations on repeated calls to <see cref="Serialize"/>. For that reason, this class is NOT
+/// thread safe.
+/// </summary>
+public class DataPacketJsonSerializer {
+   readonly MemoryStream buffer = new();
+   readonly DataPacketJsonConverter converter;
+
+   public DataPacketJsonSerializer(DataPacketJsonConverter? converter = null) =>
+      this.converter = converter ?? new() {
+         IncludeCurrent = true,
+         UseObjectRows = true,
+      };
+
+   public string? Serialize(DataPacket dp) {
+      buffer.Position = 0;
+      buffer.SetLength(0);
+      using (var jsonWriter = new Utf8JsonWriter(buffer)) {
+         var opts = new JsonSerializerOptions() {
+            NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+         };
+         converter.Write(jsonWriter, dp, opts);
+      }
+      var bytes = buffer.GetBuffer();
+      var span = new ArraySegment<byte>(bytes, 0, (int)buffer.Length);
+      return Encoding.UTF8.GetString(span);
+   }
+
+   public void Serialize(DataPacket dp, Utf8JsonWriter jsonWriter) {
+      buffer.Position = 0;
+      buffer.SetLength(0);
+      var opts = new JsonSerializerOptions() {
+         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+      };
+      converter.Write(jsonWriter, dp, opts);
+   }
+}
@@ -8,6 +8,7 @@

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
  </ItemGroup>

  <ItemGroup>
@@ -0,0 +1,8 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+public interface IPubTableBroker {
+   void create_table(IBlueRowCollection table);
+   void make_batch_changes(IBlueRowCollection new_rows, IBlueRowCollection del_rows, IBlueRowCollection added_rows);
+}
@@ -8,12 +8,14 @@ using BCG.Utilities;

 using bts;

+using Bts.DataStreamCore;
+
 namespace BCG {
   /// <summary>
   /// This class will be used by the clients to publish tabular data to the pubsub
   /// </summary>
-   public class PubTableBroker {
-      protected PublisherCore _publisher_core;
+   public class PubTableBroker : IPubTableBroker {
+      protected IPublisherCore _publisher_core;
      protected IDictionary<string, SinglePubTableBroker> _publisher_map =
         new Dictionary<string, SinglePubTableBroker>();

@@ -22,7 +24,7 @@ namespace BCG {
      public PubTableBroker()
         : this(new PublisherCore()) { }

-      public PubTableBroker(PublisherCore publisher) {
+      public PubTableBroker(IPublisherCore publisher) {
         _publisher_core = publisher;
         Log.Gen.info("PubTableBroker constructed");
      }
@@ -14,9 +14,9 @@ namespace BCG {
   public class SinglePubTableBroker {
      private readonly string _tag;
      protected IStreamIOMsgSerializer _serializer;
-      protected PublisherCore _publisher_core;
+      protected IPublisherCore _publisher_core;

-      public SinglePubTableBroker(string tag, PublisherCore publisher_core) {
+      public SinglePubTableBroker(string tag, IPublisherCore publisher_core) {
         bts.Log.Gen.info($"SinglePubTableBroker.cons created for tag: {tag}");
         _tag = tag;
         _publisher_core = publisher_core;
@@ -51,4 +51,4 @@ namespace BCG {
      }

   }
-}
+}
@@ -13,6 +13,9 @@ namespace bts {
      public PubsubServerGenericDataHandler() {
         _bw = new BinaryWriter(_str);
      }
+
+      public string TagType => "*";
+
      public void update(string tag, byte[] msg) {
         _str.Position = 0;
         _bw.Write(msg);
@@ -25,9 +28,6 @@ namespace bts {
         Log.Gen.info($"PubsubServerGenericDataHandler.get_content tag: {tag} rv: {rv.Length}");
         return rv;
      }
-      public string[] supportedTagTypes { get; }
-      public void register(InternalPublishDelegate callback) { }
-      public void doWork() { }
   }

   // This class is created by the SingleTagData for stream tag_type. Every pub
@@ -40,6 +40,7 @@ namespace bts {
      // this is where the state of the table is kept.
      BlueTable _state = null;

+      public string TagType => DSTagManager.DataStreamTagType;

      public void update(string tag, byte[] msg) {
         StreamIOMessage str_msg = _deser.deserialize(msg);
@@ -74,15 +75,6 @@ namespace bts {
         Log.Gen.info($"PubsubServerSingleStreamTagHandler.get_content rv: {rv_txt}");
         return rv;
      }
-      public string[] supportedTagTypes {
-         get {
-            return new string[] { "DS" };
-         }
-      }
-      public void register(InternalPublishDelegate callback) {
-
-      }
-      public void doWork() { }
   }

   public class PubsubServerStreamTagHandler : IPubSubServerTagHandler {
@@ -92,6 +84,9 @@ namespace bts {

      public PubsubServerStreamTagHandler() {
      }
+
+      public string TagType => DSTagManager.DataStreamTagType;
+
      public void update(string tag, byte[] msg) {
         BCG.IPubSubServerTagHandler rv = _get_handler(tag);
         rv.update(tag, msg);
@@ -100,14 +95,6 @@ namespace bts {
         BCG.IPubSubServerTagHandler rv = _get_handler(tag);
         return rv.get_content(tag);
      }
-      public string[] supportedTagTypes {
-         get {
-            List<string> rv = new List<string>();
-            foreach (var h in _handler_map.Values)
-               rv.AddRange(h.supportedTagTypes);
-            return rv.ToArray();
-         }
-      }

      private BCG.IPubSubServerTagHandler _get_handler(string tag) {
         PubsubServerSingleStreamTagHandler rv = null;
@@ -117,13 +104,5 @@ namespace bts {
         }
         return rv;
      }
-      public void register(InternalPublishDelegate callback) {
-         foreach (var h in _handler_map.Values)
-            h.register(callback);
-      }
-      public void doWork() {
-         foreach (var h in _handler_map.Values)
-            h.doWork();
-      }
   }
-}
+}
@@ -0,0 +1,21 @@
+namespace Bts.DataStreamCore.Services;
+
+using BCG;
+using BCG.Utilities.General;
+
+using Bts.PubSub.Services;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+
+public static class DataStreamCoreServices {
+   public static IServiceCollection AddPubTableBroker(this IServiceCollection services) =>
+      services.AddSingleton<IPubTableBroker, PubTableBroker>();
+
+   public static IServiceCollection AddViewStreamDatabase(this IServiceCollection services, IConfiguration configurationSection) =>
+      services.Configure<WorkerServiceConfig>(configurationSection)
+         .AddSingleton<IWorker, ViewStreamDatabase>(sp => {
+            ViewStreamDatabase.SetGlobalInstance(new ViewStreamDatabase(sp.GetRequiredService<ISubscriberCore>()));
+            return ViewStreamDatabase.instance;
+         });
+}
@@ -9,6 +9,7 @@ using System.Threading;
 using mbts;
 using System.Linq;
 using System.Diagnostics;
+using bts;

 namespace BCG {
   public class ViewStreamDatabase : IWorker, IDisposable {
@@ -230,6 +231,11 @@ namespace BCG {
         }
      }

+      internal static void SetGlobalInstance(ViewStreamDatabase vsd) {
+         Log.isnull(mySelf);
+         mySelf = vsd;
+      }
+
      private List<PendingAddRemoveWorker> _pending_worker_ops = new List<PendingAddRemoveWorker>();
      private bool _is_worker_call_in_progress = false;
      class PendingAddRemoveWorker {
@@ -9,15 +9,7 @@
 Letter                   [a-zA-Z]
 BooleanConstant          'true'|'false'
 CSStringConstant         \"([^\"\\]*|\\.)*\"
-YearInt                  [0-9]{4}
-MonthInt                 [0-9]{1,2}
-DayInt                   [0-9]{1,2}
-/**
- * DateTimeConstant supports American (mm/dd/yyyy) and European (yyyy/mm/dd)
- * formats. Month and day can be 1 or 2 digits. It does not validate that the
- * numbers represent an actual date.
- */
-DateTimeConstant         ({MonthInt}\/{DayInt}\/{YearInt}|{YearInt}\/{MonthInt}\/{DayInt})
+DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
 IntegerConstant          -?[0-9]+
 NullConstant             'null'
 RealConstant             -?[0-9]+\.[0-9]+
@@ -1,129 +1,120 @@
 {
-  "user": "bts",
-  "sparkd": "http://192.168.1.80:8000/config",
-  "pubsub": {
-    "milliseconds_between_sub_info_updates": 1000,
-    "milliseconds_between_time_slice_updates": 5000,
-    "milliseconds_between_no_buffer_space_retries": 100,
-    "retries_on_no_buffer_space": 1,
-    "do_it_all_seconds_between_logs": 0,
-    "heartbeat_interval": 15000,
-    "missing_heartbeats_before_gc": 50,
-    "use_managed_sockets": true,
-    "state_update_period": 5000,
-    "statistics_update_period": 5000,
-    "disposed_connection_check_period": 1000,
-    "message_processing_warning_threshold": 1000,
-    "server_port": 12345,
-    "publisher_routes": [
-      {
-        "name": "default",
-        "server_address": "bts-cloud://127.0.0.1:12345",
-        "tag": ".*"
-      }
-    ],
-    "subscriber_routes": [
-      {
-        "name": "default",
-        "server_address": "bts-cloud://127.0.0.1:12345",
-        "tag": ".*"
-      }
-    ],
-    "max_packet_size": 500000000,
-    "blue_comm_must_die": true,
-    "crushable_tags": [],
-    "tag_type_to_assembly_map": [
-      {
-        "handler_class": "BCG.PubSubServerTagHandler",
-        "assembly": "DataStreamCore.dll"
-      }
-    ],
-    "server_queue": "PubSubQueue",
-    "port_range": {
-      "first": 30400,
-      "last": 31700
+  "Sparkd": {
+    "Config": {
+      "ConfigServer": "http://localhost:8000/config"
    },
-    "bind_to_random_port": false,
-    "debug_tags": "^NOTAGS$",
-    "latency_report_threshold_ms": 1500,
-    "use_substations": false,
-    "use_dynamic_clients": false,
-    "is_substation": false,
-    "internally_publish_sub_info": true,
-    "use_managed_clients_in_native_code": false,
-    "compressed_tags": "^NOTAGS$",
-    "tags_to_report_compression_ratio": "^NOTAGS$",
-    "disable_msmq": false,
-    "use_prepublished_server_state": false,
-    "ip_translation": null,
-    "gc_activation_threshold": 0,
-    "wait_period_after_gc": 1000
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    },
+    "EnableAdmin": true,
+    "EnableOrders": true,
+    "EnablePosition": true
  },
-  "publishers": [
-    {
-      "name": "market_data",
-      "streams": [
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "Publisher": {
+    "Publishers": [
+      {
+        "Name": "test_data",
+        "Streams": [
+          {
+            "Name": "milliseconds_1000_100",
+            "Expression": "milliseconds(3000,5)",
+            "IsOnWebsocket": true
+          }
+        ]
+      }
+    ]
+  },
+  "PubSub": {
+    "MillisecondsBetweenSubInfoUpdates": 1000,
+    "MillisecondsBetweenTimeSliceUpdates": 5000,
+    "MillisecondsBetweenNoBufferSpaceRetries": 100,
+    "RetriesOnNoBufferSpace": 1,
+    "DoItAllSecondsBetweenLogs": 0,
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "UseManagedSockets": true,
+    "StateUpdatePeriod": 5000,
+    "StatisticsUpdatePeriod": 5000,
+    "DisposedConnectionCheckPeriod": 1000,
+    "MessageProcessingWarningThreshold": 1000,
+    "ServerPort": 12345,
+    "Routing": {
+      "PublisherRoutes": [
        {
-          "name": "bbo",
-          "expression": "bbo()",
-          "is_on_websocket": true
-        },
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
        {
-          "name": "instruments",
-          "expression": "instruments()",
-          "is_on_websocket": true
-        },
-        {
-          "name": "timesales",
-          "expression": "timesales(10000)",
-          "is_on_websocket": true
-        },
-        {
-          "name": "books",
-          "expression": "books()",
-          "is_on_websocket": true
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
        }
      ]
    },
-    {
-      "name": "test_data",
-      "streams": [
+    "MaxPacketSize": 500000000,
+    "BlueCommMustDie": true,
+    "CrushableTags": [],
+    "TagTypeToAssemblyMap": {
+      "Entries": [
        {
-          "name": "milliseconds_1000_100",
-          "expression": "milliseconds(1000,100)",
-          "is_on_websocket": true
-        },
-        {
-          "name": "test_literal",
-          "expression": "literal(\"x\", \"420.0\")",
-          "is_on_websocket": true
-        },
-        {
-          "name": "test_not_on_websocket",
-          "expression": "literal(\"x\", \"840.0\")",
-          "is_on_websocket": false
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
        }
      ]
-    }
-    // {
-    //   "name": "news_data",
-    //   "streams": [
-    //     {
-    //       "name": "benzinga_news",
-    //       "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
-    //       "is_on_websocket": true
-    //     },
-    //     {
-    //       "name": "benzinga_news_symbols",
-    //       "expression": "benzingaNewsSymbols(benzinga_news)",
-    //       "is_on_websocket": true
-    //     },
-    //     {
-    //       "name": "benzinga_news_symbol_state",
-    //       "expression": "benzingaNewsSymbolState(benzinga_news)",
-    //       "is_on_websocket": true
-    //     }
-    //   ]
-    // }
-  ]
-}
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    },
+    "ServerQueue": "PubSubQueue",
+    "PortRange": {
+      "First": 30400,
+      "Last": 31700
+    },
+    "BindToRandomPort": false,
+    "DebugTags": "^NOTAGS$",
+    "LatencyReportThresholdMs": 1500,
+    "UseSubstations": false,
+    "UseDynamicClients": false,
+    "IsSubstation": false,
+    "InternallyPublishSubInfo": true,
+    "UseManagedClientsInNativeCode": false,
+    "CompressedTags": "^NOTAGS$",
+    "TagsToReportCompressionRatio": "^NOTAGS$",
+    "DisableMsmq": false,
+    "UsePrepublishedServerState": false,
+    "IpTranslation": null,
+    "GcActivationThreshold": 0,
+    "WaitPeriodAfterGc": 1000
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:01"
+  }
+}
@@ -17,14 +17,14 @@
    "publisher_routes": [
      {
        "name": "default",
-        "server_address": "bluto://localhost:12345",
+        "server_address": "bts-cloud://localhost:12345",
        "tag": ".*"
      }
    ],
    "subscriber_routes": [
      {
        "name": "default",
-        "server_address": "bluto://localhost:12345",
+        "server_address": "bts-cloud://localhost:12345",
        "tag": ".*"
      }
    ],
@@ -9,21 +9,6 @@
          "expression": "milliseconds(1000,10)",
          "is_on_websocket": true
        },
-        {
-          "name": "benzinga_news",
-          "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
-          "is_on_websocket": true
-        },
-        {
-          "name": "benzinga_news_symbols",
-          "expression": "benzingaNewsSymbols(benzinga_news)",
-          "is_on_websocket": true
-        },
-        {
-          "name": "benzinga_news_symbol_state",
-          "expression": "benzingaNewsSymbolState(benzinga_news)",
-          "is_on_websocket": true
-        },
        {
          "name": "test_literal",
          "expression": "literal(\"x\", \"420.0\")",
@@ -0,0 +1,6 @@
+namespace Bts.DataStreamPublisher;
+
+public class CsvPublisherConfig {
+   public string? Name { get; set; }
+   public string? CsvDir { get; set; }
+}
@@ -9,6 +9,7 @@

  <ItemGroup>
    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
  </ItemGroup>

  <ItemGroup>
Author	SHA1	Message	Date
John Colvin	ba664b95b8	fix use dep injection in bts.datastreamwebapp.server	2024-07-11 11:06:29 -04:00
John Colvin	d6f370f862	fix Add spark service modules designed for DependencyInjection	2024-07-11 11:05:23 -04:00
John Colvin	079826f4de	Fix mismatched types in PubsubHeartbeatStream schema	2024-07-11 10:59:41 -04:00
John Colvin	2eee3cdaab	Change DataStreamPublisher json fields to match cs class property names verbatim	2024-07-11 10:43:24 -04:00
John Colvin	5e3215ee52	fix: Always call 'handle_internal_pubs' in PubSubMessageReactor main loop	2024-07-11 10:38:30 -04:00
John Colvin	7739cc2f3d	Use Dependency Injection in Bts.DataStreamWebApp.Server	2024-07-11 10:37:29 -04:00
John Colvin	8347a7c126	refactor: Change Bts.DataStreamWebApp.Server framework from net8 -> net7 Although it would build on net8, all other cs code targets net7. Furthermore, VS Code intellisense seems less reliable/crashes frequently when the sln file targets both frameworks).	2024-07-11 10:37:28 -04:00
John Colvin	fbfc960df6	improvement: Make DataStreamSubscription an IHostedService To properly manage its life cycle	2024-07-11 10:37:28 -04:00
John Colvin	451f721030	improvement: Accept CancellationToken, catch exceptions when running ds websocket server	2024-07-11 10:37:28 -04:00
John Colvin	bb992c0410	Fix mismatched types in PubSubServerTagStatsStream schema	2024-07-11 10:37:28 -04:00
John Colvin	db18aebf92	Add DataStreamPublisher service modules designed for Dependency Injection	2024-07-11 10:37:28 -04:00
John Colvin	8127c9c9fe	refactor: Decouple ViewPublisher from IPubTableBroker impl	2024-07-11 10:37:28 -04:00
John Colvin	1825b339b1	Change PubSubServer json fields to match cs class property names verbatim	2024-07-11 10:37:28 -04:00
John Colvin	03ab3fbe90	Get rid of JsonPropertyName attributes in PubSubConfig The Dependency Injection framework (based around dotnet ApplicationBuilder) does not support this attribute, and maps property names verbatim to json fields.	2024-07-11 10:37:28 -04:00
John Colvin	b61bf16a42	Add PubSub service modules designed for Dependency Injection	2024-07-11 10:37:28 -04:00
John Colvin	d969489bf4	refactor: Decouple RemoteQueryManager from IManagedSubscriber, IPubTableBroker impl	2024-07-11 10:37:28 -04:00
John Colvin	9cd4974f49	refactor: Decouple ManagedSubscriber from ISubscriberCore impl	2024-07-11 10:37:28 -04:00
John Colvin	793634c8cb	refactor: Only support 1 tag type in IPubSubServerTagHandler	2024-07-11 10:37:28 -04:00
John Colvin	eae20b6c94	refactor: Decouple PubTableBroker from IPublisherCore impl Reference interface rather than the PublisherCore impl.	2024-07-11 10:37:28 -04:00
John Colvin	2848bb6a5d	improvement: Accept CancellationToken in JsonRpcRequestObservable	2024-07-11 10:37:16 -04:00
John Colvin	687324998b	Add spark service modules designed for Dependency Injection	2024-07-11 10:36:04 -04:00
John Colvin	97cebd62d4	fix: Dispose market data sub in various md streams	2024-07-11 10:36:04 -04:00
John Colvin	492ca0c28f	cleanup: Remove unused pubsub tag handler code * Removed 'IPubSubServerTagHandler.register' - All implementations of this function were empty * Removed 'IPubSubServerTagHandler.supportedTagTypes' - This property wasn't referenced anywhere * Remove IWorker inheritance from IPubSubServerTagHandler - No reason these interfaces have to be tied together	2024-07-10 13:35:18 -04:00
John Colvin	baa5019039	refactor: Rename CloudStreamManager.cs -> RemoteQueryManager.cs This matches the file name	2024-07-10 01:18:21 -04:00
John Colvin	ac2d31e1fc	fix build: Update vcpkg On some vs versions, vcpkg cmake scripts would fail w/ an error saying "Could not locate a complete Visual Studio instance"	2024-07-09 15:16:36 -04:00
John Colvin	9821134bc3	Change pub sub schema prefix 'bluto://' -> 'bts-cloud://' In config json files (not actual code change)	2024-07-09 15:14:30 -04:00
John Colvin	9cf892322d	fix build: Fix prop names after refactoring PubSubRoute * Change 'server_address' -> 'ServerAddress' * Change 'tag_reg_exp' -> 'TagRegExp'	2024-07-09 15:14:30 -04:00
John Colvin	77bf91be1e	build: Add scripts to publish data stream apps	2024-07-09 15:14:30 -04:00
John Colvin	63b064ac79	Fix typo in log msg	2024-07-09 15:14:30 -04:00
John Colvin	b617775c29	Fix net7 -> net8 typo in publish_data_stream_web_app.py The DataStream Web App is on net8	2024-07-09 15:14:30 -04:00
John Colvin	4f7d77420c	Fix build scripts so they propagate the exit code back to the shell I noticed this was broken because launch.json configurations would still start the debugger even when the preLaunchTask (build) failed!	2024-07-09 15:14:30 -04:00
John Colvin	3ba6a91ca7	vscode: Add DataStreamPublisher debug cfg to skip build	2024-07-09 15:14:30 -04:00
John Colvin	0abde9eb7c	build: Use separate cmd to build DataStreamSuite/QueryAnalyzer.sln The DataStreamSuite.sln builds the new webapp now, which will (eventually) replace DSQA. We'll keep it around for now, but with a clean separation from the main build.	2024-07-09 15:14:30 -04:00
John Colvin	1531a57bc5	feat: Add DataStream WebApp Included in DataStreamSuite.sln, but not DataStreamQueryAnalyzer.sln	2024-07-09 15:14:29 -04:00
John Colvin	a62ea64c5e	build: Copy libspark to all dotnet dirs in single cmake target This is cleaner than the previous impl, which used separate build targets to copy libspark to "net7.0", and "net7.0-windows" (dirs). Note: I decided to clean this up now because I'm about to upgrade the new DataStream WebApp to net8.0, and did not want to add yet another build target to copy libspark to "net8.0".	2024-07-09 15:14:29 -04:00
John Colvin	450773f588	fix: Parse ServerAddress as normal string The "JsonValueOrArrayConverter<T>" doesn't work the way I thought it would. When it is associated with a property via the JsonConverter attribute, dotnet json parsing code throws an exception if "T" isn't assignable to the property type.	2024-07-09 15:14:29 -04:00
John Colvin	f09d6badaf	fix build (api server): Change namespace to bts.websocket_core	2024-07-09 15:14:29 -04:00
John Colvin	6b054e0718	Remove docker submodules again	2024-07-09 15:14:29 -04:00
John Colvin	5e2789d16e	Change externals url from ssh to https style So we don't need ssh keys to clone it (the builtin credential manager makes https authentication safe/easy)	2024-07-09 15:14:29 -04:00
John Colvin	9ebdb2e460	cleanup: Fix benign typos in build scripts Change BUILD_SPARK_PY var name to BUILD_PY (because these scripts to actually build 'spark')	2024-07-09 15:14:29 -04:00
John Colvin	d7bf82722b	fix build (warn): Only use Registry on windows	2024-07-09 15:14:29 -04:00
John Colvin	e1526e25a7	cleanup: Fix numerous build warnings in DataStreamSuite Nearly all warnings were due to nullability issues, since the concept of nullable reference types did not exist when the DataStream code was originally written. * PubSubClientMessage - Normalize indentation/whitespace - Fix some typos - Address nullability issues to fix build warnings - Modernize code * Compression related files - ZipEntry - ZipFile - ZipInputStream - TarHeader * RemoteMessage - make 'message' nullable to resolve warnings elsewhere (in calling code) * PubSubSingleTagStateEvent.cs * PubSubMessageReactor * PubSubState * SystemHeartbeatPublisher * SystemStatsPublisher * SingleTagData * ManagedTCPMessageHub * PubsubDeserializationController * PublisherCore * InflaterHuffmanTree * AsyncTCPConnection_APM * ThreadSafeQueue * ManagedSubscriber * SubscriberCore * PubSubServerManagedTCPConnection * AsyncTCPConnection_APM * SuperMapStream * MapStream * TradeBarsStream	2024-07-09 15:14:26 -04:00
John Colvin	089ba9b369	cleanup: Normalize indentation in TradeBarsStream	2024-07-09 15:09:02 -04:00
John Colvin	567976043f	cleanup: Disable nullability warnings * BZip2InputStream * TarInputStream * Inflater * InflaterDynHeader * BZip2OutputStream * DeflaterHuffman * ParallelJoinTest	2024-07-09 15:07:24 -04:00
John Colvin	2fd3382939	cleanup: Modernize PubSubRoute code, json parsing	2024-07-09 15:06:34 -04:00
John Colvin	a96f1c8f17	cleanup: Remove unused DateTime grammar from exp parser	2024-07-09 15:03:30 -04:00
John Colvin	6f623f9a37	Add JsonRpcRequestObservable, AsyncEnumerable	2024-07-09 15:03:30 -04:00
John Colvin	827189cdb4	Add json facilities for DataStream websocket api	2024-07-09 15:03:30 -04:00
John Colvin	123f12ab88	refactor: Add handy WebsocketExtensions * IsSocketStateOpen * IsClosedProperly	2024-07-09 14:59:06 -04:00
John Colvin	f51ba8fb56	feat: Add JsonRpc types for websocket paradigm	2024-07-09 14:59:06 -04:00
John Colvin	2506fb4f2b	cleanup: Remove empty file (clearly added by accident)	2024-07-09 14:59:06 -04:00
John Colvin	f92964114e	fix: Fix improper 'OfType' linq cast Previously, animations were expressed through polymorphism (derived from the base class CellAnimation), but the new impl uses composition. This 'OfType' call was leftover from the polymorphism implementation, an attempt to linq cast elements of base class (CellAnimation) to BackgroundAnimation.	2024-07-09 14:59:06 -04:00
John Colvin	baa0f1430b	editorconfig: Use standard c# capitalization rules * Use 'PascalCase' for public members (instead of 'snake_case') * Use 'camelCase' for private members (instead of '_snake_case')	2024-07-09 14:59:06 -04:00
John Colvin	7104bd787a	refactor: Add WebsocketClient class Basically an ISubject (I/O) wrapper for IWebsocketConnectionGateway	2024-07-09 14:59:06 -04:00
John Colvin	5db92d1f7d	refactor: Change WebsocketServerConfig.port to uint32 It should always be a number between 1-65535	2024-07-09 14:59:06 -04:00
John Colvin	eb0fcf0b84	refactor: Rename WebsocketServerConfig proto pkg 'bts.websocket_core' This matches the namespace used in spark_websocket csproj. Eventually, I'd like to use conventional capitalization (TitleCase namespaces/types, etc.), but let's shelf that until we have enough time/energy to do a more comprehensive refactoring.	2024-07-09 14:59:06 -04:00
John Colvin	a52b187313	refactor: Remove nullability constraint in GetValueOrDefault	2024-07-09 14:59:06 -04:00
John Colvin	669b059eb2	refactor: Use events to avoid circular deps for ws sync/gateway	2024-07-09 14:59:06 -04:00
John Colvin	36ab8492d3	refactor: Let caller specify nullability of T in json parse fn	2024-07-09 14:48:59 -04:00
John Colvin	54f37b2401	Merge remote-tracking branch 'origin/development' into development.ps_linux	2024-07-09 14:46:18 -04:00
				`@@ -0,0 +1 @@`
				<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>