fix use dep injection in bts.datastreamwebapp.server

fix Add spark service modules designed for DependencyInjection
Fix mismatched types in PubsubHeartbeatStream schema
2024-07-11 11:06:29 -04:00 · 2024-07-11 11:05:23 -04:00 · 2024-07-11 10:59:41 -04:00 · 2024-07-11 10:43:24 -04:00 · 2024-07-11 10:38:30 -04:00 · 2024-07-11 10:37:29 -04:00
163 changed files with 10950 additions and 834 deletions
@@ -66,6 +66,7 @@ RUN \

 # Dotnet
 RUN apt-get update && apt-get install -y dotnet7
+RUN apt-get update && apt-get install -y dotnet8

 # Node
 ARG node_major=18
@@ -19,6 +19,14 @@
    "--shm-size",
    "2gb",
    "-p",
+    "5000:5000",
+    "-p",
+    "5001:5001",
+    "-p",
+    "5173:5173",
+    "-p",
+    "5174:5174",
+    "-p",
    "8000:8000",
    "-p",
    "8001:8001",
@@ -47,4 +55,4 @@
      ]
    }
  }
-}
+}
@@ -6,6 +6,35 @@
      "type": "coreclr",
      "request": "attach"
    },
+    {
+      "name": "Bts.DataStreamWebApp.Server",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream WebApp (Debug)",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    {
+      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
    {
      "name": "DataStreamPublisher",
      "type": "coreclr",
@@ -22,7 +51,26 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
      ],
-      "cwd": "build/Debug/bin/net7.0",
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DataStreamPublisher (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "DataStreamPublisher",
+      },
+      "windows": {
+        "program": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -30,7 +78,7 @@
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
+      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
      "program": "",
      "windows": {
        "program": "DataStreamQueryAnalyzer.exe",
@@ -66,6 +114,25 @@
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
+    {
+      "name": "PubSubServer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "PubSubServer",
+      },
+      "windows": {
+        "program": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
    {
      "name": "Spark",
      "type": "coreclr",
@@ -121,6 +121,97 @@
        "color": "terminal.ansiCyan"
      },
    },
+    {
+      "label": "Build DataStreamQueryAnalyzer",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
    {
      "label": "Build Spark",
      "group": "build",
@@ -303,6 +394,22 @@
        "color": "terminal.ansiCyan"
      },
    },
+    {
+      "label": "Publish DataStreamWebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/publish_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/publish_data_stream_web_app.cmd",
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
    {
      "label": "Edit DataStream Config",
      "type": "process",
@@ -353,7 +460,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStream",
+      "dependsOn": "Build DataStreamQueryAnalyzer",
      "type": "process",
      "command": "",
      "windows": {
@@ -373,7 +480,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
      "type": "process",
      "command": "",
      "windows": {
@@ -393,7 +500,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStream (Release)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
      "type": "process",
      "command": "",
      "windows": {
@@ -31,8 +31,7 @@
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -222,6 +221,7 @@
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -258,7 +258,7 @@
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -0,0 +1,20 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.3.1' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
+  <PropertyGroup>
+    <StartupCommand>npm run dev</StartupCommand>
+    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
+    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
+    <!-- Allows the build (or compile) script located on package.json to run on Build -->
+    <ShouldRunBuildScript>true</ShouldRunBuildScript>
+    <!-- Folder where production build objects will be placed -->
+    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
+  </PropertyGroup>
+</Project>
@@ -0,0 +1,8 @@
+# React + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/ico" href="/logo.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>BTS Cloud Streams</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>
@@ -0,0 +1,38 @@
+{
+  "name": "bts.datastreamwebapp.client",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@angular/material": "^18.0.4",
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@fontsource/noto-sans": "^5.0.22",
+    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
+    "@mui/icons-material": "^5.15.20",
+    "@mui/material": "^5.15.20",
+    "@mui/x-data-grid": "^7.7.1",
+    "bulma": "^1.0.1",
+    "react": "^18.3.1",
+    "react-data-table-component": "^7.6.2",
+    "react-dom": "^18.3.1",
+    "react-use-websocket": "^4.8.1",
+    "rpc-websockets": "^7.11.2"
+  },
+  "devDependencies": {
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^8.57.0",
+    "eslint-plugin-react": "^7.34.2",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-react-refresh": "^0.4.7",
+    "vite": "^5.2.13"
+  }
+}
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
@@ -0,0 +1,28 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+h1#main-header {
+  padding: 4px 8px 4px 8px;
+  text-align: center;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}
@@ -0,0 +1,24 @@
+import { createTheme, ThemeProvider, CssBaseline } from '@mui/material';
+import useMediaQuery from '@mui/material/useMediaQuery';
+import QueryControl from './QueryControl';
+import getTheme from './Theme';
+import '@fontsource/noto-sans';
+import './App.css';
+
+const App = () => {
+  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
+  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
+
+  return (
+    <ThemeProvider theme={theme}>
+      <CssBaseline />
+      <h1 id='main-header' style={{
+        color: theme.palette.text.primary,
+        fontFamily: 'Noto Sans',
+      }}>BTS Cloud Streams</h1>
+      <QueryControl websocketUrl="ws://localhost:5000/ws" theme={theme} />
+    </ThemeProvider>
+  );
+};
+
+export default App;
@@ -0,0 +1,32 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+th {
+  font-size: large;  
+  padding: 15px;
+}
+
+td {
+  padding: 15px;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}
@@ -0,0 +1,52 @@
+import PropTypes from 'prop-types';
+import { DataGrid } from '@mui/x-data-grid';
+
+function getGridColDef(col) {
+  return {
+    field: col.name,
+    headerName: col.name,
+  };
+}
+
+export default function DataTable({ columns, rows }) {
+  return (
+    <DataGrid
+      columns={columns.map(getGridColDef)}
+      rows={rows}
+      sx={{
+        borderColor: 'primary.main',
+        backgroundColor: 'primary.dark',
+        color: 'text.primary',
+        '& .MuiDataGrid-columnHeader': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-filler': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-cell': {
+          color: 'text.primary',
+        },
+        '& .MuiDataGrid-cell:hover': {
+          color: 'primary.main',
+        },
+        '& .MuiDataGrid-row': {
+          '--DataGrid-rowBorderColor': 'secondary.dark'
+        }
+      }}
+    />
+  );
+}
+
+DataTable.propTypes = {
+  columns: PropTypes.arrayOf(
+    PropTypes.shape({
+      name: PropTypes.string.isRequired,
+      type: PropTypes.string.isRequired,
+    })
+  ).isRequired,
+  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
+};
@@ -0,0 +1,100 @@
+import PropTypes from 'prop-types';
+import DataTable from './DataTable';
+import QueryInputForm from './QueryInputForm';
+import Container from '@mui/material/Container';
+import React from 'react';
+
+var rpcReqIdCounter = 1;
+
+function subscribeReq(query) {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "subscribe",
+    params: {
+      subscription_id: "1",
+      query: query
+    }
+  };
+}
+
+function unsubscribeReq() {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "unsubscribe",
+    params: {
+      subscription_id: "1",
+    }
+  };
+}
+
+class QueryControl extends React.Component {
+  constructor({ websocketUrl, theme }) {
+    super(websocketUrl);
+    console.log('Connecting to WebSocket:', websocketUrl);
+    this.state = {
+      rows: null,
+      ws: new WebSocket(websocketUrl),
+      columns: null,
+      isRunning: false,
+      theme: theme,
+    };
+    this.state.ws.onopen = () => {
+      console.info('WebSocket open');
+    };
+    this.state.ws.onclose = () => {
+      console.info('WebSocket closed');
+    };
+    this.state.ws.onmessage = e => {
+      console.info(`Received message: ${e.data}`);
+      const msg = JSON.parse(e.data);
+      if (msg.id != null) {
+        let newState = { ...this.state };
+        newState.isRunning = msg.result.status == 'Subscribed';
+        this.setState(newState);
+      } else {
+        let newState = { ...this.state };
+        newState.rows = msg.params.all;
+        console.info(`Received rows: ${newState.rows}`);
+        newState.columns = msg.params.columns;
+        this.setState(newState);
+      }
+    };
+
+    this.start = (query) => {
+      this.state.ws.send(JSON.stringify(subscribeReq(query)));
+    };
+
+    this.stop = () => {
+      this.state.ws.send(JSON.stringify(unsubscribeReq()));
+    };
+  }
+
+  componentWillUnmount() {
+    if (this.state.ws) {
+      console.warn('Closing websocket because component is unmounting');
+      this.state.ws.close();
+    }
+  }
+
+  render() {
+    console.info('Rendering');
+    return (
+      <Container>
+        <QueryInputForm isRunning={this.state.isRunning} onStart={this.start} onStop={this.stop} />
+        {
+          this.state.rows == null ? <p style={{ color: this.state.theme.palette.warning.main }}>No data received</p> :
+            <DataTable columns={this.state.columns} rows={this.state.rows} />
+        }
+      </Container>
+    );
+  }
+}
+
+QueryControl.propTypes = {
+  websocketUrl: PropTypes.string.isRequired,
+  theme: PropTypes.object.isRequired,
+};
+
+export default QueryControl;
@@ -0,0 +1,93 @@
+import { useRef } from 'react';
+import Box from '@mui/material/Box';
+import Button from '@mui/material/Button';
+import Grid from '@mui/material/Grid';
+import Paper from '@mui/material/Paper';
+import PlayArrowIcon from '@mui/icons-material/PlayArrow';
+import PropTypes from 'prop-types';
+import StopIcon from '@mui/icons-material/Stop';
+import TextField from '@mui/material/TextField';
+import Tooltip from '@mui/material/Tooltip';
+
+const QueryInputForm = ({ isRunning, onStart, onStop }) => {
+  let textInput = useRef(null);
+
+  const onClearClick = () => {
+    textInput.current.value = '';
+  };
+
+  const onRunClick = () => {
+    onStart(textInput.current.value);
+  };
+
+  const onStopClick = () => {
+    onStop();
+  };
+
+  const onKeyDown = (e) => {
+    // Detect keyboard shortcuts.
+    //   Ctrl+Enter : Run
+    if (e.ctrlKey && e.key === 'Enter') {
+      onRunClick();
+    }
+  };
+
+  return (
+    <Box sx={{ p: 2 }}>
+      <Paper elevation={3} sx={{ color: 'primary.main' }}>
+        <Box sx={{ p: 2 }}>
+          <TextField
+            fullWidth
+            multiline
+            rows={10}
+            variant="outlined"
+            onKeyDown={onKeyDown}
+            inputRef={textInput}
+            placeholder="Enter Query..."
+          />
+        </Box>
+        <Box sx={{ p: 2, borderTop: 1 }}>
+          <Grid container spacing={2} justifyContent="flex-start">
+            <Grid item xs>
+              <Tooltip title="Clear Query Text">
+                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
+                  Clear
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Run (Ctrl+Enter)">
+                <Button onClick={onRunClick} sx={{
+                  color: `query_start_button.${isRunning ? 'active' : 'inactive'}.color`,
+                  bgcolor: `query_start_button.${isRunning ? 'active' : 'inactive'}.background_color`,
+                  padding: '11px'
+                }}>
+                  <PlayArrowIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Stop">
+                <Button onClick={onStopClick} sx={{
+                  color: `query_stop_button.${isRunning ? 'inactive' : 'active'}.color`,
+                  bgcolor: `query_stop_button.${isRunning ? 'inactive' : 'active'}.background_color`,
+                  padding: '11px'
+                }}>
+                  <StopIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+          </Grid>
+        </Box>
+      </Paper>
+    </Box>
+  );
+};
+
+QueryInputForm.propTypes = {
+  isRunning: PropTypes.bool.isRequired,
+  onStart: PropTypes.func.isRequired,
+  onStop: PropTypes.func.isRequired,
+};
+
+export default QueryInputForm;
@@ -0,0 +1,88 @@
+import { createTheme } from '@mui/material/styles';
+import PropTypes from 'prop-types';
+import '@fontsource/noto-sans';
+import './App.css';
+
+const getLightPalette = () => {
+  return {
+    mode: 'light',
+    primary: {
+      light: '#6573c3',
+      main: '#3f51b5',
+      dark: '#2c387e',
+      contrastText: '#ffffff',
+    },
+    secondary: {
+      light: '#f73378',
+      main: '#f50057',
+      dark: '#ab003c',
+      contrastText: '#ffffff',
+    },
+    text: {
+      primary: '#000000c4',
+      secondary: '#00000086',
+      disabled: '#0000005b',
+      hint: '#0000005b',
+    }
+  };
+};
+
+const getDarkPalette = () => {
+  return {
+    mode: 'dark',
+    primary: {
+      light: '#bcc5ff',
+      main: '#33baf9',
+      dark: '#202334',
+      contrastText: '#78a2ac',
+    },
+    secondary: {
+      light: '#9fd0b8',
+      main: '#5eb289',
+      dark: '#ffffff12',
+      contrastText: '#c8d4f7',
+    },
+    text: {
+      primary: '#eaf8f8',
+      dark: '#eaf8f8a0'
+    },
+    query_start_button: {
+      active: {
+        color: '#33baf9',
+        background_color: '#ffffff40',
+      },
+      inactive: {
+        color: '#ffffff40',
+        background_color: '#ffffff00',
+      }
+    },
+    warning: {
+      main: '#c4823f'
+    },
+    query_stop_button: {
+      active: {
+        color: '#ff4444a0',
+        background_color: '#ffffff40',
+      },
+      inactive: {
+        color: '#ff444450',
+        background_color: '#ffffff00',
+      }
+    },
+  };
+};
+
+const getTheme = ({ mode }) => {
+  return createTheme({
+    typography: {
+      fontFamily: '"Helvetica"'
+    },
+    palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
+  });
+};
+
+export default getTheme;
+
+getTheme.propTypes = {
+  mode: PropTypes.string.isRequired,
+};
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>
@@ -0,0 +1,51 @@
+.dark-high-contrast {
+  --md-sys-color-primary: rgb(251 250 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 0 0);
+  --md-sys-color-primary-container: rgb(173 204 255);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(250 250 255);
+  --md-sys-color-on-secondary: rgb(0 0 0);
+  --md-sys-color-secondary-container: rgb(159 208 255);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(255 249 251);
+  --md-sys-color-on-tertiary: rgb(0 0 0);
+  --md-sys-color-tertiary-container: rgb(222 193 231);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 249 249);
+  --md-sys-color-on-error: rgb(0 0 0);
+  --md-sys-color-error-container: rgb(255 186 177);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(255 255 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(251 250 255);
+  --md-sys-color-outline: rgb(200 202 212);
+  --md-sys-color-outline-variant: rgb(200 202 212);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(0 0 0);
+  --md-sys-color-inverse-primary: rgb(0 42 83);
+  --md-sys-color-primary-fixed: rgb(219 231 255);
+  --md-sys-color-on-primary-fixed: rgb(0 0 0);
+  --md-sys-color-primary-fixed-dim: rgb(173 204 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
+  --md-sys-color-secondary-fixed: rgb(214 233 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 0 0);
+  --md-sys-color-secondary-fixed-dim: rgb(159 208 255);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
+  --md-sys-color-tertiary-fixed: rgb(249 222 255);
+  --md-sys-color-on-tertiary-fixed: rgb(0 0 0);
+  --md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}
@@ -0,0 +1,51 @@
+.dark-medium-contrast {
+  --md-sys-color-primary: rgb(173 204 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 22 49);
+  --md-sys-color-primary-container: rgb(112 146 198);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(159 208 255);
+  --md-sys-color-on-secondary: rgb(0 24 42);
+  --md-sys-color-secondary-container: rgb(100 149 193);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(222 193 231);
+  --md-sys-color-on-tertiary: rgb(33 14 42);
+  --md-sys-color-tertiary-container: rgb(162 136 171);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 186 177);
+  --md-sys-color-on-error: rgb(55 0 1);
+  --md-sys-color-error-container: rgb(255 84 73);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(251 250 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(200 202 212);
+  --md-sys-color-outline: rgb(160 163 171);
+  --md-sys-color-outline-variant: rgb(128 131 139);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(40 42 47);
+  --md-sys-color-inverse-primary: rgb(35 73 120);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 17 40);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 18 34);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(28 9 37);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}
@@ -0,0 +1,51 @@
+.dark {
+  --md-sys-color-primary: rgb(165 200 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 49 94);
+  --md-sys-color-primary-container: rgb(34 72 118);
+  --md-sys-color-on-primary-container: rgb(212 227 255);
+  --md-sys-color-secondary: rgb(154 203 250);
+  --md-sys-color-on-secondary: rgb(0 51 83);
+  --md-sys-color-secondary-container: rgb(12 74 114);
+  --md-sys-color-on-secondary-container: rgb(206 229 255);
+  --md-sys-color-tertiary: rgb(218 189 226);
+  --md-sys-color-on-tertiary: rgb(61 41 70);
+  --md-sys-color-tertiary-container: rgb(85 63 93);
+  --md-sys-color-on-tertiary-container: rgb(247 216 255);
+  --md-sys-color-error: rgb(255 180 171);
+  --md-sys-color-on-error: rgb(105 0 5);
+  --md-sys-color-error-container: rgb(147 0 10);
+  --md-sys-color-on-error-container: rgb(255 218 214);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(225 226 233);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(195 198 207);
+  --md-sys-color-outline: rgb(141 145 153);
+  --md-sys-color-outline-variant: rgb(67 71 78);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(46 48 53);
+  --md-sys-color-inverse-primary: rgb(60 96 144);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}
@@ -0,0 +1,51 @@
+.light-high-contrast {
+  --md-sys-color-primary: rgb(0 34 70);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(29 68 114);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(0 36 61);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(3 70 110);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(46 26 55);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(81 59 89);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(78 0 2);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(140 0 9);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(0 0 0);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(32 36 43);
+  --md-sys-color-outline: rgb(63 67 74);
+  --md-sys-color-outline-variant: rgb(63 67 74);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(255 255 255);
+  --md-sys-color-inverse-primary: rgb(228 236 255);
+  --md-sys-color-primary-fixed: rgb(29 68 114);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(0 45 88);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(3 70 110);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(0 47 77);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(81 59 89);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}
@@ -0,0 +1,51 @@
+.light-medium-contrast {
+  --md-sys-color-primary: rgb(29 68 114);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(83 118 167);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(3 70 110);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(71 121 163);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(81 59 89);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(133 108 141);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(140 0 9);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(218 52 46);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(63 67 74);
+  --md-sys-color-outline: rgb(91 95 103);
+  --md-sys-color-outline-variant: rgb(119 123 131);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(83 118 167);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(58 93 141);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(71 121 163);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(43 96 137);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(133 108 141);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}
@@ -0,0 +1,51 @@
+.light {
+  --md-sys-color-primary: rgb(60 96 144);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(212 227 255);
+  --md-sys-color-on-primary-container: rgb(0 28 58);
+  --md-sys-color-secondary: rgb(46 98 140);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(206 229 255);
+  --md-sys-color-on-secondary-container: rgb(0 29 50);
+  --md-sys-color-tertiary: rgb(110 86 118);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(247 216 255);
+  --md-sys-color-on-tertiary-container: rgb(39 20 48);
+  --md-sys-color-error: rgb(186 26 26);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(255 218 214);
+  --md-sys-color-on-error-container: rgb(65 0 2);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(67 71 78);
+  --md-sys-color-outline: rgb(116 119 127);
+  --md-sys-color-outline-variant: rgb(195 198 207);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}
@@ -0,0 +1,107 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+button#query-button {
+  height: 5em;
+  vertical-align: top;
+}
+
+button#play-button {
+  padding: '10px 20px';
+  font-size: '16px';
+  cursor: 'pointer';
+  background-color: '#28a745';
+  color: '#fff';
+  border: 'none';
+  border-radius: '5px';
+};
+
+button#pause-button {
+  background-color: #ffc107;
+}
+
+button#stop-button {
+  background-color: #dc3545;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+p#footer {
+  float: none;
+  clear: both;
+  vertical-align: bottom;
+}
+
+table#query-output {
+  float: none;
+  clear: both;
+}
+
+textarea#query-input {
+  height: 5em;
+  width: 50em;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}
@@ -0,0 +1,9 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App';
+
+ReactDOM.createRoot(document.getElementById('root')).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+)
@@ -0,0 +1,69 @@
+import { fileURLToPath, URL } from 'node:url';
+import { defineConfig } from 'vite';
+import plugin from '@vitejs/plugin-react';
+import fs from 'fs';
+import path from 'path';
+import child_process from 'child_process';
+import process, { env } from 'process';
+
+const baseFolder =
+  process.env.APPDATA !== undefined && process.env.APPDATA !== ''
+    ? `${process.env.APPDATA}/ASP.NET/https`
+    : `${process.env.HOME}/.aspnet/https`;
+
+const certificateName = "bts.datastreamwebapp.client";
+const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
+const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
+
+if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
+  if (0 !== child_process.spawnSync('dotnet', [
+    'dev-certs',
+    'https',
+    '--export-path',
+    certFilePath,
+    '--format',
+    'Pem',
+    '--no-password',
+  ], { stdio: 'inherit', }).status) {
+    throw new Error("Could not create certificate.");
+  }
+}
+
+const target = env.ASPNETCORE_HTTPS_PORT
+  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
+  : env.ASPNETCORE_URLS
+    ? env.ASPNETCORE_URLS.split(';')[0]
+    : 'https://localhost:5001';
+
+export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
+  return {
+    plugins: [plugin()],
+    resolve: {
+      alias: {
+        '@': fileURLToPath(new URL('./src', import.meta.url))
+      }
+    },
+    build: {
+      minify: false,
+      sourcemap: true,
+      outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
+    },
+    server: {
+      proxy: {
+        '^/api': {
+          target,
+          secure: false
+        },
+        '^/ws': {
+          target,
+          secure: false
+        }
+      },
+      port: 5173,
+      https: {
+        key: fs.readFileSync(keyFilePath),
+        cert: fs.readFileSync(certFilePath),
+      }
+    }
+  };
+});
@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <InvariantGlobalization>true</InvariantGlobalization>
+    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
+    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
+    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+    </ProjectReference>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+  </ItemGroup>
+
+  <!-- <ItemGroup>
+    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
+  </ItemGroup> -->
+</Project>
@@ -0,0 +1,106 @@
+{
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:00.1"
+  },
+  "Sparkd": {
+    "Config": {
+      "ConfigServer": "http://localhost:8000/config"
+    },
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    },
+    "EnableAdmin": true,
+    "EnableOrders": true,
+    "EnablePosition": true
+  },
+  "PubSub": {
+    "MillisecondsBetweenSubInfoUpdates": 1000,
+    "MillisecondsBetweenTimeSliceUpdates": 5000,
+    "MillisecondsBetweenNoBufferSpaceRetries": 100,
+    "RetriesOnNoBufferSpace": 1,
+    "DoItAllSecondsBetweenLogs": 0,
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "UseManagedSockets": true,
+    "StateUpdatePeriod": 5000,
+    "StatisticsUpdatePeriod": 5000,
+    "DisposedConnectionCheckPeriod": 1000,
+    "MessageProcessingWarningThreshold": 1000,
+    "ServerPort": 12345,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "MaxPacketSize": 500000000,
+    "BlueCommMustDie": true,
+    "CrushableTags": [],
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "ServerQueue": "PubSubQueue",
+    "PortRange": {
+      "First": 30400,
+      "Last": 31700
+    },
+    "BindToRandomPort": false,
+    "DebugTags": "^NOTAGS$",
+    "LatencyReportThresholdMs": 1500,
+    "UseSubstations": false,
+    "UseDynamicClients": false,
+    "IsSubstation": false,
+    "InternallyPublishSubInfo": true,
+    "UseManagedClientsInNativeCode": false,
+    "CompressedTags": "^NOTAGS$",
+    "TagsToReportCompressionRatio": "^NOTAGS$",
+    "DisableMsmq": false,
+    "UsePrepublishedServerState": false,
+    "IpTranslation": null,
+    "GcActivationThreshold": 0,
+    "WaitPeriodAfterGc": 1000,
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+         "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  }
+}
@@ -0,0 +1,4 @@
+{
+  "content_root_path": "build/Bts.DataStreamWebApp.Client/production",
+  "web_root_path": "dist"
+}
@@ -0,0 +1,30 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using Microsoft.AspNetCore.Mvc;
+
+[Route("/ws")]
+public class DataStreamSubscriptionController : ControllerBase {
+   readonly ILogger logger;
+
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
+      this.logger = logger;
+      logger.LogInformation("Creating {ControllerType}", GetType().Name);
+   }
+
+   [Route("/ws")]
+   public async Task Get(CancellationToken cancellationToken) {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      logger.LogInformation("Establishing websocket connection...");
+      try {
+         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+         logger.LogInformation("Websocket connection established.");
+         using var server = new DataStreamWebSocketServer(webSocket, logger);
+         await server.Run(cancellationToken);
+      } catch (Exception ex) {
+         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
+      }
+   }
+}
@@ -0,0 +1,125 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamSubscription : IDisposable, IHostedService {
+   public readonly ViewStream ViewStream;
+   public DataPacket? LastFired;
+
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+
+   public DataStreamSubscription(string id, JsonRpcId requestId, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
+      ViewStream = viewStream;
+      RequestId = requestId;
+      Id = id;
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public string Id { get; }
+   public JsonRpcId RequestId { get; }
+   public SubscriptionStatus Status { get; private set; }
+
+   public void Start() {
+      if (Status == SubscriptionStatus.Subscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Subscribed;
+      ViewStream.onChange += onChange;
+   }
+
+   public Task StartAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Start();
+      return Task.CompletedTask;
+   }
+
+   public Task Clear() {
+      if (LastFired is null) {
+         return Task.CompletedTask;
+      }
+      logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
+      return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
+   }
+
+   public void Stop() {
+      if (Status == SubscriptionStatus.Unsubscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Unsubscribed;
+      ViewStream.onChange -= onChange;
+   }
+
+   public Task StopAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Stop();
+      return Task.CompletedTask;
+   }
+
+   public void Dispose() =>
+      Stop();
+
+   async void onChange(object newValue) {
+      if (newValue is not DataPacket dp) {
+         logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
+         return;
+      }
+      if (webSocket.State != WebSocketState.Open) {
+         logger.LogError(
+            "Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
+         Dispose();
+         return;
+      }
+      await sendDataPacket("subscribe", dp);
+   }
+
+   async Task sendDataPacket(string method, DataPacket dp) {
+      var serializerOpts = new JsonSerializerOptions {
+         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+      };
+      serializerOpts.Converters.Add(new DataPacketJsonConverter() {
+         IncludeCurrent = true,
+         UseObjectRows = true,
+      });
+      var res = JsonRpcRequest.CreateNotification(method, dp);
+      var str = serialize(res, serializerOpts);
+      if (str is null) {
+         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
+         Dispose();
+         return;
+      }
+      logger.LogInformation(
+         "Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
+         dp.newRows.Count,
+         dp.schema.columnNames.Length,
+         str.Length);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
+      LastFired = dp;
+   }
+
+   string? serialize<T>(T message, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      try {
+         return JsonSerializer.Serialize(message, opts);
+      } catch (Exception ex) {
+         logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
+      }
+      return null;
+   }
+}
@@ -0,0 +1,89 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using Bts.PubSub;
+
+using CommandLine;
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamWebAppServerConfig {
+   /// <summary>
+   /// The login user for sparkd
+   /// </summary>
+   [JsonPropertyName("user")]
+   [Option("user", HelpText = "The sparkd login user")]
+   public string? User { get; set; }
+
+   /// <summary>
+   /// The login password for sparkd
+   /// </summary>
+   [JsonPropertyName("password")]
+   [Option("pass", HelpText = "The sparkd login password")]
+   public string? Password { get; set; }
+
+   /// <summary>
+   /// The config server url for sparkd
+   /// </summary>
+   [JsonPropertyName("sparkd")]
+   [Option("sparkd", HelpText = "The sparkd config server")]
+   public string? Sparkd { get; set; }
+
+   [JsonPropertyName("pubsub")]
+   public PubSubConfig? PubSub { get; set; }
+
+   [Option("content-root-path")]
+   [JsonPropertyName("content_root_path")]
+   public string? ContentRootPath { get; set; }
+
+   [Option("web-root-path")]
+   [JsonPropertyName("web_root_path")]
+   public string? WebRootPath { get; set; }
+
+   [Option("environment-name")]
+   [JsonPropertyName("environment_name")]
+   public string? EnvironmentName { get; set; }
+
+   public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
+
+   [Option(
+      "config",
+      MetaValue = "PATH",
+      HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
+   [JsonIgnore]
+   public string? ConfigFile { get; set; }
+
+   public static DataStreamWebAppServerConfig Default() =>
+      new() {
+         User = "bts",
+         PubSub = PubSubConfig.Default(),
+      };
+
+   public void MergeWith(DataStreamWebAppServerConfig other) {
+      User ??= other.User;
+      Password ??= other.Password;
+      PubSub ??= other.PubSub;
+      Sparkd ??= other.Sparkd;
+      ContentRootPath ??= other.ContentRootPath;
+      WebRootPath ??= other.WebRootPath;
+      ConfigFile ??= other.ConfigFile;
+   }
+
+   /// <summary>
+   /// Merges options with those defined in `ConfigFile` (does not overwrite
+   /// non-null existing values).
+   /// </summary>
+   /// <exception cref="InvalidOperationException">
+   /// JsonSerializer.Deserialize returns null
+   /// </exception>
+   public void MergeWithConfigFile() {
+      if (ConfigFile is null) {
+         return;
+      }
+      var cfgString = File.ReadAllText(ConfigFile);
+      var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
+         throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
+      MergeWith(parsedCfg);
+      ConfigFile = null;
+   }
+}
@@ -0,0 +1,121 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
+
+   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public async Task<int> Run(CancellationToken cancellationToken) {
+      logger.LogInformation("DataStreamWebSocketServer running");
+      await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger, cancellationToken)) {
+         await (req.Method switch {
+            "subscribe" => onSubscribe(req, cancellationToken),
+            "unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>(), cancellationToken),
+            _ => Task.CompletedTask,
+         });
+      }
+      logger.LogInformation("DataStreamWebSocketServer stopped");
+      return 0;
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions. To additionally notify the client of the subscription status
+   /// change(s), use <see cref="DisposeAsync"/>.
+   /// </summary>
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      disposeSubscriptions();
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions (like <see cref="Dispose"/>), then sends 'subscription status change'
+   /// notifications to the client (unlike <see cref="Dispose"/>).
+   /// </summary>
+   public async ValueTask DisposeAsync() {
+      GC.SuppressFinalize(this);
+      var finalSubs = subscriptions.Values.Select(s => (s.RequestId, s.Id)).ToList();
+      disposeSubscriptions();
+      // Send a 'subscription status change' msg to the client for each one.
+      foreach (var (requestId, id) in finalSubs) {
+         await sendSubscriptionStatus(requestId, id, "unsubscribe", SubscriptionStatus.Unsubscribed, CancellationToken.None);
+      }
+   }
+
+   async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      if (!SubscribeRequest.ParseQuery(req, out var vs, out var subId, out var errMsg, out var errCode)) {
+         logger.LogError("Invalid 'subscribe' request: {Reason}", errMsg);
+         var rpcError = new JsonRpcError<JsonNode>() {
+            Code = (long)errCode,
+            Message = $"{errCode}: {errMsg}",
+         };
+         await send(rpcError, cancellationToken);
+         return;
+      }
+      if (subscriptions.TryGetValue(subId, out var oldSub)) {
+         oldSub.Dispose();
+      }
+      var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
+      subscriptions[subId] = sub;
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
+      await sub.StartAsync(cancellationToken);
+   }
+
+   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      var subId = req.Params?.SubscriptionId ?? string.Empty;
+      if (!subscriptions.TryGetValue(subId, out var sub)) {
+         logger.LogInformation("Ignoring '{Method}' request because no subscription was found with id {Id}.", req.Method, subId);
+         return;
+      }
+      logger.LogInformation("Disposing subscription");
+      sub.Dispose();
+      subscriptions.Remove(subId);
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   async Task unsubscribe(JsonRpcId requestId, string subscriptionId, CancellationToken cancellationToken) {
+      await sendSubscriptionStatus(requestId, subscriptionId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
+      logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
+      var response = JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
+         SubscriptionId = subId,
+         Status = status,
+      });
+      return send(response, cancellationToken);
+   }
+
+   Task send<T>(T message, CancellationToken cancellationToken) {
+      if (message is null) {
+         return Task.CompletedTask;
+      }
+      var str = JsonSerializer.Serialize(message);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
+      return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken);
+   }
+
+   void disposeSubscriptions() {
+      logger.LogInformation("Disposing DataStreamWebSocketServer subscriptions. Count: {Count}", subscriptions.Count);
+      subscriptions.Values.DisposeAll();
+      subscriptions.Clear();
+   }
+}
@@ -0,0 +1,6 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+enum ErrorCode {
+   None = 0,
+   InvalidSubscriptionRequest,
+}
@@ -0,0 +1,67 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamExpressions;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+public class SubscribeRequest {
+   [JsonPropertyName("query")]
+   public string? Query { get; set; }
+
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   internal static bool ParseQuery(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(true)] out string? subscriptionId, [NotNullWhen(false)] out string? errorMessage, out ErrorCode errorCode) {
+      stream = null;
+      subscriptionId = null;
+      errorCode = ErrorCode.None;
+      var paramsJsonObj = req.Params?.AsObject();
+      if (paramsJsonObj is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params' object not found";
+         return false;
+      }
+      var subReq = paramsJsonObj.Deserialize<SubscribeRequest>();
+      if (subReq is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params' object invalid";
+         return false;
+      }
+      subscriptionId = subReq.SubscriptionId ?? string.Empty;
+      var query = subReq.Query;
+      if (query is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params.query' field not found (expected a string)";
+         return false;
+      }
+      errorMessage = null;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(query);
+      if (exp is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params.query' expression has invalid syntax";
+         return false;
+      }
+      string? exceptionMsg = null;
+      try {
+         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      } catch (Exception ex) {
+         exceptionMsg = ex.ToString();
+      }
+      if (stream is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
+         if (exceptionMsg is not null) {
+            errorMessage += ". Error" + exceptionMsg;
+         }
+         return false;
+      }
+      return true;
+   }
+}
@@ -0,0 +1,18 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+enum SubscriptionStatus {
+   None,
+   Subscribed,
+   Unsubscribed,
+}
+
+class SubscriptionStatusMessage {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   [JsonPropertyName("status")]
+   [JsonConverter(typeof(JsonStringEnumConverter))]
+   public SubscriptionStatus Status { get; set; } = SubscriptionStatus.None;
+}
@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+public class UnsubscribeRequest {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+}
@@ -0,0 +1,59 @@
+using Bts.DataStreamCore.Services;
+using Bts.PubSub.Services;
+
+using Spark.Api.Services;
+
+using System.Reactive.Linq;
+
+var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
+   Args = args,
+   ApplicationName = "Bts.DataStreamWebApp.Server",
+});
+ConfigureApp(builder.Configuration);
+ConfigureServices(builder.Services, builder.Configuration);
+
+var app = builder.Build();
+if (app.Environment.IsDevelopment()) {
+   app.UseDirectoryBrowser();
+}
+var wsOpts = new WebSocketOptions {
+   KeepAliveInterval = TimeSpan.FromMinutes(2)
+};
+app.UseWebSockets(wsOpts);
+app.UseDefaultFiles();
+app.UseStaticFiles();
+if (app.Environment.IsDevelopment()) {
+   app.UseSwagger();
+   app.UseSwaggerUI();
+}
+app.UseHttpsRedirection();
+app.UseAuthorization();
+app.MapControllers();
+try {
+   Console.Write($"Starting Application");
+   app.Run();
+   Console.Write($"Application exited gracefully.");
+} catch (Exception ex) {
+   Console.Error.Write($"Application crashed: {ex}");
+   return 1;
+}
+return 0;
+
+static void ConfigureApp(ConfigurationManager configBuilder) {
+   configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
+   var cfgRoot = configBuilder.GetSection("config");
+   if (cfgRoot?.Value is not null) {
+      configBuilder.AddJsonFile(cfgRoot.Value);
+   }
+}
+
+static void ConfigureServices(IServiceCollection services, IConfiguration config) {
+   services.AddDirectoryBrowser();
+   services.AddControllers();
+   services.AddEndpointsApiExplorer();
+   services.AddSwaggerGen();
+   services.AddMasterWorker(config.GetSection("MasterWorker"));
+   services.AddViewStreamDatabase(config.GetSection("ViewStreamDatabase"));
+   services.AddPubSubClient(config.GetSection("PubSub"));
+   services.AddSparkd(config.GetSection("Sparkd"));
+}
@@ -0,0 +1,26 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "applicationUrl": "http://localhost:5000",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:5000;https://localhost:5001",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+  },
+}
@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}
@@ -0,0 +1,168 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>ALT WebSocket Sample Application</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Message to send:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            var data = sendMessage.value;
+            socket.send(data);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>
@@ -0,0 +1,176 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>DataStreams via WebSocket</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Query:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            const rpcReq = {
+              jsonrpc: "2.0",
+              method: "subscribe",
+              params: {
+                query: sendMessage.value,
+              },
+              id: 1,
+            };
+            const rpcReqStr = JSON.stringify(rpcReq);
+            socket.send(rpcReqStr);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>
@@ -4,6 +4,8 @@ using System.Collections.Generic;
 using Bts.DataStreamExpressions;
 using BCG.Utilities.General;
 using bts;
+using Bts.PubSub;
+using Bts.DataStreamCore;

 namespace BCG {
   // listens to the requests posted on "cloud_streams" request table, and 
@@ -40,13 +42,15 @@ namespace BCG {
      private IDictionary<string, StreamPubUnit>
         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
      private ViewStream _stream = new TableStream("cloud_streams");
-      private PubTableBroker _broker = new PubTableBroker();
-      private ManagedSubscriber _sub = ManagedSubscriber.instance;
+      readonly IPubTableBroker _broker;
+      readonly IManagedSubscriber _sub;


-      public RemoteQueryManager() {
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
+         _sub = managedSubscriber;
+         _broker = pubTableBroker;
         _stream.onChange += _on_str;
-         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
+         _sub.Subscribe("RemoteRequest", (string tag, byte[] data) => {
            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
         });
@@ -8,6 +8,7 @@ using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
 using bts.api;
+using Bts.PubSub.Routing;

 namespace bts {
   [DataStreamUserRespresentation("pss_echo")]
@@ -87,8 +88,10 @@ namespace bts {
      IOState _state = IOState.SEND;

      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd) {
-         var route = new PubSubRoute(server_url, ".*");
-         _router = new PubSubRouter(route);
+         _router = new PubSubRouter(new PubSubRouterConfig() {
+            PublisherRoutes = new() { new(server_url, ".*") },
+            SubscriberRoutes = new() { new(server_url, ".*") },
+         });
         _vsd = vsd;
         _payload_sz = payload_size;
         var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -196,4 +199,4 @@ namespace bts {
      }

   }
-}
+}
@@ -617,6 +617,7 @@ namespace BCG {


        private static TestResult _run_single_tag_data_test_clean() {
+            // var tagHandlerFactory
            // create message
            var test_name = MethodBase.GetCurrentMethod()?.Name;
            var schema = create_schema_with_all_types();
@@ -624,7 +625,7 @@ namespace BCG {
            BlueTable table = new BlueTable(schema);
            IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
            IStreamMsgDeserilizer deser = new StreamIODeserializationController();
-            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
+            bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType, new PubsubServerStreamTagHandler());
            // create with 100 rows
            int n_rows = 100;
            bool is_succ = true;
@@ -1,3 +1,5 @@
+using Bts.DataStreamCore;
+
 using System;

 namespace BCG {
@@ -7,14 +9,15 @@ namespace BCG {
   /// </summary>
 	public class CircularPublisher {
      private CircularRowCollection myBuffer;
-      private PubTableBroker myBroker;
+      private readonly IPubTableBroker myBroker;
      private int myBufferSize;
-      public CircularPublisher(int bufferSize) {
+      public CircularPublisher(int bufferSize, IPubTableBroker broker) {
         myBufferSize = bufferSize;
+         myBroker = broker;
      }

-      public CircularPublisher(int bufferSize, IBlueRowCollection rows)
-         : this(bufferSize) {
+      public CircularPublisher(int bufferSize, IPubTableBroker broker, IBlueRowCollection rows)
+         : this(bufferSize, broker) {
         publishRows(rows);
      }

@@ -23,7 +26,6 @@ namespace BCG {
            myBuffer = new CircularRowCollection(
               new BlueTableSchema(nr.schema.tableName, nr.schema.columnInfo),
               myBufferSize);
-            myBroker = new PubTableBroker();
            myBroker.create_table(myBuffer);
         }
         IBlueRow br = myBuffer.addRowWithReturn(nr);
@@ -38,7 +40,6 @@ namespace BCG {
               new BlueTableSchema(BlueDataUtility.extractRow(nrs, 0).schema.tableName,
               BlueDataUtility.extractRow(nrs, 0).schema.columnInfo),
               myBufferSize);
-            myBroker = new PubTableBroker();
            myBroker.create_table(myBuffer);
         }
         BlueTable deletes = new BlueTable(myBuffer.schema);
@@ -4,10 +4,12 @@ using System.Data;

 using BCG.Utilities;

+using Bts.DataStreamCore;
+
 namespace BCG {
   public class ViewPublisher : IDisposable {
      protected ViewPublisher() { }
-      private readonly PubTableBroker _broker;
+      private readonly IPubTableBroker _broker;
      protected readonly ViewStream _stream;
      private int _n_updates = 0;

@@ -15,7 +17,7 @@ namespace BCG {
      /// in the schema (be careful with that because this
      /// could lead to double publishes if the schema name
      /// is not uniquely chosen)
-      public ViewPublisher(string name, PubTableBroker broker, ViewStream vs) {
+      public ViewPublisher(string name, IPubTableBroker broker, ViewStream vs) {
         _stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
         _broker = broker;
         _stream.onChange += new DataStream.ChangeDelegate(_handle_update);
@@ -0,0 +1,11 @@
+namespace Bts.DataStream.Services;
+
+using BCG;
+using BCG.Utilities.General;
+
+using Microsoft.Extensions.DependencyInjection;
+
+public static class DataStreamServices {
+   public static IServiceCollection AddRemoteQueryManager(this IServiceCollection services) =>
+      services.AddSingleton<IWorker, RemoteQueryManager>();
+}
@@ -0,0 +1,11 @@
+namespace Bts.DataStreamCore;
+
+public enum DataType {
+   Null,
+   String,
+   Float32,
+   Float64,
+   Int32,
+   Int64,
+   DateTime,
+}
@@ -11,7 +11,7 @@ using System.Text.Json.Serialization;

 namespace Bts.DataStreamCore;

-class DataPacketJsonConverter : JsonConverter<DataPacket> {
+public class DataPacketJsonConverter : JsonConverter<DataPacket> {
   /// <summary>
   /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
   /// default, only incremental changes are included ('added' and 'deleted').
@@ -58,91 +58,74 @@ class DataPacketJsonConverter : JsonConverter<DataPacket> {

   public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
      using var _ = writer.WriteObject();
-      using (writer.WriteArray("schema")) {
+      using (writer.WriteArray("columns")) {
         foreach (var col in value.schema.columnInfo) {
            using (writer.WriteObject()) {
               writer.WriteString("name", col.colName);
               writer.WriteNumber("index", col.colNo);
               // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
-               writer.WriteString("type", col.colType.ToString());
+               writer.WriteString("type", col.type_code().ToString());
            }
         }
      }

-      var cellWriters = value.schema.columnInfo.Select(getCellWriter).ToList();
-      using (writer.WriteArray("added")) {
-         foreach (IBlueRow row in value.addedRows) {
-            using (writer.WriteArray()) {
-               for (int i = 0; i < cellWriters.Count; i++) {
-                  cellWriters[i](row[i]);
-               }
-            }
-         }
-      }
-      using (writer.WriteArray("deleted")) {
-         foreach (IBlueRow row in value.deletedRows) {
-            using (writer.WriteArray()) {
-               for (int i = 0; i < cellWriters.Count; i++) {
-                  cellWriters[i](row[i]);
-               }
-            }
-         }
-      }
+      var columns = value.schema.columnInfo.Select(c => (Column: c, Write: GetCellValueWriter(c))).ToList();
+      WriteRows("added", value.addedRows);
+      WriteRows("deleted", value.deletedRows);
      if (IncludeCurrent) {
-         using (writer.WriteArray("current")) {
-            foreach (IBlueRow row in value.newRows) {
-               using (writer.WriteArray()) {
-                  for (int i = 0; i < cellWriters.Count; i++) {
-                     cellWriters[i](row[i]);
-                  }
-               }
+         WriteRows("all", value.newRows);
+      }
+
+      void WriteRows(string name, IBlueRowCollection rows) {
+         using (writer.WriteArray(name)) {
+            foreach (IBlueRow row in rows) {
+               WriteRow(row);
            }
         }
      }

-      Action<object?> getCellWriter(ColumnInfo colInfo) {
-         var writeVal = getCellValueWriter(colInfo);
-         if (!UseObjectRows) {
-            return writeVal;
+      void WriteRow(IBlueRow row) {
+         using var rowCase = UseObjectRows ? writer.WriteObject() : writer.WriteArray();
+         for (int i = 0; i < columns.Count; i++) {
+            if (UseObjectRows) {
+               writer.WritePropertyName(columns[i].Column.colName);
+            }
+            columns[i].Write(row[i]);
         }
-         return v => {
-            writer.WritePropertyName(colInfo.colName);
-            writeVal(v);
-         };
      }

-      Action<object?> getCellValueWriter(ColumnInfo colInfo) {
+      Action<object?> GetCellValueWriter(ColumnInfo colInfo) {
         var t = colInfo.colType;
         if (t == typeof(int)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
+            return NullCheckWrapper(v => writer.WriteNumberValue((int)v));
         }
         if (t == typeof(uint)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
+            return NullCheckWrapper(v => writer.WriteNumberValue((uint)v));
         }
         if (t == typeof(long)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
+            return NullCheckWrapper(v => writer.WriteNumberValue((long)v));
         }
         if (t == typeof(ulong)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
+            return NullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
         }
         if (t == typeof(short)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
+            return NullCheckWrapper(v => writer.WriteNumberValue((short)v));
         }
         if (t == typeof(ushort)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
+            return NullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
         }
         if (t == typeof(double)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
+            return NullCheckWrapper(v => JsonSerializer.Serialize(writer, (double)v, options));
         }
         if (t == typeof(float)) {
-            return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
+            return NullCheckWrapper(v => writer.WriteNumberValue((float)v));
         }
         if (t == typeof(bool)) {
-            return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
+            return NullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
         }
-         return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
+         return NullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));

-         Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
+         Action<object?> NullCheckWrapper(Action<object> writeNonNull) =>
            v => {
               if (v is null) {
                  writer.WriteNullValue();
@@ -0,0 +1,49 @@
+#nullable enable
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using System;
+using System.IO;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+/// <summary>
+/// This class seriailzes DataPackets to json strings. A common buffer is used to reduce memory
+/// allocations on repeated calls to <see cref="Serialize"/>. For that reason, this class is NOT
+/// thread safe.
+/// </summary>
+public class DataPacketJsonSerializer {
+   readonly MemoryStream buffer = new();
+   readonly DataPacketJsonConverter converter;
+
+   public DataPacketJsonSerializer(DataPacketJsonConverter? converter = null) =>
+      this.converter = converter ?? new() {
+         IncludeCurrent = true,
+         UseObjectRows = true,
+      };
+
+   public string? Serialize(DataPacket dp) {
+      buffer.Position = 0;
+      buffer.SetLength(0);
+      using (var jsonWriter = new Utf8JsonWriter(buffer)) {
+         var opts = new JsonSerializerOptions() {
+            NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+         };
+         converter.Write(jsonWriter, dp, opts);
+      }
+      var bytes = buffer.GetBuffer();
+      var span = new ArraySegment<byte>(bytes, 0, (int)buffer.Length);
+      return Encoding.UTF8.GetString(span);
+   }
+
+   public void Serialize(DataPacket dp, Utf8JsonWriter jsonWriter) {
+      buffer.Position = 0;
+      buffer.SetLength(0);
+      var opts = new JsonSerializerOptions() {
+         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+      };
+      converter.Write(jsonWriter, dp, opts);
+   }
+}
@@ -8,6 +8,7 @@

  <ItemGroup>
    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
  </ItemGroup>

  <ItemGroup>
@@ -0,0 +1,8 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+public interface IPubTableBroker {
+   void create_table(IBlueRowCollection table);
+   void make_batch_changes(IBlueRowCollection new_rows, IBlueRowCollection del_rows, IBlueRowCollection added_rows);
+}
@@ -8,12 +8,14 @@ using BCG.Utilities;

 using bts;

+using Bts.DataStreamCore;
+
 namespace BCG {
   /// <summary>
   /// This class will be used by the clients to publish tabular data to the pubsub
   /// </summary>
-   public class PubTableBroker {
-      protected PublisherCore _publisher_core;
+   public class PubTableBroker : IPubTableBroker {
+      protected IPublisherCore _publisher_core;
      protected IDictionary<string, SinglePubTableBroker> _publisher_map =
         new Dictionary<string, SinglePubTableBroker>();

@@ -22,7 +24,7 @@ namespace BCG {
      public PubTableBroker()
         : this(new PublisherCore()) { }

-      public PubTableBroker(PublisherCore publisher) {
+      public PubTableBroker(IPublisherCore publisher) {
         _publisher_core = publisher;
         Log.Gen.info("PubTableBroker constructed");
      }
@@ -14,9 +14,9 @@ namespace BCG {
   public class SinglePubTableBroker {
      private readonly string _tag;
      protected IStreamIOMsgSerializer _serializer;
-      protected PublisherCore _publisher_core;
+      protected IPublisherCore _publisher_core;

-      public SinglePubTableBroker(string tag, PublisherCore publisher_core) {
+      public SinglePubTableBroker(string tag, IPublisherCore publisher_core) {
         bts.Log.Gen.info($"SinglePubTableBroker.cons created for tag: {tag}");
         _tag = tag;
         _publisher_core = publisher_core;
@@ -51,4 +51,4 @@ namespace BCG {
      }

   }
-}
+}
@@ -13,6 +13,9 @@ namespace bts {
      public PubsubServerGenericDataHandler() {
         _bw = new BinaryWriter(_str);
      }
+
+      public string TagType => "*";
+
      public void update(string tag, byte[] msg) {
         _str.Position = 0;
         _bw.Write(msg);
@@ -25,9 +28,6 @@ namespace bts {
         Log.Gen.info($"PubsubServerGenericDataHandler.get_content tag: {tag} rv: {rv.Length}");
         return rv;
      }
-      public string[] supportedTagTypes { get; }
-      public void register(InternalPublishDelegate callback) { }
-      public void doWork() { }
   }

   // This class is created by the SingleTagData for stream tag_type. Every pub
@@ -40,6 +40,7 @@ namespace bts {
      // this is where the state of the table is kept.
      BlueTable _state = null;

+      public string TagType => DSTagManager.DataStreamTagType;

      public void update(string tag, byte[] msg) {
         StreamIOMessage str_msg = _deser.deserialize(msg);
@@ -74,15 +75,6 @@ namespace bts {
         Log.Gen.info($"PubsubServerSingleStreamTagHandler.get_content rv: {rv_txt}");
         return rv;
      }
-      public string[] supportedTagTypes {
-         get {
-            return new string[] { "DS" };
-         }
-      }
-      public void register(InternalPublishDelegate callback) {
-
-      }
-      public void doWork() { }
   }

   public class PubsubServerStreamTagHandler : IPubSubServerTagHandler {
@@ -92,6 +84,9 @@ namespace bts {

      public PubsubServerStreamTagHandler() {
      }
+
+      public string TagType => DSTagManager.DataStreamTagType;
+
      public void update(string tag, byte[] msg) {
         BCG.IPubSubServerTagHandler rv = _get_handler(tag);
         rv.update(tag, msg);
@@ -100,14 +95,6 @@ namespace bts {
         BCG.IPubSubServerTagHandler rv = _get_handler(tag);
         return rv.get_content(tag);
      }
-      public string[] supportedTagTypes {
-         get {
-            List<string> rv = new List<string>();
-            foreach (var h in _handler_map.Values)
-               rv.AddRange(h.supportedTagTypes);
-            return rv.ToArray();
-         }
-      }

      private BCG.IPubSubServerTagHandler _get_handler(string tag) {
         PubsubServerSingleStreamTagHandler rv = null;
@@ -117,13 +104,5 @@ namespace bts {
         }
         return rv;
      }
-      public void register(InternalPublishDelegate callback) {
-         foreach (var h in _handler_map.Values)
-            h.register(callback);
-      }
-      public void doWork() {
-         foreach (var h in _handler_map.Values)
-            h.doWork();
-      }
   }
-}
+}
@@ -0,0 +1,21 @@
+namespace Bts.DataStreamCore.Services;
+
+using BCG;
+using BCG.Utilities.General;
+
+using Bts.PubSub.Services;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+
+public static class DataStreamCoreServices {
+   public static IServiceCollection AddPubTableBroker(this IServiceCollection services) =>
+      services.AddSingleton<IPubTableBroker, PubTableBroker>();
+
+   public static IServiceCollection AddViewStreamDatabase(this IServiceCollection services, IConfiguration configurationSection) =>
+      services.Configure<WorkerServiceConfig>(configurationSection)
+         .AddSingleton<IWorker, ViewStreamDatabase>(sp => {
+            ViewStreamDatabase.SetGlobalInstance(new ViewStreamDatabase(sp.GetRequiredService<ISubscriberCore>()));
+            return ViewStreamDatabase.instance;
+         });
+}
@@ -9,6 +9,7 @@ using System.Threading;
 using mbts;
 using System.Linq;
 using System.Diagnostics;
+using bts;

 namespace BCG {
   public class ViewStreamDatabase : IWorker, IDisposable {
@@ -230,6 +231,11 @@ namespace BCG {
         }
      }

+      internal static void SetGlobalInstance(ViewStreamDatabase vsd) {
+         Log.isnull(mySelf);
+         mySelf = vsd;
+      }
+
      private List<PendingAddRemoveWorker> _pending_worker_ops = new List<PendingAddRemoveWorker>();
      private bool _is_worker_call_in_progress = false;
      class PendingAddRemoveWorker {
@@ -1,109 +1,120 @@
 {
-  "user": "bts",
-  "sparkd": "http://127.0.0.1:8000/config",
-  "pubsub": {
-    "milliseconds_between_sub_info_updates": 1000,
-    "milliseconds_between_time_slice_updates": 5000,
-    "milliseconds_between_no_buffer_space_retries": 100,
-    "retries_on_no_buffer_space": 1,
-    "do_it_all_seconds_between_logs": 0,
-    "heartbeat_interval": 15000,
-    "missing_heartbeats_before_gc": 50,
-    "use_managed_sockets": true,
-    "state_update_period": 5000,
-    "statistics_update_period": 5000,
-    "disposed_connection_check_period": 1000,
-    "message_processing_warning_threshold": 1000,
-    "server_port": 12345,
-    "publisher_routes": [
-      {
-        "name": "default",
-        "server_address": "bts-cloud://127.0.0.1:12345",
-        "tag": ".*"
-      }
-    ],
-    "subscriber_routes": [
-      {
-        "name": "default",
-        "server_address": "bts-cloud://127.0.0.1:12345",
-        "tag": ".*"
-      }
-    ],
-    "max_packet_size": 500000000,
-    "blue_comm_must_die": true,
-    "crushable_tags": [],
-    "tag_type_to_assembly_map": [
-      {
-        "handler_class": "BCG.PubSubServerTagHandler",
-        "assembly": "DataStreamCore.dll"
-      }
-    ],
-    "server_queue": "PubSubQueue",
-    "port_range": {
-      "first": 30400,
-      "last": 31700
+  "Sparkd": {
+    "Config": {
+      "ConfigServer": "http://localhost:8000/config"
    },
-    "bind_to_random_port": false,
-    "debug_tags": "^NOTAGS$",
-    "latency_report_threshold_ms": 1500,
-    "use_substations": false,
-    "use_dynamic_clients": false,
-    "is_substation": false,
-    "internally_publish_sub_info": true,
-    "use_managed_clients_in_native_code": false,
-    "compressed_tags": "^NOTAGS$",
-    "tags_to_report_compression_ratio": "^NOTAGS$",
-    "disable_msmq": false,
-    "use_prepublished_server_state": false,
-    "ip_translation": null,
-    "gc_activation_threshold": 0,
-    "wait_period_after_gc": 1000
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    },
+    "EnableAdmin": true,
+    "EnableOrders": true,
+    "EnablePosition": true
  },
-  "publishers": [
-    {
-      "name": "market_data",
-      "streams": [
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "Publisher": {
+    "Publishers": [
+      {
+        "Name": "test_data",
+        "Streams": [
+          {
+            "Name": "milliseconds_1000_100",
+            "Expression": "milliseconds(3000,5)",
+            "IsOnWebsocket": true
+          }
+        ]
+      }
+    ]
+  },
+  "PubSub": {
+    "MillisecondsBetweenSubInfoUpdates": 1000,
+    "MillisecondsBetweenTimeSliceUpdates": 5000,
+    "MillisecondsBetweenNoBufferSpaceRetries": 100,
+    "RetriesOnNoBufferSpace": 1,
+    "DoItAllSecondsBetweenLogs": 0,
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "UseManagedSockets": true,
+    "StateUpdatePeriod": 5000,
+    "StatisticsUpdatePeriod": 5000,
+    "DisposedConnectionCheckPeriod": 1000,
+    "MessageProcessingWarningThreshold": 1000,
+    "ServerPort": 12345,
+    "Routing": {
+      "PublisherRoutes": [
        {
-          "name": "bbo",
-          "expression": "bbo()",
-          "is_on_websocket": true
-        },
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
        {
-          "name": "instruments",
-          "expression": "instruments()",
-          "is_on_websocket": true
-        },
-        {
-          "name": "timesales",
-          "expression": "timesales(10000)",
-          "is_on_websocket": true
-        },
-        {
-          "name": "books",
-          "expression": "books()",
-          "is_on_websocket": true
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
        }
      ]
    },
-    {
-      "name": "test_data",
-      "streams": [
+    "MaxPacketSize": 500000000,
+    "BlueCommMustDie": true,
+    "CrushableTags": [],
+    "TagTypeToAssemblyMap": {
+      "Entries": [
        {
-          "name": "milliseconds_1000_100",
-          "expression": "milliseconds(1000,100)",
-          "is_on_websocket": true
-        },
-        {
-          "name": "test_literal",
-          "expression": "literal(\"x\", \"420.0\")",
-          "is_on_websocket": true
-        },
-        {
-          "name": "test_not_on_websocket",
-          "expression": "literal(\"x\", \"840.0\")",
-          "is_on_websocket": false
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
        }
      ]
-    }
-  ]
-}
+    },
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    },
+    "ServerQueue": "PubSubQueue",
+    "PortRange": {
+      "First": 30400,
+      "Last": 31700
+    },
+    "BindToRandomPort": false,
+    "DebugTags": "^NOTAGS$",
+    "LatencyReportThresholdMs": 1500,
+    "UseSubstations": false,
+    "UseDynamicClients": false,
+    "IsSubstation": false,
+    "InternallyPublishSubInfo": true,
+    "UseManagedClientsInNativeCode": false,
+    "CompressedTags": "^NOTAGS$",
+    "TagsToReportCompressionRatio": "^NOTAGS$",
+    "DisableMsmq": false,
+    "UsePrepublishedServerState": false,
+    "IpTranslation": null,
+    "GcActivationThreshold": 0,
+    "WaitPeriodAfterGc": 1000
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:01"
+  }
+}
@@ -17,14 +17,14 @@
    "publisher_routes": [
      {
        "name": "default",
-        "server_address": "bluto://localhost:12345",
+        "server_address": "bts-cloud://localhost:12345",
        "tag": ".*"
      }
    ],
    "subscriber_routes": [
      {
        "name": "default",
-        "server_address": "bluto://localhost:12345",
+        "server_address": "bts-cloud://localhost:12345",
        "tag": ".*"
      }
    ],
@@ -0,0 +1,6 @@
+namespace Bts.DataStreamPublisher;
+
+public class CsvPublisherConfig {
+   public string? Name { get; set; }
+   public string? CsvDir { get; set; }
+}
@@ -9,6 +9,7 @@

  <ItemGroup>
    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
  </ItemGroup>

  <ItemGroup>
@@ -1,7 +1,5 @@
 namespace Bts.DataStreamPublisher;

-using Bts.PubSub;
-
 using CommandLine;

 using System;
@@ -11,35 +9,12 @@ using System.Text.Json;
 using System.Text.Json.Serialization;

 public class DataStreamPublisherConfig {
-   /// <summary>
-   /// The login user for sparkd
-   /// </summary>
-   [JsonPropertyName("user")]
-   [Option("user", HelpText = "The sparkd login user")]
-   public string? User { get; set; }
-
-   /// <summary>
-   /// The login password for sparkd
-   /// </summary>
-   [JsonPropertyName("password")]
-   [Option("pass", HelpText = "The sparkd login password")]
-   public string? Password { get; set; }
-
-   /// <summary>
-   /// The config server url for sparkd
-   /// </summary>
-   [JsonPropertyName("sparkd")]
-   [Option("sparkd", HelpText = "The sparkd config server")]
-   public string? Sparkd { get; set; }
-
   /// <summary>
   /// The tables to publish on startup. Each key/value represents a table name/query to publish.
   /// </summary>
-   [JsonPropertyName("publishers")]
   public List<PublisherNode>? Publishers { get; set; }

-   [JsonPropertyName("pubsub")]
-   public PubSubConfig? PubSub { get; set; }
+   public List<string>? CsvDirs { get; set; }

   [Option(
      "config",
@@ -49,16 +24,10 @@ public class DataStreamPublisherConfig {
   public string? ConfigFile { get; set; }

   public static DataStreamPublisherConfig Default() =>
-      new() {
-         PubSub = PubSubConfig.Default(),
-      };
+      new() { };

   public void MergeWith(DataStreamPublisherConfig other) {
-      User ??= other.User;
-      Password ??= other.Password;
-      Sparkd ??= other.Sparkd;
      Publishers ??= other.Publishers;
-      PubSub ??= other.PubSub;
      ConfigFile ??= other.ConfigFile;
   }

@@ -1,140 +1,34 @@
-using BCG;
-using BCG.Utilities.General;
+using Bts.DataStreamPublisher.Services;

-using bts;
-using bts.api.core;
-using bts.utils;
-
-using Bts.DataStreamExpressions;
-using Bts.DataStreamPublisher;
-using Bts.PubSub;
-
-using CommandLine;
-
-using mbts.config;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;

 using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Text.Json;
-using System.Threading.Tasks;

-await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
-   .MapResult(RunAsync, errs => Task.FromResult(1));
+var host = Host.CreateDefaultBuilder(args)
+   .ConfigureAppConfiguration(ConfigureApp)
+   .ConfigureServices(ConfigureServices)
+   .Build();

-async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
-   try {
-      cfg.MergeWithConfigFile();
-      cfg.MergeWith(DataStreamPublisherConfig.Default());
-      PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
-      Configuration.initialize(cfg.Sparkd);
-      Log.Gen.info("Starting DataStreamPublisher");
-      var service_ctx = ServiceContext.create(cfg.Sparkd);
-      await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
-      _publish_streams(cfg.Publishers ?? new(0));
-   } catch (Exception ex) {
-      // Write the error to console first, in case the logging system is compromised.
-      var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
-      Console.Error.WriteLine(error_msg);
-      Console.Error.Flush();
-      bts.Log.Gen.error(error_msg);
-      return 1;
-   }
-   return 0;
+try {
+   Console.Write("Starting DataStreamPublisher");
+   host.Run();
+   Console.Write("DataStreamPublisher exited gracefully.");
+} catch (Exception ex) {
+   Console.Error.Write($"DataStreamPublisher crashed: {ex}");
+   return 1;
 }
+return 0;

-static ViewStream? build_stream(string? stream_expression) {
-   if (stream_expression is null) {
-      return null;
-   }
-   try {
-      bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
-      var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-      var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
-      return rv;
-
-   } catch (Exception ex) {
-      bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
-      return null;
+static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) {
+   configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
+   var cfgRoot = configBuilder.Build();
+   var cfg = cfgRoot["config"];
+   if (cfg is not null) {
+      configBuilder.AddJsonFile(cfg, false);
   }
 }

-void _publish_streams(IEnumerable<PublisherNode> publishers) {
-   PubTableBroker pb_broker = new PubTableBroker();
-   RemoteQueryManager csm = new RemoteQueryManager();
-   var csvPublishers = new List<CSVPublisher>();
-
-   foreach (var publisher in publishers) {
-      if (Directory.Exists(publisher.CsvDir)) {
-         csvPublishers.Add(new(publisher.CsvDir, pb_broker));
-      }
-   }
-
-   var viewPublishers = publishers
-      .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher))
-      .ToList();
-
-   while (true) {
-      System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
-      ViewStreamDatabase.instance.doWork();
-      csm.doWork();
-      MasterWorker.getInstance(false).doWork();
-      ManagedSubscriber.instance.doWork();
-   }
-
-   ViewPublisher? createViewPublisher(PublishedStream ps) {
-      Log.Gen.info(
-         $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
-      if (string.IsNullOrEmpty(ps.Name)) {
-         Log.Gen.warn(
-            $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
-         return null;
-      }
-      var vs = build_stream(ps.Expression);
-      if (vs is null) {
-         Log.Gen.warn(
-            $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
-         return null;
-      }
-      var vpub = new ViewPublisher(ps.Name, pb_broker, vs);
-      vs.onChange += on_change;
-      return vpub;
-   }
-}
-
-void on_change(object obj) {
-   bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
-}
-
-public class CSVPublisher : IWorker {
-   private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
-   private readonly string _csv_directory;
-   private PubTableBroker _pub_table_broker;
-
-   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
-      _csv_directory = csv_directory;
-      _pub_table_broker = pub_table_broker;
-      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
-   }
-   public void doWork() {
-      var file_names = Directory.GetFiles(_csv_directory);
-      foreach (var fn in file_names) {
-         if (!_pubs.ContainsKey(fn)) {
-            _handle_new_file(fn);
-         }
-      }
-   }
-
-   private void _handle_new_file(string file_name) {
-      if (file_name.EndsWith(".csv")) {
-         ViewStream vs = new CSVReaderStream(file_name);
-         vs = new SchemaGuesserStream(vs);
-         var toks = file_name.Split('\\');
-         string table_name = toks.Last();
-         table_name = table_name.Remove(table_name.Length - ".csv".Length);
-         ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
-         _pubs.Add(file_name, vp);
-      }
-   }
-}
+static void ConfigureServices(HostBuilderContext context, IServiceCollection services) =>
+   services.AddDataStreamPublisher(context.Configuration);
@@ -9,7 +9,4 @@ public class PublisherNode {

   [JsonPropertyName("streams")]
   public List<PublishedStream>? Streams { get; set; }
-
-   [JsonPropertyName("csv_dir")]
-   public string? CsvDir { get; set; }
 }
@@ -0,0 +1,39 @@
+namespace Bts.DataStreamPublisher.Services;
+
+using BCG;
+using BCG.Utilities.General;
+
+using bts;
+
+using Bts.DataStreamCore;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+public class CsvPublisher : IWorker, IDisposable {
+   readonly Dictionary<string, ViewPublisher> publishers = new();
+   readonly string directory;
+   readonly IPubTableBroker pubTableBroker;
+
+   public CsvPublisher(string directory, IPubTableBroker pubTableBroker) {
+      this.directory = directory;
+      this.pubTableBroker = pubTableBroker;
+   }
+
+   public void doWork() {
+      var newFiles = Directory.GetFiles(directory, "*.csv").WhereNot(publishers.ContainsKey);
+      foreach (var file in newFiles) {
+         ViewStream vs = new CSVReaderStream(file);
+         vs = new SchemaGuesserStream(vs);
+         var tableName = Path.GetFileNameWithoutExtension(file);
+         var vp = new ViewPublisher(tableName, pubTableBroker, vs);
+         publishers.Add(file, vp);
+      }
+   }
+
+   public void Dispose() {
+      publishers.Values.DisposeAll();
+      publishers.Clear();
+   }
+}
@@ -0,0 +1,105 @@
+namespace Bts.DataStreamPublisher.Services;
+
+using BCG;
+
+using bts;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamExpressions;
+using Bts.PubSub;
+
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Options;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+/// <summary>
+/// An <see cref="IHostedService"/> that publishes <see cref="ViewStream"/>s to the PubSub.
+/// It can be added to the application's services via <see cref="DataStreamPublisherServices.AddDataStreamPublisher"/>
+/// </summary>
+public class DataStreamPublisherService : IHostedService, IDisposable {
+   readonly DataStreamPublisherConfig config;
+   readonly IPubTableBroker pubTableBroker;
+   readonly IWorkScheduler workScheduler;
+   List<IDisposable>? resources = new();
+
+   public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, IOptions<DataStreamPublisherConfig> config)
+      : this(pubTableBroker, workScheduler, config.Value) { }
+
+   public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, DataStreamPublisherConfig config) {
+      this.pubTableBroker = pubTableBroker;
+      this.workScheduler = workScheduler;
+      this.config = config;
+   }
+
+   public Task StartAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      resources = new();
+      foreach (var publisherNode in config.Publishers.empty_if_null()) {
+         foreach (var publishedStream in publisherNode.Streams.empty_if_null()) {
+            if (string.IsNullOrEmpty(publishedStream.Name)) {
+               Log.Gen.warn("Skipping PublishedStream because it doesn't have a name (must define 'Name' in config json).");
+               continue;
+            }
+            var vs = CreateViewStream(publishedStream);
+            if (vs is null) {
+               continue;
+            }
+            resources.Add(new ViewPublisher(publishedStream.Name, pubTableBroker, vs));
+         }
+      }
+      var csvPublishers = config.CsvDirs.empty_if_null().Select(csvDir => new CsvPublisher(csvDir, pubTableBroker)).ToList();
+      resources.AddRange(csvPublishers);
+      resources.Add(workScheduler.Subscribe(new CompositeWorker(csvPublishers)));
+      return Task.CompletedTask;
+   }
+
+   public Task StopAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Dispose();
+      return Task.CompletedTask;
+   }
+
+   public void Dispose() {
+      resources?.DisposeAll();
+      resources?.Clear();
+   }
+
+   static ViewStream? CreateViewStream(PublishedStream ps) {
+      Log.Gen.info(
+         $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
+      if (string.IsNullOrEmpty(ps.Name)) {
+         Log.Gen.warn(
+            $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
+         return null;
+      }
+      var vs = buildStream(ps.Expression);
+      if (vs is null) {
+         Log.Gen.warn(
+            $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
+         return null;
+      }
+      return vs;
+   }
+
+   static ViewStream? buildStream(string? streamExpression) {
+      if (streamExpression is null) {
+         return null;
+      }
+      try {
+         Log.Gen.info($"Building ViewStream: {streamExpression}");
+         var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression);
+         return exp.getView(new Dictionary<string, StatementExpression>(0));
+      } catch (Exception ex) {
+         Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}");
+         return null;
+      }
+   }
+}
@@ -0,0 +1,62 @@
+namespace Bts.DataStreamPublisher.Services;
+
+using BCG;
+using BCG.Utilities.General;
+
+using bts;
+
+using Bts.DataStream.Services;
+using Bts.DataStreamCore;
+using Bts.DataStreamCore.Services;
+using Bts.DataStreamExpressions;
+using Bts.PubSub.Services;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+
+using Spark.Api.Services;
+using Spark.Core.Services;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+
+public static class DataStreamPublisherServices {
+   public static IServiceCollection AddDataStreamPublisher(this IServiceCollection services, IConfigurationSection configurationSection) =>
+      services.Configure<DataStreamPublisherConfig>(configurationSection)
+         .AddHostedService<DataStreamPublisherService>();
+
+   public static IServiceCollection AddCsvPublishers(this IServiceCollection services, IConfigurationSection configurationSection) {
+      foreach (var csvDir in configurationSection.Get<List<string>>().empty_if_null()) {
+         services.AddSingleton<IWorker, CsvPublisher>(sp => new(csvDir, sp.GetRequiredService<IPubTableBroker>()));
+      }
+      return services;
+   }
+
+   public static IServiceCollection AddDataStreamPublisher(this IServiceCollection services, IConfiguration configurationSection) {
+      services.AddMasterWorker(configurationSection.GetSection("MasterWorker"))
+         .AddViewStreamDatabase(configurationSection.GetSection("ViewStreamDatabase"))
+         .AddPubSubClient(configurationSection.GetSection("PubSub"))
+         .AddPubTableBroker()
+         .AddRemoteQueryManager()
+         .AddDataStreamPublisher(configurationSection.GetSection("DataStreamPublisher"))
+         .AddSpark(configurationSection.GetSection("Spark"))
+         .AddSparkd(configurationSection.GetSection("Sparkd"));
+      return services;
+   }
+
+   static ViewStream? buildStream(string? streamExpression) {
+      if (streamExpression is null) {
+         return null;
+      }
+      try {
+         Log.Gen.info($"build_stream stream_expression: {streamExpression}");
+         var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression);
+         return exp.getView(new Dictionary<string, StatementExpression>(0));
+      } catch (Exception ex) {
+         Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}");
+         return null;
+      }
+   }
+}
@@ -17,14 +17,14 @@
    "publisher_routes": [
      {
        "name": "default",
-        "server_address": "bluto://192.168.1.80:12345",
+        "server_address": "bts-cloud://192.168.1.80:12345",
        "tag": ".*"
      }
    ],
    "subscriber_routes": [
      {
        "name": "default",
-        "server_address": "bluto://192.168.1.80:12345",
+        "server_address": "bts-cloud://192.168.1.80:12345",
        "tag": ".*"
      }
    ],
@@ -58,4 +58,4 @@
    "gc_activation_threshold": 0,
    "wait_period_after_gc": 1000
  }
-}
+}
@@ -36,6 +36,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStreamWebApp.Server", "Bts.DataStreamWebApp.Server\Bts.DataStreamWebApp.Server.csproj", "{6DE5E4CE-423E-4571-81F7-539D268D6059}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_websocket", "..\client\spark_websocket\spark_websocket.csproj", "{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -113,5 +117,13 @@ Global
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6DE5E4CE-423E-4571-81F7-539D268D6059}.Release|Any CPU.Build.0 = Release|Any CPU
+		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Debug|Any CPU.Build.0 = Debug|x64
+		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Release|Any CPU.ActiveCfg = Release|x64
+		{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Release|Any CPU.Build.0 = Release|x64
 	EndGlobalSection
 EndGlobal
@@ -1,17 +1,11 @@
-using System;
-using System.IO;
-using BCG.Utilities.General;
-using System.Collections.Generic;
-using mbts;
-
 namespace BCG {
-   public delegate void InternalPublishDelegate(ICollection<Triplet<string, string, byte[]>> pubData);
   /// <summary>
   /// The Pubsub holds an instance of this for every tag
   /// type. this interface holds the state of each tag
   /// in the pubsub.
   /// </summary>
-   public interface IPubSubServerTagHandler : IWorker {
+   public interface IPubSubServerTagHandler {
+      string TagType { get; }
      /// <summary>
      /// This is called when a new data gets published, so IPubSubServerTagHandler can
      ///interpret the data and update its state (which it will be asked to return in get_content)
@@ -25,17 +19,5 @@ namespace BCG {
      /// <param name="tag"></param>
      /// <returns></returns>
      byte[] get_content(string tag);
-      /// <summary>
-      /// list of tagTypes this interface supports
-      /// </summary>
-      string[] supportedTagTypes { get; }
-      /// <summary>
-      /// registering a callback gives this interface
-      /// the extra flexibility to publish its own
-      /// data back to the pubsub. this function will
-      /// be ignored executed outside the pubsub server
-      /// </summary>
-      /// <param name="callback"></param>
-      void register(InternalPublishDelegate callback);
   }
 }
@@ -0,0 +1,7 @@
+namespace Bts.PubSub;
+
+using BCG;
+
+public interface IPubSubTagHandlerFactory {
+   IPubSubServerTagHandler? GetHandler(string tagType);
+}
@@ -0,0 +1,42 @@
+namespace Bts.PubSub;
+
+using BCG;
+
+using System.Collections.Generic;
+
+public class PubSubTagTypeHandlerConfig {
+   public string? TagType { get; set; }
+   public string? Assembly { get; set; }
+   public string? Class { get; set; }
+}
+
+public class PubSubTagHandlerFactoryConfig {
+   public List<PubSubTagTypeHandlerConfig>? Handlers { get; set; }
+
+   public static PubSubTagHandlerFactoryConfig Default() {
+      return new() {
+         Handlers = new() {
+            new() {
+               TagType = string.Empty,
+               Assembly = "DataStreamCore",
+               Class = "bts.PubsubServerStreamTagHandler",
+            },
+            new() {
+               TagType = DSTagManager.DataStreamTagType,
+               Assembly = "DataStreamCore",
+               Class = "bts.PubsubServerStreamTagHandler",
+            },
+            new() {
+               TagType = "System_Sub",
+               Assembly = "PubSub",
+               Class = "BCG.SystemSubscriptionMapPublisher",
+            },
+            new() {
+               TagType = "System_Stats",
+               Assembly = "PubSub",
+               Class = "BCG.SystemStatsPublisher",
+            }
+         }
+      };
+   }
+}
@@ -1,76 +1,106 @@
 using System;
-using System.Collections;
 using System.Reflection;
-using BCG.Utilities;
 using System.Collections.Generic;
 using BCG;
-using System.Runtime.InteropServices;
 using System.IO;
-
-
+using System.Linq;
+using Bts.PubSub;
+using Microsoft.Extensions.Options;
+using System.Text.RegularExpressions;
+using System.Text.Json;
+using Microsoft.Extensions.DependencyInjection;

 namespace bts {
-   public static class PubsubTagHandlerFactory {
-      // each tag type is represented by this
-      public class TagHandlerEntry {
-         public readonly string tag_type;
-         public readonly string assembly_name;
-         public readonly string handler_class_name;
-         public TagHandlerEntry(string tag_type_, string assembly_name_, string handler_class_name_) {
-            tag_type = tag_type_;
-            assembly_name = assembly_name_;
-            handler_class_name = handler_class_name_;
-         }
+   public class PubsubTagHandlerFactory : IPubSubTagHandlerFactory {
+      readonly List<TagTypeHandlerData> handlers;
+      readonly IServiceProvider serviceProvider;
+
+      public PubsubTagHandlerFactory(IServiceProvider serviceProvider, IOptions<PubSubTagHandlerFactoryConfig> config)
+         : this(serviceProvider, config.Value) { }
+
+      public PubsubTagHandlerFactory(IServiceProvider serviceProvider, PubSubTagHandlerFactoryConfig config) {
+         this.serviceProvider = serviceProvider;
+         Log.notnull(config.Handlers);
+         handlers = config.Handlers.Select(h => new TagTypeHandlerData(h)).ToList();
      }
-      private static Dictionary<string, TagHandlerEntry> _config_map = new Dictionary<string, TagHandlerEntry>()
-      {
-         {String.Empty, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")},
-      {DSTagManager.DataStreamTagType, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")},
-      {"System_Sub", new TagHandlerEntry("System_Sub", "PubSub", "BCG.SystemSubscriptionMapPublisher")},
-      {"System_Stats", new TagHandlerEntry("System_Stats", "PubSub", "BCG.SystemStatsPublisher")}
+
+      public IPubSubServerTagHandler? GetHandler(string tag_type) {
+         if (_handler_map.TryGetValue(tag_type, out var handler)) {
+            return handler;
+         }
+         return get_handler(tag_type, true);
+      }
+
+      private static PubSubTagTypeHandlerConfig _wildcard_handler = new() {
+         TagType = "*",
+         Assembly = "DataStreamCore",
+         Class = "bts.PubsubServerGenericDataHandler",
      };
-      private static TagHandlerEntry _wildcard_handler = new TagHandlerEntry("*", "DataStreamCore", "bts.PubsubServerGenericDataHandler");
-      private static Dictionary<string, IPubSubServerTagHandler> _handler_map = new Dictionary<string, IPubSubServerTagHandler>();
+      private Dictionary<string, IPubSubServerTagHandler> _handler_map = new Dictionary<string, IPubSubServerTagHandler>();

-      public static string assembly_location => Assembly.GetExecutingAssembly().Location;
+      public string assembly_location => Assembly.GetExecutingAssembly().Location;

-      public static BCG.IPubSubServerTagHandler get_handler(string tag_type, bool on_pubsub_server) {
-         TagHandlerEntry handler_entry = _find_config_entry(tag_type);
-         Log.assert(handler_entry != null);
-         return _find_or_build_handler(handler_entry);
+      public BCG.IPubSubServerTagHandler get_handler(string tag_type, bool on_pubsub_server) {
+         var handler_entry = _find_config_entry(tag_type);
+         return _find_or_build_handler(handler_entry.Config);
      }

-      private static TagHandlerEntry _find_config_entry(string tag_type) {
-         TagHandlerEntry rv = null;
-         if (_config_map.TryGetValue(tag_type, out rv))
-            return rv;
-         else
-            return _wildcard_handler;
-      }
-
-      private static BCG.IPubSubServerTagHandler _find_or_build_handler(TagHandlerEntry entry) {
-         IPubSubServerTagHandler rv = null;
-         if (_handler_map.TryGetValue(entry.tag_type, out rv)) {
-            return rv;
+      private TagTypeHandlerData _find_config_entry(string tag_type) {
+         // Look for an exact match.
+         foreach (var handler in handlers) {
+            if (handler.TagType == tag_type) {
+               return handler;
+            }
         }
-         {
+         // If an exact match wasn't found, look for a regex match
+         foreach (var handler in handlers) {
+            if (handler.TagTypeRegex.IsMatch(tag_type)) {
+               return handler;
+            }
+         }
+         Log.assert(false);
+         throw new InvalidOperationException($"Failed to handler config for tag type '{tag_type}'");
+      }
+
+      private BCG.IPubSubServerTagHandler _find_or_build_handler(PubSubTagTypeHandlerConfig entry) {
+         ArgumentNullException.ThrowIfNull(entry.TagType);
+         lock (_handler_map) {
+            if (_handler_map.TryGetValue(entry.TagType, out var rv)) {
+               return rv;
+            }
            rv = _build_handler(entry);
-            _handler_map.Add(entry.tag_type, rv);
+            _handler_map.Add(entry.TagType, rv);
+            return rv;
         }
-         return rv;
      }

-      private static BCG.IPubSubServerTagHandler _build_handler(TagHandlerEntry entry) {
+      private BCG.IPubSubServerTagHandler _build_handler(PubSubTagTypeHandlerConfig entry) {
+         Log.notnull(entry.Assembly);
+         Log.notnull(entry.Class);
         var ass_loc = assembly_location;
         bts.Log.Gen.info($"PubsubTagHandlerFactory.load_handler entry: {entry} assembly_location: {ass_loc}");
-         var assem = Assembly.LoadFrom(Path.ChangeExtension(entry.assembly_name, "dll"))
+         var assem = Assembly.LoadFrom(Path.ChangeExtension(entry.Assembly, "dll"))
            ?? throw new ArgumentException("Failed to instantiate TagHandlerEntry");
         bts.Log.Gen.info($"PubsubTagHandlerFactory.load_handler successfully loaded assembly: {assem}");
-         Type t = assem.GetType(entry.handler_class_name, true, true);
-         object obj = Activator.CreateInstance(t);
-         BCG.IPubSubServerTagHandler tag_handler = obj as BCG.IPubSubServerTagHandler;
-         bts.Log.Gen.info($"PubsubTagHandlerFactory.load_handler successfully created obj: {obj} of type: {t} from assem: {assem}");
+         var t = assem.GetType(entry.Class, true, true);
+         Log.notnull(t);
+         var obj = ActivatorUtilities.GetServiceOrCreateInstance(serviceProvider, t);
+         var tag_handler = Log.notnull(obj as IPubSubServerTagHandler);
+         Log.Gen.info($"PubsubTagHandlerFactory created handler. {JsonSerializer.Serialize(entry)}");
         return tag_handler;
      }
+
+      readonly struct TagTypeHandlerData {
+         public TagTypeHandlerData(PubSubTagTypeHandlerConfig config) {
+            Config = config;
+            Log.notnull(config.TagType);
+            TagType = config.TagType;
+            TagTypeRegex = new(TagType);
+         }
+
+         public PubSubTagTypeHandlerConfig Config { get; }
+         public string TagType { get; }
+         public Regex TagTypeRegex { get; }
+      }
   }
 }
@@ -1,3 +1,5 @@
+using bts;
+
 using mbts.utils;

 using System;
@@ -8,7 +10,7 @@ using System.Text.RegularExpressions;

 namespace BCG;

-public class PubSubRoute : IEquatable<PubSubRoute>, IComparable<PubSubRoute> {
+public class PubSubRoute : IEquatable<PubSubRoute>, IComparable<PubSubRoute>, ICloneable<PubSubRoute> {
   [JsonPropertyName("server_address")]
   public string? ServerAddress { get; set; }

@@ -42,6 +44,13 @@ public class PubSubRoute : IEquatable<PubSubRoute>, IComparable<PubSubRoute> {
      return myRegex.IsMatch(tag);
   }

+   public PubSubRoute Clone() =>
+      new() {
+         ServerAddress = ServerAddress,
+         TagRegExp = TagRegExp,
+         Name = Name,
+      };
+
   public int CompareTo(PubSubRoute? other) =>
      other is null ? -1 : comparer.Compare(this, other);

@@ -1,14 +1,15 @@
-using System;
-using System.Text.RegularExpressions;
-using System.Collections;
-using System.Collections.Generic;
-
-using BCG;
-using Bts.PubSub;
 using bts;
+
+using Bts.PubSub;
+using Bts.PubSub.Routing;
+
+using mbts.utils;
+
+using Microsoft.Extensions.Options;
+
+using System;
+using System.Collections.Generic;
 using System.Linq;
-//using BCG.Utilities;
-//using bts;

 namespace BCG {
   public class PubSubRouter : IPubSubRouter {
@@ -16,13 +17,15 @@ namespace BCG {
      private PubSubRoute[] _pub_routes;
      private PubSubRoute[] _sub_routes;

-      public PubSubRouter() : this(
-         PubSubConfig.Instance.SubscriberRoutes?.ToArray()
-            ?? throw new InvalidOperationException("Subscriber routes are not configured"),
-         PubSubConfig.Instance.PublisherRoutes?.ToArray()
-            ?? throw new InvalidOperationException("Publisher routes are not configured")) { }
+      public PubSubRouter()
+         : this(Log.notnull(PubSubConfig.Instance.Routing)) { }

-      public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes) { }
+      public PubSubRouter(IOptions<PubSubRouterConfig> config)
+         : this(config.Value) { }
+
+      public PubSubRouter(PubSubRouterConfig config) : this(
+         ExceptionAssert.Argument.NotNullOrEmpty(config.SubscriberRoutes?.ToArray()),
+         ExceptionAssert.Argument.NotNullOrEmpty(config.PublisherRoutes?.ToArray())) { }

      public PubSubRouter(PubSubRoute[] sub_routes, PubSubRoute[] pub_routes) {
         string log_msg = String.Empty;
@@ -0,0 +1,40 @@
+namespace Bts.PubSub.Routing;
+
+using BCG;
+
+using bts;
+
+using System.Collections.Generic;
+using System.Linq;
+
+public class PubSubRouterConfig : ICloneable<PubSubRouterConfig> {
+   public List<PubSubRoute>? PublisherRoutes { get; set; }
+
+   public List<PubSubRoute>? SubscriberRoutes { get; set; }
+
+   public void MergeWith(PubSubRouterConfig other) {
+      PublisherRoutes = MergeRoutes(PublisherRoutes, other.PublisherRoutes);
+      SubscriberRoutes = MergeRoutes(SubscriberRoutes, other.SubscriberRoutes);
+
+      List<PubSubRoute>? MergeRoutes(List<PubSubRoute>? routes, List<PubSubRoute>? otherRoutes) {
+         if (otherRoutes is null) {
+            return routes;
+         }
+         if (routes is null) {
+            return otherRoutes;
+         }
+         foreach (var route in otherRoutes) {
+            if (!routes.Any(r => r.TagRegExp == route.TagRegExp || r.Name == route.Name)) {
+               routes.Add(route);
+            }
+         }
+         return routes;
+      }
+   }
+   
+   public PubSubRouterConfig Clone() =>
+      new() {
+         PublisherRoutes = PublisherRoutes?.Select(r => r.Clone()).ToList(),
+         SubscriberRoutes = SubscriberRoutes?.Select(r => r.Clone()).ToList(),
+      };
+}
@@ -0,0 +1,7 @@
+namespace Bts.PubSub;
+
+using System;
+
+public interface IManagedSubscriber {
+   void Subscribe(string tag, Action<string, byte[]> onData);
+}
@@ -5,6 +5,7 @@ using System.Linq;

 using BCG.Utilities.General;

+using Bts.PubSub;

 using OnData = System.Action<string, byte[]>;

@@ -16,9 +17,9 @@ namespace BCG {
   /// Provides a callback based interface to the SubscriptionCore (this class is mainly here to allow backward compatibility
   /// with PubSub, as some clients are still using ManagedSubcribers)
   /// </summary>
-   public class ManagedSubscriber : IWorker, IDisposable {
+   public class ManagedSubscriber : IManagedSubscriber, IWorker, IDisposable {
      private static ManagedSubscriber? _self;
-      private SubscriberCore _subcore;
+      private ISubscriberCore _subcore;
      // private IDictionary myTypedDispatchTable = new Hashtable(),
      //    myRawDispatchTable = new Hashtable();
      private Dictionary<string, List<OnData>> _dispatch_table = new Dictionary<string, List<OnData>>();
@@ -26,7 +27,7 @@ namespace BCG {

      public ManagedSubscriber() : this(new SubscriberCore()) { }

-      public ManagedSubscriber(SubscriberCore subsCore) { _subcore = subsCore; }
+      public ManagedSubscriber(ISubscriberCore subsCore) { _subcore = subsCore; }

      public void doWork() {
         var msg_map = new Dictionary<string, List<bts.PubSubServerMessage>>();
@@ -82,6 +83,9 @@ namespace BCG {
         }
      }

+      void IManagedSubscriber.Subscribe(string tag, OnData onData) =>
+         subscribe(tag, onData);
+
      private void _fire_events(IEnumerable<bts.PubSubServerMessage> updates) {
         // TODO
         foreach (var u in updates) {
@@ -4,9 +4,10 @@ using BCG;

 using bts;

+using Bts.PubSub.Routing;
+
 using System;
 using System.Collections.Generic;
-using System.Text.Json.Serialization;

 public class PubSubConfig {
   static PubSubConfig? instance;
@@ -21,79 +22,62 @@ public class PubSubConfig {
      instance = Log.notnull(cfg);
   }

-   [JsonPropertyName("server_port")]
   public short? ServerPort { get; set; }

-   [JsonPropertyName("simple_echo_server")]
   public bool? SimpleEchoServer { get; set; }

   // unused
-   [JsonPropertyName("milliseconds_between_sub_info_updates")]
   public int? MillisecondsBetweenSubInfoUpdates { get; set; } = 1000;

   // unused
-   [JsonPropertyName("milliseconds_between_time_slice_updates")]
   public int? MillisecondsBetweenTimeSliceUpdates { get; set; } = 5000;

   // unused
-   [JsonPropertyName("milliseconds_between_no_buffer_space_retries")]
   public int? MillisecondsBetweenNoBufferRetries { get; set; } = 100;

   // unused
-   [JsonPropertyName("retries_on_no_buffer_space")]
   public int? RetriesOnNoBufferSpace { get; set; } = 1;

   // unused
-   [JsonPropertyName("do_it_all_seconds_between_log")]
   public int? DoItAllSecondsBetweenLog { get; set; } = 0;

   // Used by the PubSubMessageReactor to pass to the ManagedTCPMessageHb (but unsed within it)
-   [JsonPropertyName("heartbeat_interval")]
   public int? HeartbeatInterval { get; set; } = 15000;

-   [JsonPropertyName("missing_heartbeats_before_gc")]
   public int? MissingHeartbeatsBeforeGc { get; set; } = 50;

   // unused
   // Milliseconds
-   [JsonPropertyName("state_update_period")]
   public int? StateUpdatePeriod { get; set; } = 5000;

   // unused
   // Milliseconds
-   [JsonPropertyName("statistics_update_period")]
   public int? StatisticsUpdatePeriod { get; set; } = 5000;

   // Milliseconds
-   [JsonPropertyName("disposed_connection_check_period")]
   public int? DisposedConnectionCheckPeriod { get; set; } = 1000;

-   [JsonPropertyName("tag_type_to_assembly_map")]
   public TagTypeToAssembly? TagTypeToAssemblyMap { get; set; }

+   public PubSubTagHandlerFactoryConfig? TagHandlerFactory { get; set; }
+
   // unused
-   [JsonPropertyName("latency_report_threshold_ms")]
   public int? LatencyReportThresholdMs { get; set; }

   // unused
-   [JsonPropertyName("internally_publish_sub_info")]
   public bool? InternallyPublishSubInfo { get; set; }

-   [JsonPropertyName("compressed_tags")]
   public string? CompressedTags { get; set; }

-   [JsonPropertyName("gc_activation_threshold")]
   public int? GcActivationThreshold { get; set; }

   // Milliseconds
-   [JsonPropertyName("wait_period_after_gc")]
   public int? WaitPeriodAfterGc { get; set; }

-   [JsonPropertyName("publisher_routes")]
-   public List<PubSubRoute>? PublisherRoutes { get; set; }
+   public PubSubRouterConfig? Routing { get; set; }

-   [JsonPropertyName("subscriber_routes")]
-   public List<PubSubRoute>? SubscriberRoutes { get; set; }
+   public List<PubSubRoute>? PublisherRoutes => Routing?.PublisherRoutes;
+   public List<PubSubRoute>? SubscriberRoutes => Routing?.SubscriberRoutes;

   public static PubSubConfig Default() {
      const string NOTAGS = "^NOTAGS$";
@@ -110,17 +94,20 @@ public class PubSubConfig {
         DisposedConnectionCheckPeriod = 1000,
         ServerPort = 12345,
         TagTypeToAssemblyMap = new(),
+         TagHandlerFactory = PubSubTagHandlerFactoryConfig.Default(),
         LatencyReportThresholdMs = 1500,
         InternallyPublishSubInfo = true,
         CompressedTags = NOTAGS,
         GcActivationThreshold = 0,
         WaitPeriodAfterGc = 1000,
         SimpleEchoServer = false,
-         PublisherRoutes = new() {
-            new("bts-cloud://127.0.0.1:12345", ".*")
-         },
-         SubscriberRoutes = new() {
-            new("bts-cloud://127.0.0.1:12345", ".*")
+         Routing = new() {
+            PublisherRoutes = new() {
+               new("bts-cloud://127.0.0.1:12345", ".*")
+            },
+            SubscriberRoutes = new() {
+               new("bts-cloud://127.0.0.1:12345", ".*")
+            }
         }
      };
      return rv;
@@ -139,12 +126,20 @@ public class PubSubConfig {
      DisposedConnectionCheckPeriod ??= other.DisposedConnectionCheckPeriod;
      ServerPort ??= other.ServerPort;
      TagTypeToAssemblyMap ??= other.TagTypeToAssemblyMap;
+      TagHandlerFactory ??= other.TagHandlerFactory;
      LatencyReportThresholdMs ??= other.LatencyReportThresholdMs;
      InternallyPublishSubInfo ??= other.InternallyPublishSubInfo;
      CompressedTags ??= other.CompressedTags;
      GcActivationThreshold ??= other.GcActivationThreshold;
      WaitPeriodAfterGc ??= other.WaitPeriodAfterGc;
      SimpleEchoServer ??= other.SimpleEchoServer;
+      if (other.Routing is not null) {
+         if (Routing is null) {
+            Routing = other.Routing?.Clone();
+         } else {
+            Routing.MergeWith(other.Routing);
+         }
+      }
   }
 }

@@ -50,7 +50,7 @@ namespace bts {
         _tcp_client = tcp_client;
         _nw_stream = _tcp_client.GetStream();
         _connection_id = new ConnectionID(TCPSocketUtils.get_self_address(_tcp_client.Client), TCPSocketUtils.get_peer_address(_tcp_client.Client), GetType());
-         Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons connection: {_connection_id}");
+         Log.Gen.info($"AsyncTCPConnection_AsyncWrapper.cons connection: {_connection_id}");
      }

      private void _detect_connection_status_change() {
@@ -0,0 +1,18 @@
+namespace Bts.PubSub;
+
+using BCG.Utilities.General;
+
+using System.Collections.Generic;
+
+public class CompositeWorker : IWorker {
+   readonly List<IWorker> workers;
+
+   public CompositeWorker(IEnumerable<IWorker> workers) =>
+      this.workers = new(workers);
+
+   public void doWork() {
+      foreach (var worker in workers) {
+         worker.doWork();
+      }
+   }
+}
@@ -0,0 +1,10 @@
+namespace Bts.PubSub;
+
+using BCG.Utilities.General;
+
+using System;
+
+public interface IParameterizedWorker : IWorker {
+   string? Group => null;
+   TimeSpan Interval => TimeSpan.Zero;
+}
@@ -0,0 +1,10 @@
+namespace Bts.PubSub;
+
+using BCG.Utilities.General;
+
+using System;
+
+public interface IWorkScheduler {
+   void Add(IWorker worker, TimeSpan interval);
+   void Remove(IWorker worker);
+}
@@ -2,6 +2,8 @@ using BCG.Utilities.General;

 using bts;

+using Bts.PubSub;
+
 using mbts;

 using System;
@@ -9,7 +11,7 @@ using System.Collections.Generic;
 using System.Diagnostics;

 namespace BCG {
-   public class MasterWorker : IWorker {
+   public class MasterWorker : IWorker, IWorkScheduler {

      private DateTime myLastDoWorkTime = DateTime.MinValue,
                        myLastLogTime = DateTime.MinValue;
@@ -86,6 +88,12 @@ namespace BCG {

      }

+      void IWorkScheduler.Add(IWorker worker, TimeSpan interval) =>
+         addWorker(worker, interval);
+
+      void IWorkScheduler.Remove(IWorker worker) =>
+         removeWorker(worker);
+
      protected virtual void myLogEntries() {
         if (DateTime.UtcNow - myLastLogTime > myLogPeriod) {
            myLastLogTime = DateTime.UtcNow;
@@ -0,0 +1,19 @@
+namespace Bts.PubSub;
+
+using BCG.Utilities.General;
+
+using Spark.Core.Disposables;
+
+using System;
+
+public static class WorkSchedulerExtensions {
+   public static void Add(this IWorkScheduler scheduler, IWorker worker) =>
+      scheduler.Add(worker, (worker as IParameterizedWorker)?.Interval ?? TimeSpan.Zero);
+
+   public static IDisposable Subscribe(this IWorkScheduler scheduler, IWorker worker, TimeSpan? interval = null) {
+      scheduler.Add(worker, interval ?? TimeSpan.Zero);
+      return Disposable.Create(Dispose);
+      void Dispose() =>
+         scheduler.Remove(worker); 
+   }
+}
@@ -1,5 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">

+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
+  </ItemGroup>
+
  <ItemGroup>
    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
  </ItemGroup>
@@ -1,3 +1,5 @@
+using Bts.PubSub;
+
 using System;
 using System.Collections.Generic;

@@ -8,14 +10,19 @@ namespace bts {
   // PublisherCore calls get_content to get the data to re-publish, to 
   // re-seed the PubSubServer
   public class PublishedCache {
+      readonly IPubSubTagHandlerFactory tagHandlerFactory;
      private Dictionary<string, SingleTagData> _data_map = new Dictionary<string, SingleTagData>();

+      public PublishedCache(IPubSubTagHandlerFactory tagHandlerFactory) =>
+         this.tagHandlerFactory = tagHandlerFactory;
+
      public void publish(PublishMsg msg) {
         Log.Gen.info($"PublishedCache.publish msg: {msg}");
         ArgumentException.ThrowIfNullOrEmpty(msg.tag);
         if (!_data_map.TryGetValue(msg.tag, out var tag_data)) {
            ArgumentException.ThrowIfNullOrEmpty(msg.tag_type);
-            tag_data = new SingleTagData(msg.tag, msg.tag_type);
+            var handler = Log.notnull(tagHandlerFactory.GetHandler(msg.tag_type));
+            tag_data = new SingleTagData(msg.tag, msg.tag_type, handler);
            _data_map.Add(msg.tag, tag_data);
         }
         tag_data.handle_publish(msg);
@@ -3,6 +3,8 @@ using System.IO;
 using System.Collections.Generic;
 using System.Threading;
 using bts;
+using Bts.PubSub;
+using Bts.PubSub.Services;

 namespace BCG {

@@ -21,7 +23,7 @@ namespace BCG {
      // all the data published through this instance of PublisherCore
      // is kept there, in a form that can be used repopulate a pubsub server
      // in the event of a conection re-establishment to the PubSubServer
-      protected PublishedCache _state = new PublishedCache();
+      protected PublishedCache _state;

      // every PubSubServer is communicated with a adifferent IPubSubServerConnection object.

@@ -41,15 +43,11 @@ namespace BCG {

      #region constructors

-      public PublisherCore()
-         // TODOds
-         : this(new PubSubRouter()) { }
-
-      public PublisherCore(string server_url)
-         : this(new PubSubRouter(new PubSubRoute(server_url, ".*"))) { }
-
-      public PublisherCore(IPubSubRouter r) {
-         _router = r;
+      public PublisherCore(IPubSubRouter? router = null, IPubSubTagHandlerFactory? tagHandlerFactory = null) {
+         router ??= new PubSubRouter();
+         tagHandlerFactory ??= new PubsubTagHandlerFactory(PubSubServices.ServiceProvider, PubSubTagHandlerFactoryConfig.Default());
+         _router = router;
+         _state = new(tagHandlerFactory);
         Thread pump_thread = new Thread(new ThreadStart(_message_pump));
         pump_thread.IsBackground = true;
         pump_thread.Start();
@@ -114,6 +112,10 @@ namespace BCG {

      #endregion

+      // static void setGlobalInstance() {
+         
+      // }
+
      #region Message Pump Thread - this is where messages are dequeued and the action performed
      // all the state manahement happens here, the public interface methods merely enqueue the operation
      private void _message_pump() {
@@ -0,0 +1,7 @@
+namespace Bts.PubSub.Server;
+
+using bts;
+
+interface IInternalPublisher {
+   void Publish(PublishMsg message);
+}
@@ -0,0 +1,16 @@
+namespace Bts.PubSub.Server;
+
+using bts;
+
+using System;
+using System.Collections.Generic;
+
+/// <summary>
+/// An instance of this interface is passed to the (<see cref="InternalPublisher"/>) so it can enqueue 
+/// messages to be published. It is also passed to the <see cref="BCG.PubSubMessageReactor"/>, so it
+/// can dequeue these messages and send them over the wire.
+/// </summary>
+interface IInternalPublisherQueue {
+   void Enqueue(PublishMsg message);
+   List<PublishMsg> DequeueMultiple(TimeSpan ts, int maxElements);
+}
Author	SHA1	Message	Date
John Colvin	ba664b95b8	fix use dep injection in bts.datastreamwebapp.server	2024-07-11 11:06:29 -04:00
John Colvin	d6f370f862	fix Add spark service modules designed for DependencyInjection	2024-07-11 11:05:23 -04:00
John Colvin	079826f4de	Fix mismatched types in PubsubHeartbeatStream schema	2024-07-11 10:59:41 -04:00
John Colvin	2eee3cdaab	Change DataStreamPublisher json fields to match cs class property names verbatim	2024-07-11 10:43:24 -04:00
John Colvin	5e3215ee52	fix: Always call 'handle_internal_pubs' in PubSubMessageReactor main loop	2024-07-11 10:38:30 -04:00
John Colvin	7739cc2f3d	Use Dependency Injection in Bts.DataStreamWebApp.Server	2024-07-11 10:37:29 -04:00
John Colvin	8347a7c126	refactor: Change Bts.DataStreamWebApp.Server framework from net8 -> net7 Although it would build on net8, all other cs code targets net7. Furthermore, VS Code intellisense seems less reliable/crashes frequently when the sln file targets both frameworks).	2024-07-11 10:37:28 -04:00
John Colvin	fbfc960df6	improvement: Make DataStreamSubscription an IHostedService To properly manage its life cycle	2024-07-11 10:37:28 -04:00
John Colvin	451f721030	improvement: Accept CancellationToken, catch exceptions when running ds websocket server	2024-07-11 10:37:28 -04:00
John Colvin	bb992c0410	Fix mismatched types in PubSubServerTagStatsStream schema	2024-07-11 10:37:28 -04:00
John Colvin	db18aebf92	Add DataStreamPublisher service modules designed for Dependency Injection	2024-07-11 10:37:28 -04:00
John Colvin	8127c9c9fe	refactor: Decouple ViewPublisher from IPubTableBroker impl	2024-07-11 10:37:28 -04:00
John Colvin	1825b339b1	Change PubSubServer json fields to match cs class property names verbatim	2024-07-11 10:37:28 -04:00
John Colvin	03ab3fbe90	Get rid of JsonPropertyName attributes in PubSubConfig The Dependency Injection framework (based around dotnet ApplicationBuilder) does not support this attribute, and maps property names verbatim to json fields.	2024-07-11 10:37:28 -04:00
John Colvin	b61bf16a42	Add PubSub service modules designed for Dependency Injection	2024-07-11 10:37:28 -04:00
John Colvin	d969489bf4	refactor: Decouple RemoteQueryManager from IManagedSubscriber, IPubTableBroker impl	2024-07-11 10:37:28 -04:00
John Colvin	9cd4974f49	refactor: Decouple ManagedSubscriber from ISubscriberCore impl	2024-07-11 10:37:28 -04:00
John Colvin	793634c8cb	refactor: Only support 1 tag type in IPubSubServerTagHandler	2024-07-11 10:37:28 -04:00
John Colvin	eae20b6c94	refactor: Decouple PubTableBroker from IPublisherCore impl Reference interface rather than the PublisherCore impl.	2024-07-11 10:37:28 -04:00
John Colvin	2848bb6a5d	improvement: Accept CancellationToken in JsonRpcRequestObservable	2024-07-11 10:37:16 -04:00
John Colvin	687324998b	Add spark service modules designed for Dependency Injection	2024-07-11 10:36:04 -04:00
John Colvin	97cebd62d4	fix: Dispose market data sub in various md streams	2024-07-11 10:36:04 -04:00
John Colvin	492ca0c28f	cleanup: Remove unused pubsub tag handler code * Removed 'IPubSubServerTagHandler.register' - All implementations of this function were empty * Removed 'IPubSubServerTagHandler.supportedTagTypes' - This property wasn't referenced anywhere * Remove IWorker inheritance from IPubSubServerTagHandler - No reason these interfaces have to be tied together	2024-07-10 13:35:18 -04:00
John Colvin	baa5019039	refactor: Rename CloudStreamManager.cs -> RemoteQueryManager.cs This matches the file name	2024-07-10 01:18:21 -04:00
John Colvin	ac2d31e1fc	fix build: Update vcpkg On some vs versions, vcpkg cmake scripts would fail w/ an error saying "Could not locate a complete Visual Studio instance"	2024-07-09 15:16:36 -04:00
John Colvin	9821134bc3	Change pub sub schema prefix 'bluto://' -> 'bts-cloud://' In config json files (not actual code change)	2024-07-09 15:14:30 -04:00
John Colvin	9cf892322d	fix build: Fix prop names after refactoring PubSubRoute * Change 'server_address' -> 'ServerAddress' * Change 'tag_reg_exp' -> 'TagRegExp'	2024-07-09 15:14:30 -04:00
John Colvin	77bf91be1e	build: Add scripts to publish data stream apps	2024-07-09 15:14:30 -04:00
John Colvin	63b064ac79	Fix typo in log msg	2024-07-09 15:14:30 -04:00
John Colvin	b617775c29	Fix net7 -> net8 typo in publish_data_stream_web_app.py The DataStream Web App is on net8	2024-07-09 15:14:30 -04:00
John Colvin	4f7d77420c	Fix build scripts so they propagate the exit code back to the shell I noticed this was broken because launch.json configurations would still start the debugger even when the preLaunchTask (build) failed!	2024-07-09 15:14:30 -04:00
John Colvin	3ba6a91ca7	vscode: Add DataStreamPublisher debug cfg to skip build	2024-07-09 15:14:30 -04:00
John Colvin	0abde9eb7c	build: Use separate cmd to build DataStreamSuite/QueryAnalyzer.sln The DataStreamSuite.sln builds the new webapp now, which will (eventually) replace DSQA. We'll keep it around for now, but with a clean separation from the main build.	2024-07-09 15:14:30 -04:00
John Colvin	1531a57bc5	feat: Add DataStream WebApp Included in DataStreamSuite.sln, but not DataStreamQueryAnalyzer.sln	2024-07-09 15:14:29 -04:00
John Colvin	a62ea64c5e	build: Copy libspark to all dotnet dirs in single cmake target This is cleaner than the previous impl, which used separate build targets to copy libspark to "net7.0", and "net7.0-windows" (dirs). Note: I decided to clean this up now because I'm about to upgrade the new DataStream WebApp to net8.0, and did not want to add yet another build target to copy libspark to "net8.0".	2024-07-09 15:14:29 -04:00
				`@@ -0,0 +1 @@`
				<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>