Add: Incremental Books

Remove: code to remove secuties
Add: code to connect to Live databento
2024-10-04 17:05:02 +00:00 · 2024-10-03 18:03:10 +05:00 · 2024-10-03 18:03:08 +05:00 · 2024-10-03 18:01:49 +05:00 · 2024-10-03 18:01:49 +05:00 · 2024-10-03 18:01:49 +05:00
499 changed files with 10702 additions and 18958 deletions
@@ -66,7 +66,6 @@ RUN \

 # Dotnet
 RUN apt-get update && apt-get install -y dotnet7
-RUN apt-get update && apt-get install -y dotnet8

 # Node
 ARG node_major=18
@@ -88,6 +87,9 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev

+# Databento (deps)
+RUN apt-get update && apt-get -y install libzstd-dev
+
 # Set up default executables (update-alternatives)
 RUN \
   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
@@ -19,14 +19,6 @@
    "--shm-size",
    "2gb",
    "-p",
-    "5000:5000",
-    "-p",
-    "5001:5001",
-    "-p",
-    "5173:5173",
-    "-p",
-    "5174:5174",
-    "-p",
    "8000:8000",
    "-p",
    "8001:8001",
@@ -35,7 +27,7 @@
    "-p",
    "8003:8003",
    "-p",
-    "12345:12345",
+    "12345:12345"
  ],
  "mounts": [
    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
@@ -55,4 +47,4 @@
      ]
    }
  }
-}
+}
@@ -207,23 +207,17 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private

-# local symbols
-dotnet_naming_symbols.local_symbols.applicable_kinds = local
-
-# parameter symbols
-dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
-
 # Naming styles

-dotnet_naming_style.pascal_case.required_prefix =
-dotnet_naming_style.pascal_case.required_suffix =
-dotnet_naming_style.pascal_case.word_separator =
-dotnet_naming_style.pascal_case.capitalization = pascal_case
+dotnet_naming_style.snake_case.required_prefix =
+dotnet_naming_style.snake_case.required_suffix =
+dotnet_naming_style.snake_case.word_separator = _
+dotnet_naming_style.snake_case.capitalization = all_lower

-dotnet_naming_style.camel_case.required_prefix =
-dotnet_naming_style.camel_case.required_suffix =
-dotnet_naming_style.camel_case.word_separator =
-dotnet_naming_style.camel_case.capitalization = camel_case
+dotnet_naming_style.private_snake_case.required_prefix = _
+dotnet_naming_style.private_snake_case.required_suffix =
+dotnet_naming_style.private_snake_case.word_separator = _
+dotnet_naming_style.private_snake_case.capitalization = all_lower

 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -235,10 +229,6 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case

-dotnet_naming_style.local.capitalization = camel_case
-
-dotnet_naming_style.parameter.capitalization = camel_case
-
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -248,18 +238,14 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion

-dotnet_naming_rule.public_members.symbols = public_symbols
-dotnet_naming_rule.public_members.style = pascal_case
-dotnet_naming_rule.public_members.severity = suggestion
+dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
+dotnet_naming_rule.public_members_snake_case.style = snake_case
+dotnet_naming_rule.public_members_snake_case.severity = suggestion

-dotnet_naming_rule.private_members.symbols = private_symbols
-dotnet_naming_rule.private_members.style = camel_case
-dotnet_naming_rule.private_members.severity = suggestion
+dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
+dotnet_naming_rule.private_members_snake_case.style = private_snake_case
+dotnet_naming_rule.private_members_snake_case.severity = suggestion

-dotnet_naming_rule.variables_are_camel_case.severity = suggestion
-dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
-dotnet_naming_rule.variables_are_camel_case.style = local
-
-dotnet_naming_rule.parameters.symbols = parameter_symbols
-dotnet_naming_rule.parameters.style = parameter
+dotnet_naming_rule.parameters.symbols = parameters
+dotnet_naming_rule.parameters.style = snake_case
 dotnet_naming_rule.parameters.severity = suggestion
@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = https://bitbucket.org/bluetradesys/externals.git
+	url = git@bitbucket.org:bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git
@@ -6,35 +6,6 @@
      "type": "coreclr",
      "request": "attach"
    },
-    {
-      "name": "Bts.DataStreamWebApp.Server",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream WebApp (Debug)",
-      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
-      "args": [
-        "--sparkd",
-        "http://localhost:8000/config",
-      ],
-      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-    {
-      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "${workspaceFolder}/build/Debug/bin/net8.0/Bts.DataStreamWebApp.Server.dll",
-      // "args": [
-      //   "--config",
-      //   "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      // ],
-      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
    {
      "name": "DataStreamPublisher",
      "type": "coreclr",
@@ -51,34 +22,27 @@
        "--config",
        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
      ],
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+      "cwd": "build/Debug/bin/net7.0",
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
    {
-      "name": "DataStreamPublisher (Skip Build)",
-      "type": "coreclr",
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "DataStreamPublisher",
-      },
-      "windows": {
-        "program": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
-      ],
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": ["run"],
      "stopAtEntry": false,
-      "console": "integratedTerminal"
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
    },
    {
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
      "request": "launch",
-      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
      "program": "",
      "windows": {
        "program": "DataStreamQueryAnalyzer.exe",
@@ -114,25 +78,6 @@
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
-    {
-      "name": "PubSubServer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "PubSubServer",
-      },
-      "windows": {
-        "program": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
-      "cwd": "build/Debug/bin/net7.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
    {
      "name": "Spark",
      "type": "coreclr",
@@ -210,4 +155,4 @@
      "default": "http://localhost:8000/config",
    },
  ]
-}
+}
@@ -121,97 +121,6 @@
        "color": "terminal.ansiCyan"
      },
    },
-    {
-      "label": "Build DataStreamQueryAnalyzer",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream WebApp",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream_web_app.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream WebApp (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream_web_app.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
    {
      "label": "Build Spark",
      "group": "build",
@@ -394,22 +303,6 @@
        "color": "terminal.ansiCyan"
      },
    },
-    {
-      "label": "Publish DataStreamWebApp",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/publish_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/publish_data_stream_web_app.cmd",
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
    {
      "label": "Edit DataStream Config",
      "type": "process",
@@ -460,7 +353,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStreamQueryAnalyzer",
+      "dependsOn": "Build DataStream",
      "type": "process",
      "command": "",
      "windows": {
@@ -480,7 +373,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
      "type": "process",
      "command": "",
      "windows": {
@@ -500,7 +393,7 @@
    },
    {
      "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
+      "dependsOn": "Build DataStream (Release)",
      "type": "process",
      "command": "",
      "windows": {
@@ -132,6 +132,8 @@ else()
      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
      COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
      COMMAND
        ln ARGS -sf
        ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
@@ -31,7 +31,8 @@
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -221,7 +222,6 @@
        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    },
@@ -258,7 +258,7 @@
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -8,6 +8,7 @@
   public int Integer;
   public double Real;
   public string Identifier;
+   public DateTime DateTime; 
   public ImmutableList<IArithmeticExpression> Arguments;
   public ImmutableList<Assignment> Assignments;
   public IArithmeticExpression Argument;
@@ -1,7 +1,4 @@
 using System;
-using System.Runtime.InteropServices;
-using System.Runtime.Versioning;
-
 using Microsoft.Win32;

 namespace BCG
@@ -13,11 +10,13 @@ namespace BCG
         rk.SetValue (name, value.ToString ());
      }

-      [SupportedOSPlatform("windows")]
      public static double getValue (RegistryKey rk, string name)
      {
-         var str = rk.GetValue(name)?.ToString();
-         return str is not null && double.TryParse(str, out var dbl) ? dbl : double.NaN;
+         object v = rk.GetValue (name);
+         if (v != null) 
+            return double.Parse (v.ToString ());
+         else
+            return double.NaN;
      }
   }
 }
@@ -1,20 +0,0 @@
-module.exports = {
-  root: true,
-  env: { browser: true, es2020: true },
-  extends: [
-    'eslint:recommended',
-    'plugin:react/recommended',
-    'plugin:react/jsx-runtime',
-    'plugin:react-hooks/recommended',
-  ],
-  ignorePatterns: ['dist', '.eslintrc.cjs'],
-  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
-  settings: { react: { version: '18.3.1' } },
-  plugins: ['react-refresh'],
-  rules: {
-    'react-refresh/only-export-components': [
-      'warn',
-      { allowConstantExport: true },
-    ],
-  },
-}
@@ -1,24 +0,0 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-lerna-debug.log*
-
-node_modules
-dist
-dist-ssr
-*.local
-
-# Editor directories and files
-.vscode/*
-!.vscode/extensions.json
-.idea
-.DS_Store
-*.suo
-*.ntvs*
-*.njsproj
-*.sln
-*.sw?
@@ -1,11 +0,0 @@
-<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
-  <PropertyGroup>
-    <StartupCommand>npm run dev</StartupCommand>
-    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
-    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
-    <!-- Allows the build (or compile) script located on package.json to run on Build -->
-    <ShouldRunBuildScript>true</ShouldRunBuildScript>
-    <!-- Folder where production build objects will be placed -->
-    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
-  </PropertyGroup>
-</Project>
@@ -1,8 +0,0 @@
-# React + Vite
-
-This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
-
-Currently, two official plugins are available:
-
- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
@@ -1,13 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <link rel="icon" type="image/ico" href="/logo.ico" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>BTS Cloud Streams</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.jsx"></script>
-  </body>
-</html>
@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-  <packageSources>
-    <clear />
-    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
-  </packageSources>
-  <disabledPackageSources>
-    <clear />
-  </disabledPackageSources>
-</configuration>
@@ -1,38 +0,0 @@
-{
-  "name": "bts.datastreamwebapp.client",
-  "private": true,
-  "version": "0.0.0",
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
-    "preview": "vite preview"
-  },
-  "dependencies": {
-    "@angular/material": "^18.0.4",
-    "@emotion/react": "^11.11.4",
-    "@emotion/styled": "^11.11.5",
-    "@fontsource/noto-sans": "^5.0.22",
-    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
-    "@mui/icons-material": "^5.15.20",
-    "@mui/material": "^5.15.20",
-    "@mui/x-data-grid": "^7.7.1",
-    "bulma": "^1.0.1",
-    "react": "^18.3.1",
-    "react-data-table-component": "^7.6.2",
-    "react-dom": "^18.3.1",
-    "react-use-websocket": "^4.8.1",
-    "rpc-websockets": "^7.11.2"
-  },
-  "devDependencies": {
-    "@types/react": "^18.3.3",
-    "@types/react-dom": "^18.3.0",
-    "@vitejs/plugin-react": "^4.3.1",
-    "eslint": "^8.57.0",
-    "eslint-plugin-react": "^7.34.2",
-    "eslint-plugin-react-hooks": "^4.6.2",
-    "eslint-plugin-react-refresh": "^0.4.7",
-    "vite": "^5.2.13"
-  }
-}
@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
@@ -1,28 +0,0 @@
-@use "@material/checkbox";
-@use "@material/icon-button";
-@use "@material/data-table/data-table";
-
-h1#main-header {
-  padding: 4px 8px 4px 8px;
-  text-align: center;
-}
-
-td:hover {
-  background-color: cadetblue;
-}
-
-td#cell-decimal {
-  text-align: right;
-}
-
-td#cell-integer {
-  text-align: right;
-}
-
-td#cell-string {
-  text-align: center;
-}
-
-td#cell-date-time {
-  text-align: center;
-}
@@ -1,24 +0,0 @@
-import { createTheme, ThemeProvider, CssBaseline } from '@mui/material';
-import useMediaQuery from '@mui/material/useMediaQuery';
-import QueryControl from './QueryControl';
-import getTheme from './Theme';
-import '@fontsource/noto-sans';
-import './App.css';
-
-const App = () => {
-  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
-  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
-
-  return (
-    <ThemeProvider theme={theme}>
-      <CssBaseline />
-      <h1 id='main-header' style={{
-        color: theme.palette.text.primary,
-        fontFamily: 'Noto Sans',
-      }}>BTS Cloud Streams</h1>
-      <QueryControl websocketUrl="ws://localhost:5000/ws" theme={theme} />
-    </ThemeProvider>
-  );
-};
-
-export default App;
@@ -1,32 +0,0 @@
-@use "@material/checkbox";
-@use "@material/icon-button";
-@use "@material/data-table/data-table";
-
-th {
-  font-size: large;  
-  padding: 15px;
-}
-
-td {
-  padding: 15px;
-}
-
-td:hover {
-  background-color: cadetblue;
-}
-
-td#cell-decimal {
-  text-align: right;
-}
-
-td#cell-integer {
-  text-align: right;
-}
-
-td#cell-string {
-  text-align: center;
-}
-
-td#cell-date-time {
-  text-align: center;
-}
@@ -1,52 +0,0 @@
-import PropTypes from 'prop-types';
-import { DataGrid } from '@mui/x-data-grid';
-
-function getGridColDef(col) {
-  return {
-    field: col.name,
-    headerName: col.name,
-  };
-}
-
-export default function DataTable({ columns, rows }) {
-  return (
-    <DataGrid
-      columns={columns.map(getGridColDef)}
-      rows={rows}
-      sx={{
-        borderColor: 'primary.main',
-        backgroundColor: 'primary.dark',
-        color: 'text.primary',
-        '& .MuiDataGrid-columnHeader': {
-          color: 'primary.main',
-          fontWeight: 700,
-          backgroundColor: 'primary.dark'
-        },
-        '& .MuiDataGrid-filler': {
-          color: 'primary.main',
-          fontWeight: 700,
-          backgroundColor: 'primary.dark'
-        },
-        '& .MuiDataGrid-cell': {
-          color: 'text.primary',
-        },
-        '& .MuiDataGrid-cell:hover': {
-          color: 'primary.main',
-        },
-        '& .MuiDataGrid-row': {
-          '--DataGrid-rowBorderColor': 'secondary.dark'
-        }
-      }}
-    />
-  );
-}
-
-DataTable.propTypes = {
-  columns: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string.isRequired,
-      type: PropTypes.string.isRequired,
-    })
-  ).isRequired,
-  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
-};
@@ -1,100 +0,0 @@
-import PropTypes from 'prop-types';
-import DataTable from './DataTable';
-import QueryInputForm from './QueryInputForm';
-import Container from '@mui/material/Container';
-import React from 'react';
-
-var rpcReqIdCounter = 1;
-
-function subscribeReq(query) {
-  return {
-    jsonrpc: "2.0",
-    id: rpcReqIdCounter++,
-    method: "subscribe",
-    params: {
-      subscription_id: "1",
-      query: query
-    }
-  };
-}
-
-function unsubscribeReq() {
-  return {
-    jsonrpc: "2.0",
-    id: rpcReqIdCounter++,
-    method: "unsubscribe",
-    params: {
-      subscription_id: "1",
-    }
-  };
-}
-
-class QueryControl extends React.Component {
-  constructor({ websocketUrl, theme }) {
-    super(websocketUrl);
-    console.log('Connecting to WebSocket:', websocketUrl);
-    this.state = {
-      rows: null,
-      ws: new WebSocket(websocketUrl),
-      columns: null,
-      isRunning: false,
-      theme: theme,
-    };
-    this.state.ws.onopen = () => {
-      console.info('WebSocket open');
-    };
-    this.state.ws.onclose = () => {
-      console.info('WebSocket closed');
-    };
-    this.state.ws.onmessage = e => {
-      console.info(`Received message: ${e.data}`);
-      const msg = JSON.parse(e.data);
-      if (msg.id != null) {
-        let newState = { ...this.state };
-        newState.isRunning = msg.result.status == 'Subscribed';
-        this.setState(newState);
-      } else {
-        let newState = { ...this.state };
-        newState.rows = msg.params.all;
-        console.info(`Received rows: ${newState.rows}`);
-        newState.columns = msg.params.columns;
-        this.setState(newState);
-      }
-    };
-
-    this.start = (query) => {
-      this.state.ws.send(JSON.stringify(subscribeReq(query)));
-    };
-
-    this.stop = () => {
-      this.state.ws.send(JSON.stringify(unsubscribeReq()));
-    };
-  }
-
-  componentWillUnmount() {
-    if (this.state.ws) {
-      console.warn('Closing websocket because component is unmounting');
-      this.state.ws.close();
-    }
-  }
-
-  render() {
-    console.info('Rendering');
-    return (
-      <Container>
-        <QueryInputForm isRunning={this.state.isRunning} onStart={this.start} onStop={this.stop} />
-        {
-          this.state.rows == null ? <p style={{ color: this.state.theme.palette.warning.main }}>No data received</p> :
-            <DataTable columns={this.state.columns} rows={this.state.rows} />
-        }
-      </Container>
-    );
-  }
-}
-
-QueryControl.propTypes = {
-  websocketUrl: PropTypes.string.isRequired,
-  theme: PropTypes.object.isRequired,
-};
-
-export default QueryControl;
@@ -1,93 +0,0 @@
-import { useRef } from 'react';
-import Box from '@mui/material/Box';
-import Button from '@mui/material/Button';
-import Grid from '@mui/material/Grid';
-import Paper from '@mui/material/Paper';
-import PlayArrowIcon from '@mui/icons-material/PlayArrow';
-import PropTypes from 'prop-types';
-import StopIcon from '@mui/icons-material/Stop';
-import TextField from '@mui/material/TextField';
-import Tooltip from '@mui/material/Tooltip';
-
-const QueryInputForm = ({ isRunning, onStart, onStop }) => {
-  let textInput = useRef(null);
-
-  const onClearClick = () => {
-    textInput.current.value = '';
-  };
-
-  const onRunClick = () => {
-    onStart(textInput.current.value);
-  };
-
-  const onStopClick = () => {
-    onStop();
-  };
-
-  const onKeyDown = (e) => {
-    // Detect keyboard shortcuts.
-    //   Ctrl+Enter : Run
-    if (e.ctrlKey && e.key === 'Enter') {
-      onRunClick();
-    }
-  };
-
-  return (
-    <Box sx={{ p: 2 }}>
-      <Paper elevation={3} sx={{ color: 'primary.main' }}>
-        <Box sx={{ p: 2 }}>
-          <TextField
-            fullWidth
-            multiline
-            rows={10}
-            variant="outlined"
-            onKeyDown={onKeyDown}
-            inputRef={textInput}
-            placeholder="Enter Query..."
-          />
-        </Box>
-        <Box sx={{ p: 2, borderTop: 1 }}>
-          <Grid container spacing={2} justifyContent="flex-start">
-            <Grid item xs>
-              <Tooltip title="Clear Query Text">
-                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
-                  Clear
-                </Button>
-              </Tooltip>
-            </Grid>
-            <Grid item>
-              <Tooltip title="Run (Ctrl+Enter)">
-                <Button onClick={onRunClick} sx={{
-                  color: `query_start_button.${isRunning ? 'active' : 'inactive'}.color`,
-                  bgcolor: `query_start_button.${isRunning ? 'active' : 'inactive'}.background_color`,
-                  padding: '11px'
-                }}>
-                  <PlayArrowIcon />
-                </Button>
-              </Tooltip>
-            </Grid>
-            <Grid item>
-              <Tooltip title="Stop">
-                <Button onClick={onStopClick} sx={{
-                  color: `query_stop_button.${isRunning ? 'inactive' : 'active'}.color`,
-                  bgcolor: `query_stop_button.${isRunning ? 'inactive' : 'active'}.background_color`,
-                  padding: '11px'
-                }}>
-                  <StopIcon />
-                </Button>
-              </Tooltip>
-            </Grid>
-          </Grid>
-        </Box>
-      </Paper>
-    </Box>
-  );
-};
-
-QueryInputForm.propTypes = {
-  isRunning: PropTypes.bool.isRequired,
-  onStart: PropTypes.func.isRequired,
-  onStop: PropTypes.func.isRequired,
-};
-
-export default QueryInputForm;
@@ -1,88 +0,0 @@
-import { createTheme } from '@mui/material/styles';
-import PropTypes from 'prop-types';
-import '@fontsource/noto-sans';
-import './App.css';
-
-const getLightPalette = () => {
-  return {
-    mode: 'light',
-    primary: {
-      light: '#6573c3',
-      main: '#3f51b5',
-      dark: '#2c387e',
-      contrastText: '#ffffff',
-    },
-    secondary: {
-      light: '#f73378',
-      main: '#f50057',
-      dark: '#ab003c',
-      contrastText: '#ffffff',
-    },
-    text: {
-      primary: '#000000c4',
-      secondary: '#00000086',
-      disabled: '#0000005b',
-      hint: '#0000005b',
-    }
-  };
-};
-
-const getDarkPalette = () => {
-  return {
-    mode: 'dark',
-    primary: {
-      light: '#bcc5ff',
-      main: '#33baf9',
-      dark: '#202334',
-      contrastText: '#78a2ac',
-    },
-    secondary: {
-      light: '#9fd0b8',
-      main: '#5eb289',
-      dark: '#ffffff12',
-      contrastText: '#c8d4f7',
-    },
-    text: {
-      primary: '#eaf8f8',
-      dark: '#eaf8f8a0'
-    },
-    query_start_button: {
-      active: {
-        color: '#33baf9',
-        background_color: '#ffffff40',
-      },
-      inactive: {
-        color: '#ffffff40',
-        background_color: '#ffffff00',
-      }
-    },
-    warning: {
-      main: '#c4823f'
-    },
-    query_stop_button: {
-      active: {
-        color: '#ff4444a0',
-        background_color: '#ffffff40',
-      },
-      inactive: {
-        color: '#ff444450',
-        background_color: '#ffffff00',
-      }
-    },
-  };
-};
-
-const getTheme = ({ mode }) => {
-  return createTheme({
-    typography: {
-      fontFamily: '"Helvetica"'
-    },
-    palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
-  });
-};
-
-export default getTheme;
-
-getTheme.propTypes = {
-  mode: PropTypes.string.isRequired,
-};
@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>
@@ -1,51 +0,0 @@
-.dark-high-contrast {
-  --md-sys-color-primary: rgb(251 250 255);
-  --md-sys-color-surface-tint: rgb(165 200 255);
-  --md-sys-color-on-primary: rgb(0 0 0);
-  --md-sys-color-primary-container: rgb(173 204 255);
-  --md-sys-color-on-primary-container: rgb(0 0 0);
-  --md-sys-color-secondary: rgb(250 250 255);
-  --md-sys-color-on-secondary: rgb(0 0 0);
-  --md-sys-color-secondary-container: rgb(159 208 255);
-  --md-sys-color-on-secondary-container: rgb(0 0 0);
-  --md-sys-color-tertiary: rgb(255 249 251);
-  --md-sys-color-on-tertiary: rgb(0 0 0);
-  --md-sys-color-tertiary-container: rgb(222 193 231);
-  --md-sys-color-on-tertiary-container: rgb(0 0 0);
-  --md-sys-color-error: rgb(255 249 249);
-  --md-sys-color-on-error: rgb(0 0 0);
-  --md-sys-color-error-container: rgb(255 186 177);
-  --md-sys-color-on-error-container: rgb(0 0 0);
-  --md-sys-color-background: rgb(17 19 24);
-  --md-sys-color-on-background: rgb(225 226 233);
-  --md-sys-color-surface: rgb(17 19 24);
-  --md-sys-color-on-surface: rgb(255 255 255);
-  --md-sys-color-surface-variant: rgb(67 71 78);
-  --md-sys-color-on-surface-variant: rgb(251 250 255);
-  --md-sys-color-outline: rgb(200 202 212);
-  --md-sys-color-outline-variant: rgb(200 202 212);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(225 226 233);
-  --md-sys-color-inverse-on-surface: rgb(0 0 0);
-  --md-sys-color-inverse-primary: rgb(0 42 83);
-  --md-sys-color-primary-fixed: rgb(219 231 255);
-  --md-sys-color-on-primary-fixed: rgb(0 0 0);
-  --md-sys-color-primary-fixed-dim: rgb(173 204 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
-  --md-sys-color-secondary-fixed: rgb(214 233 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 0 0);
-  --md-sys-color-secondary-fixed-dim: rgb(159 208 255);
-  --md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
-  --md-sys-color-tertiary-fixed: rgb(249 222 255);
-  --md-sys-color-on-tertiary-fixed: rgb(0 0 0);
-  --md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
-  --md-sys-color-surface-dim: rgb(17 19 24);
-  --md-sys-color-surface-bright: rgb(55 57 62);
-  --md-sys-color-surface-container-lowest: rgb(12 14 19);
-  --md-sys-color-surface-container-low: rgb(25 28 32);
-  --md-sys-color-surface-container: rgb(29 32 36);
-  --md-sys-color-surface-container-high: rgb(40 42 47);
-  --md-sys-color-surface-container-highest: rgb(50 53 58);
-}
@@ -1,51 +0,0 @@
-.dark-medium-contrast {
-  --md-sys-color-primary: rgb(173 204 255);
-  --md-sys-color-surface-tint: rgb(165 200 255);
-  --md-sys-color-on-primary: rgb(0 22 49);
-  --md-sys-color-primary-container: rgb(112 146 198);
-  --md-sys-color-on-primary-container: rgb(0 0 0);
-  --md-sys-color-secondary: rgb(159 208 255);
-  --md-sys-color-on-secondary: rgb(0 24 42);
-  --md-sys-color-secondary-container: rgb(100 149 193);
-  --md-sys-color-on-secondary-container: rgb(0 0 0);
-  --md-sys-color-tertiary: rgb(222 193 231);
-  --md-sys-color-on-tertiary: rgb(33 14 42);
-  --md-sys-color-tertiary-container: rgb(162 136 171);
-  --md-sys-color-on-tertiary-container: rgb(0 0 0);
-  --md-sys-color-error: rgb(255 186 177);
-  --md-sys-color-on-error: rgb(55 0 1);
-  --md-sys-color-error-container: rgb(255 84 73);
-  --md-sys-color-on-error-container: rgb(0 0 0);
-  --md-sys-color-background: rgb(17 19 24);
-  --md-sys-color-on-background: rgb(225 226 233);
-  --md-sys-color-surface: rgb(17 19 24);
-  --md-sys-color-on-surface: rgb(251 250 255);
-  --md-sys-color-surface-variant: rgb(67 71 78);
-  --md-sys-color-on-surface-variant: rgb(200 202 212);
-  --md-sys-color-outline: rgb(160 163 171);
-  --md-sys-color-outline-variant: rgb(128 131 139);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(225 226 233);
-  --md-sys-color-inverse-on-surface: rgb(40 42 47);
-  --md-sys-color-inverse-primary: rgb(35 73 120);
-  --md-sys-color-primary-fixed: rgb(212 227 255);
-  --md-sys-color-on-primary-fixed: rgb(0 17 40);
-  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
-  --md-sys-color-secondary-fixed: rgb(206 229 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 18 34);
-  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
-  --md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
-  --md-sys-color-tertiary-fixed: rgb(247 216 255);
-  --md-sys-color-on-tertiary-fixed: rgb(28 9 37);
-  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
-  --md-sys-color-surface-dim: rgb(17 19 24);
-  --md-sys-color-surface-bright: rgb(55 57 62);
-  --md-sys-color-surface-container-lowest: rgb(12 14 19);
-  --md-sys-color-surface-container-low: rgb(25 28 32);
-  --md-sys-color-surface-container: rgb(29 32 36);
-  --md-sys-color-surface-container-high: rgb(40 42 47);
-  --md-sys-color-surface-container-highest: rgb(50 53 58);
-}
@@ -1,51 +0,0 @@
-.dark {
-  --md-sys-color-primary: rgb(165 200 255);
-  --md-sys-color-surface-tint: rgb(165 200 255);
-  --md-sys-color-on-primary: rgb(0 49 94);
-  --md-sys-color-primary-container: rgb(34 72 118);
-  --md-sys-color-on-primary-container: rgb(212 227 255);
-  --md-sys-color-secondary: rgb(154 203 250);
-  --md-sys-color-on-secondary: rgb(0 51 83);
-  --md-sys-color-secondary-container: rgb(12 74 114);
-  --md-sys-color-on-secondary-container: rgb(206 229 255);
-  --md-sys-color-tertiary: rgb(218 189 226);
-  --md-sys-color-on-tertiary: rgb(61 41 70);
-  --md-sys-color-tertiary-container: rgb(85 63 93);
-  --md-sys-color-on-tertiary-container: rgb(247 216 255);
-  --md-sys-color-error: rgb(255 180 171);
-  --md-sys-color-on-error: rgb(105 0 5);
-  --md-sys-color-error-container: rgb(147 0 10);
-  --md-sys-color-on-error-container: rgb(255 218 214);
-  --md-sys-color-background: rgb(17 19 24);
-  --md-sys-color-on-background: rgb(225 226 233);
-  --md-sys-color-surface: rgb(17 19 24);
-  --md-sys-color-on-surface: rgb(225 226 233);
-  --md-sys-color-surface-variant: rgb(67 71 78);
-  --md-sys-color-on-surface-variant: rgb(195 198 207);
-  --md-sys-color-outline: rgb(141 145 153);
-  --md-sys-color-outline-variant: rgb(67 71 78);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(225 226 233);
-  --md-sys-color-inverse-on-surface: rgb(46 48 53);
-  --md-sys-color-inverse-primary: rgb(60 96 144);
-  --md-sys-color-primary-fixed: rgb(212 227 255);
-  --md-sys-color-on-primary-fixed: rgb(0 28 58);
-  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
-  --md-sys-color-secondary-fixed: rgb(206 229 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
-  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
-  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
-  --md-sys-color-tertiary-fixed: rgb(247 216 255);
-  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
-  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
-  --md-sys-color-surface-dim: rgb(17 19 24);
-  --md-sys-color-surface-bright: rgb(55 57 62);
-  --md-sys-color-surface-container-lowest: rgb(12 14 19);
-  --md-sys-color-surface-container-low: rgb(25 28 32);
-  --md-sys-color-surface-container: rgb(29 32 36);
-  --md-sys-color-surface-container-high: rgb(40 42 47);
-  --md-sys-color-surface-container-highest: rgb(50 53 58);
-}
@@ -1,51 +0,0 @@
-.light-high-contrast {
-  --md-sys-color-primary: rgb(0 34 70);
-  --md-sys-color-surface-tint: rgb(60 96 144);
-  --md-sys-color-on-primary: rgb(255 255 255);
-  --md-sys-color-primary-container: rgb(29 68 114);
-  --md-sys-color-on-primary-container: rgb(255 255 255);
-  --md-sys-color-secondary: rgb(0 36 61);
-  --md-sys-color-on-secondary: rgb(255 255 255);
-  --md-sys-color-secondary-container: rgb(3 70 110);
-  --md-sys-color-on-secondary-container: rgb(255 255 255);
-  --md-sys-color-tertiary: rgb(46 26 55);
-  --md-sys-color-on-tertiary: rgb(255 255 255);
-  --md-sys-color-tertiary-container: rgb(81 59 89);
-  --md-sys-color-on-tertiary-container: rgb(255 255 255);
-  --md-sys-color-error: rgb(78 0 2);
-  --md-sys-color-on-error: rgb(255 255 255);
-  --md-sys-color-error-container: rgb(140 0 9);
-  --md-sys-color-on-error-container: rgb(255 255 255);
-  --md-sys-color-background: rgb(249 249 255);
-  --md-sys-color-on-background: rgb(25 28 32);
-  --md-sys-color-surface: rgb(249 249 255);
-  --md-sys-color-on-surface: rgb(0 0 0);
-  --md-sys-color-surface-variant: rgb(224 226 236);
-  --md-sys-color-on-surface-variant: rgb(32 36 43);
-  --md-sys-color-outline: rgb(63 67 74);
-  --md-sys-color-outline-variant: rgb(63 67 74);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(46 48 53);
-  --md-sys-color-inverse-on-surface: rgb(255 255 255);
-  --md-sys-color-inverse-primary: rgb(228 236 255);
-  --md-sys-color-primary-fixed: rgb(29 68 114);
-  --md-sys-color-on-primary-fixed: rgb(255 255 255);
-  --md-sys-color-primary-fixed-dim: rgb(0 45 88);
-  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-secondary-fixed: rgb(3 70 110);
-  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
-  --md-sys-color-secondary-fixed-dim: rgb(0 47 77);
-  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed: rgb(81 59 89);
-  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-surface-dim: rgb(217 218 224);
-  --md-sys-color-surface-bright: rgb(249 249 255);
-  --md-sys-color-surface-container-lowest: rgb(255 255 255);
-  --md-sys-color-surface-container-low: rgb(243 243 250);
-  --md-sys-color-surface-container: rgb(237 237 244);
-  --md-sys-color-surface-container-high: rgb(231 232 238);
-  --md-sys-color-surface-container-highest: rgb(225 226 233);
-}
@@ -1,51 +0,0 @@
-.light-medium-contrast {
-  --md-sys-color-primary: rgb(29 68 114);
-  --md-sys-color-surface-tint: rgb(60 96 144);
-  --md-sys-color-on-primary: rgb(255 255 255);
-  --md-sys-color-primary-container: rgb(83 118 167);
-  --md-sys-color-on-primary-container: rgb(255 255 255);
-  --md-sys-color-secondary: rgb(3 70 110);
-  --md-sys-color-on-secondary: rgb(255 255 255);
-  --md-sys-color-secondary-container: rgb(71 121 163);
-  --md-sys-color-on-secondary-container: rgb(255 255 255);
-  --md-sys-color-tertiary: rgb(81 59 89);
-  --md-sys-color-on-tertiary: rgb(255 255 255);
-  --md-sys-color-tertiary-container: rgb(133 108 141);
-  --md-sys-color-on-tertiary-container: rgb(255 255 255);
-  --md-sys-color-error: rgb(140 0 9);
-  --md-sys-color-on-error: rgb(255 255 255);
-  --md-sys-color-error-container: rgb(218 52 46);
-  --md-sys-color-on-error-container: rgb(255 255 255);
-  --md-sys-color-background: rgb(249 249 255);
-  --md-sys-color-on-background: rgb(25 28 32);
-  --md-sys-color-surface: rgb(249 249 255);
-  --md-sys-color-on-surface: rgb(25 28 32);
-  --md-sys-color-surface-variant: rgb(224 226 236);
-  --md-sys-color-on-surface-variant: rgb(63 67 74);
-  --md-sys-color-outline: rgb(91 95 103);
-  --md-sys-color-outline-variant: rgb(119 123 131);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(46 48 53);
-  --md-sys-color-inverse-on-surface: rgb(240 240 247);
-  --md-sys-color-inverse-primary: rgb(165 200 255);
-  --md-sys-color-primary-fixed: rgb(83 118 167);
-  --md-sys-color-on-primary-fixed: rgb(255 255 255);
-  --md-sys-color-primary-fixed-dim: rgb(58 93 141);
-  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-secondary-fixed: rgb(71 121 163);
-  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
-  --md-sys-color-secondary-fixed-dim: rgb(43 96 137);
-  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed: rgb(133 108 141);
-  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-surface-dim: rgb(217 218 224);
-  --md-sys-color-surface-bright: rgb(249 249 255);
-  --md-sys-color-surface-container-lowest: rgb(255 255 255);
-  --md-sys-color-surface-container-low: rgb(243 243 250);
-  --md-sys-color-surface-container: rgb(237 237 244);
-  --md-sys-color-surface-container-high: rgb(231 232 238);
-  --md-sys-color-surface-container-highest: rgb(225 226 233);
-}
@@ -1,51 +0,0 @@
-.light {
-  --md-sys-color-primary: rgb(60 96 144);
-  --md-sys-color-surface-tint: rgb(60 96 144);
-  --md-sys-color-on-primary: rgb(255 255 255);
-  --md-sys-color-primary-container: rgb(212 227 255);
-  --md-sys-color-on-primary-container: rgb(0 28 58);
-  --md-sys-color-secondary: rgb(46 98 140);
-  --md-sys-color-on-secondary: rgb(255 255 255);
-  --md-sys-color-secondary-container: rgb(206 229 255);
-  --md-sys-color-on-secondary-container: rgb(0 29 50);
-  --md-sys-color-tertiary: rgb(110 86 118);
-  --md-sys-color-on-tertiary: rgb(255 255 255);
-  --md-sys-color-tertiary-container: rgb(247 216 255);
-  --md-sys-color-on-tertiary-container: rgb(39 20 48);
-  --md-sys-color-error: rgb(186 26 26);
-  --md-sys-color-on-error: rgb(255 255 255);
-  --md-sys-color-error-container: rgb(255 218 214);
-  --md-sys-color-on-error-container: rgb(65 0 2);
-  --md-sys-color-background: rgb(249 249 255);
-  --md-sys-color-on-background: rgb(25 28 32);
-  --md-sys-color-surface: rgb(249 249 255);
-  --md-sys-color-on-surface: rgb(25 28 32);
-  --md-sys-color-surface-variant: rgb(224 226 236);
-  --md-sys-color-on-surface-variant: rgb(67 71 78);
-  --md-sys-color-outline: rgb(116 119 127);
-  --md-sys-color-outline-variant: rgb(195 198 207);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(46 48 53);
-  --md-sys-color-inverse-on-surface: rgb(240 240 247);
-  --md-sys-color-inverse-primary: rgb(165 200 255);
-  --md-sys-color-primary-fixed: rgb(212 227 255);
-  --md-sys-color-on-primary-fixed: rgb(0 28 58);
-  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
-  --md-sys-color-secondary-fixed: rgb(206 229 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
-  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
-  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
-  --md-sys-color-tertiary-fixed: rgb(247 216 255);
-  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
-  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
-  --md-sys-color-surface-dim: rgb(217 218 224);
-  --md-sys-color-surface-bright: rgb(249 249 255);
-  --md-sys-color-surface-container-lowest: rgb(255 255 255);
-  --md-sys-color-surface-container-low: rgb(243 243 250);
-  --md-sys-color-surface-container: rgb(237 237 244);
-  --md-sys-color-surface-container-high: rgb(231 232 238);
-  --md-sys-color-surface-container-highest: rgb(225 226 233);
-}
@@ -1,107 +0,0 @@
-:root {
-  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
-  line-height: 1.5;
-  font-weight: 400;
-
-  color-scheme: light dark;
-  color: rgba(0, 0, 0, 0.87);
-  background-color: #242424;
-
-  font-synthesis: none;
-  text-rendering: optimizeLegibility;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-}
-
-a {
-  font-weight: 500;
-  color: #646cff;
-  text-decoration: inherit;
-}
-a:hover {
-  color: #535bf2;
-}
-
-body {
-  margin: 0;
-  display: flex;
-  place-items: center;
-  min-width: 320px;
-  min-height: 100vh;
-}
-
-button {
-  border-radius: 8px;
-  border: 1px solid transparent;
-  padding: 0.6em 1.2em;
-  font-size: 1em;
-  font-weight: 500;
-  font-family: inherit;
-  background-color: #1a1a1a;
-  cursor: pointer;
-  transition: border-color 0.25s;
-}
-button:hover {
-  border-color: #646cff;
-}
-button:focus,
-button:focus-visible {
-  outline: 4px auto -webkit-focus-ring-color;
-}
-
-button#query-button {
-  height: 5em;
-  vertical-align: top;
-}
-
-button#play-button {
-  padding: '10px 20px';
-  font-size: '16px';
-  cursor: 'pointer';
-  background-color: '#28a745';
-  color: '#fff';
-  border: 'none';
-  border-radius: '5px';
-};
-
-button#pause-button {
-  background-color: #ffc107;
-}
-
-button#stop-button {
-  background-color: #dc3545;
-}
-
-h1 {
-  font-size: 3.2em;
-  line-height: 1.1;
-}
-
-p#footer {
-  float: none;
-  clear: both;
-  vertical-align: bottom;
-}
-
-table#query-output {
-  float: none;
-  clear: both;
-}
-
-textarea#query-input {
-  height: 5em;
-  width: 50em;
-}
-
-@media (prefers-color-scheme: light) {
-  :root {
-    color: #213547;
-    background-color: #ffffff;
-  }
-  a:hover {
-    color: #747bff;
-  }
-  button {
-    background-color: #f9f9f9;
-  }
-}
@@ -1,9 +0,0 @@
-import React from 'react';
-import ReactDOM from 'react-dom/client';
-import App from './App';
-
-ReactDOM.createRoot(document.getElementById('root')).render(
-  <React.StrictMode>
-    <App />
-  </React.StrictMode>
-)
@@ -1,69 +0,0 @@
-import { fileURLToPath, URL } from 'node:url';
-import { defineConfig } from 'vite';
-import plugin from '@vitejs/plugin-react';
-import fs from 'fs';
-import path from 'path';
-import child_process from 'child_process';
-import process, { env } from 'process';
-
-const baseFolder =
-  process.env.APPDATA !== undefined && process.env.APPDATA !== ''
-    ? `${process.env.APPDATA}/ASP.NET/https`
-    : `${process.env.HOME}/.aspnet/https`;
-
-const certificateName = "bts.datastreamwebapp.client";
-const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
-const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
-
-if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
-  if (0 !== child_process.spawnSync('dotnet', [
-    'dev-certs',
-    'https',
-    '--export-path',
-    certFilePath,
-    '--format',
-    'Pem',
-    '--no-password',
-  ], { stdio: 'inherit', }).status) {
-    throw new Error("Could not create certificate.");
-  }
-}
-
-const target = env.ASPNETCORE_HTTPS_PORT
-  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
-  : env.ASPNETCORE_URLS
-    ? env.ASPNETCORE_URLS.split(';')[0]
-    : 'https://localhost:5001';
-
-export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
-  return {
-    plugins: [plugin()],
-    resolve: {
-      alias: {
-        '@': fileURLToPath(new URL('./src', import.meta.url))
-      }
-    },
-    build: {
-      minify: false,
-      sourcemap: true,
-      outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
-    },
-    server: {
-      proxy: {
-        '^/api': {
-          target,
-          secure: false
-        },
-        '^/ws': {
-          target,
-          secure: false
-        }
-      },
-      port: 5173,
-      https: {
-        key: fs.readFileSync(keyFilePath),
-        cert: fs.readFileSync(certFilePath),
-      }
-    }
-  };
-});
@@ -1,36 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
-
-  <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <InvariantGlobalization>true</InvariantGlobalization>
-    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
-    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
-    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="8.0.6" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="8.0.6" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-    <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
-    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
-    <ProjectReference Include="..\Bts.Extensions.DependencyInjection\Bts.Extensions.DependencyInjection.csproj" />
-  </ItemGroup>
-
-  <!-- <ItemGroup>
-    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
-  </ItemGroup> -->
-</Project>
@@ -1,33 +0,0 @@
-{
-  "Spark": {
-    "ConfigServer": "http://localhost:8000/config"
-  },
-  "SparkdCredentials": {
-    "User": "bts",
-    "Password": ""
-  },
-  "MasterWorker": {
-    "Interval": "00:00:00.1"
-  },
-  "PubSub": {
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:12345",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:12345",
-          "TagRegExp": ".*"
-        }
-      ]
-    }
-  },
-  "ViewStreamDatabase": {
-    "Interval": "00:00:01"
-  }
-}
@@ -1,4 +0,0 @@
-{
-  "content_root_path": "build/Bts.DataStreamWebApp.Client/production",
-  "web_root_path": "dist"
-}
@@ -1,36 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using BCG;
-
-using bts.api.core;
-
-using Microsoft.AspNetCore.Mvc;
-
-[Route("/ws")]
-public class DataStreamSubscriptionController : ControllerBase {
-   readonly ILogger logger;
-   readonly PrimitiveManager primitiveManager;
-   readonly IServiceProvider serviceProvider;
-
-   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger, PrimitiveManager primitiveManager, IServiceProvider serviceProvider) {
-      this.logger = logger;
-      this.primitiveManager = primitiveManager;
-      this.serviceProvider = serviceProvider;
-      logger.LogInformation("Creating {ControllerType}", GetType().Name);
-   }
-
-   [Route("/ws")]
-   public async Task Get() {
-      if (!HttpContext.WebSockets.IsWebSocketRequest) {
-         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
-         return;
-      }
-      logger.LogInformation("Establishing websocket connection...");
-      using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
-      logger.LogInformation("Websocket connection established.");
-      var context = ServiceContext.create();
-      await context.login("bts", string.Empty);
-      var server = new DataStreamWebSocketServer(webSocket, logger, primitiveManager, serviceProvider);
-      await server.Run();
-   }
-}
@@ -1,112 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using BCG;
-
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamCore;
-using Bts.DataStreamWebApp.Server.Messages;
-
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class DataStreamSubscription : IDisposable {
-   public readonly ViewStream ViewStream;
-   public DataPacket? LastFired;
-   public readonly string Id;
-
-   readonly ILogger logger;
-   readonly WebSocket webSocket;
-   readonly DataPacketJsonSerializer dataPacketSerializer = new();
-
-   public DataStreamSubscription(string id, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
-      ViewStream = viewStream;
-      Id = id;
-      this.webSocket = webSocket;
-      this.logger = logger;
-   }
-
-   public SubscriptionStatus Status { get; private set; }
-
-   public void Start() {
-      if (Status == SubscriptionStatus.Subscribed) {
-         logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
-         return;
-      }
-      logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
-      Status = SubscriptionStatus.Subscribed;
-      ViewStream.onChange += onChange;
-   }
-
-   public Task Clear() {
-      if (LastFired is null) {
-         return Task.CompletedTask;
-      }
-      logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
-      return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
-   }
-
-   public void Stop() {
-      if (Status == SubscriptionStatus.Unsubscribed) {
-         logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
-         return;
-      }
-      logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
-      Status = SubscriptionStatus.Unsubscribed;
-      ViewStream.onChange -= onChange;
-   }
-
-   public void Dispose() =>
-      Stop();
-
-   async void onChange(object newValue) {
-      if (newValue is not DataPacket dp) {
-         logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
-         return;
-      }
-      if (webSocket.State != WebSocketState.Open) {
-         logger.LogError(
-            "Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
-         Dispose();
-         return;
-      }
-      await sendDataPacket("subscribe", dp);
-   }
-
-   async Task sendDataPacket(string method, DataPacket dp) {
-      var serializerOpts = new JsonSerializerOptions {
-         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
-      };
-      serializerOpts.Converters.Add(new DataPacketJsonConverter() {
-         IncludeCurrent = true,
-         UseObjectRows = true,
-      });
-      var res = JsonRpcRequest.CreateNotification(method, dp);
-      var str = serialize(res, serializerOpts);
-      if (str is null) {
-         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
-         Dispose();
-         return;
-      }
-      logger.LogInformation(
-         "Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
-         dp.newRows.Count,
-         dp.schema.columnNames.Length,
-         str.Length);
-      var bytes = Encoding.UTF8.GetBytes(str);
-      await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
-      LastFired = dp;
-   }
-
-   string? serialize<T>(T message, JsonSerializerOptions? opts) {
-      opts ??= JsonSerializerOptions.Default;
-      try {
-         return JsonSerializer.Serialize(message, opts);
-      } catch (Exception ex) {
-         logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
-      }
-      return null;
-   }
-}
@@ -1,89 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using Bts.PubSub;
-
-using CommandLine;
-
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class DataStreamWebAppServerConfig {
-   /// <summary>
-   /// The login user for sparkd
-   /// </summary>
-   [JsonPropertyName("user")]
-   [Option("user", HelpText = "The sparkd login user")]
-   public string? User { get; set; }
-
-   /// <summary>
-   /// The login password for sparkd
-   /// </summary>
-   [JsonPropertyName("password")]
-   [Option("pass", HelpText = "The sparkd login password")]
-   public string? Password { get; set; }
-
-   /// <summary>
-   /// The config server url for sparkd
-   /// </summary>
-   [JsonPropertyName("sparkd")]
-   [Option("sparkd", HelpText = "The sparkd config server")]
-   public string? Sparkd { get; set; }
-
-   [JsonPropertyName("pubsub")]
-   public PubSubConfig? PubSub { get; set; }
-
-   [Option("content-root-path")]
-   [JsonPropertyName("content_root_path")]
-   public string? ContentRootPath { get; set; }
-
-   [Option("web-root-path")]
-   [JsonPropertyName("web_root_path")]
-   public string? WebRootPath { get; set; }
-
-   [Option("environment-name")]
-   [JsonPropertyName("environment_name")]
-   public string? EnvironmentName { get; set; }
-
-   public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
-
-   [Option(
-      "config",
-      MetaValue = "PATH",
-      HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
-   [JsonIgnore]
-   public string? ConfigFile { get; set; }
-
-   public static DataStreamWebAppServerConfig Default() =>
-      new() {
-         User = "bts",
-         PubSub = PubSubConfig.Default(),
-      };
-
-   public void MergeWith(DataStreamWebAppServerConfig other) {
-      User ??= other.User;
-      Password ??= other.Password;
-      PubSub ??= other.PubSub;
-      Sparkd ??= other.Sparkd;
-      ContentRootPath ??= other.ContentRootPath;
-      WebRootPath ??= other.WebRootPath;
-      ConfigFile ??= other.ConfigFile;
-   }
-
-   /// <summary>
-   /// Merges options with those defined in `ConfigFile` (does not overwrite
-   /// non-null existing values).
-   /// </summary>
-   /// <exception cref="InvalidOperationException">
-   /// JsonSerializer.Deserialize returns null
-   /// </exception>
-   public void MergeWithConfigFile() {
-      if (ConfigFile is null) {
-         return;
-      }
-      var cfgString = File.ReadAllText(ConfigFile);
-      var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
-         throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
-      MergeWith(parsedCfg);
-      ConfigFile = null;
-   }
-}
@@ -1,93 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using BCG;
-
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamWebApp.Server.Messages;
-
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-
-public class DataStreamWebSocketServer {
-   readonly ILogger logger;
-   readonly WebSocket webSocket;
-   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
-   readonly PrimitiveManager primitiveManager;
-   readonly IServiceProvider serviceProvider;
-
-   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger, PrimitiveManager primitiveManager, IServiceProvider serviceProvider) {
-      this.webSocket = webSocket;
-      this.logger = logger;
-      this.primitiveManager = primitiveManager;
-      this.serviceProvider = serviceProvider;
-   }
-
-   public async Task<int> Run() {
-      logger.LogInformation("DataStreamWebSocketServer running");
-      await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger)) {
-         await (req.Method switch {
-            "subscribe" => onSubscribe(req),
-            "unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>()),
-            _ => Task.CompletedTask,
-         });
-      }
-      logger.LogInformation("DataStreamWebSocketServer stopped");
-      return 0;
-   }
-
-   async Task onSubscribe(JsonRpcRequest<JsonNode> req) {
-      ArgumentNullException.ThrowIfNull(req.Id);
-      ArgumentException.ThrowIfNullOrEmpty(req.Method);
-      if (!SubscribeRequest.ParseQuery(req, primitiveManager, out var vs, out var subId, out var errMsg, out var errCode)) {
-         logger.LogError("Invalid 'subscribe' request: {Reason}", errMsg);
-         await send(new JsonRpcError<JsonNode>() {
-            Code = (long)errCode,
-            Message = $"{errCode}: {errMsg}",
-         });
-         return;
-      }
-      if (subscriptions.TryGetValue(subId, out var oldSub)) {
-         oldSub.Dispose();
-      }
-      vs.Reset(serviceProvider);
-      var sub = new DataStreamSubscription(subId, vs, webSocket, logger);
-      subscriptions[subId] = sub;
-      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed);
-      sub.Start();
-   }
-
-   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req) {
-      ArgumentNullException.ThrowIfNull(req.Id);
-      ArgumentException.ThrowIfNullOrEmpty(req.Method);
-      var subId = req.Params?.SubscriptionId ?? string.Empty;
-      if (!subscriptions.TryGetValue(subId, out var sub)) {
-         logger.LogInformation("Ignoring '{Method}' request because no subscription was found with id {Id}.", req.Method, subId);
-         return;
-      }
-      logger.LogInformation("Disposing subscription");
-      sub.Dispose();
-      subscriptions.Remove(subId);
-      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Unsubscribed);
-   }
-
-   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status) {
-      logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
-      return send(JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
-         SubscriptionId = subId,
-         Status = status,
-      }));
-   }
-
-   Task send<T>(T message) {
-      if (message is null) {
-         return Task.CompletedTask;
-      }
-      var str = JsonSerializer.Serialize(message);
-      var bytes = Encoding.UTF8.GetBytes(str);
-      logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
-      return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
-   }
-}
@@ -1,6 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-enum ErrorCode {
-   None = 0,
-   InvalidSubscriptionRequest,
-}
@@ -1,74 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-using BCG;
-
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamExpressions;
-
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-using System.Text.Json.Serialization;
-
-public class SubscribeRequest {
-   [JsonPropertyName("query")]
-   public string? Query { get; set; }
-
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-
-   internal static bool ParseQuery(
-      JsonRpcRequest<JsonNode> req,
-      PrimitiveManager primitiveManager,
-      [NotNullWhen(true)] out ViewStream? stream,
-      [NotNullWhen(true)] out string? subscriptionId,
-      [NotNullWhen(false)] out string? errorMessage,
-      out ErrorCode errorCode
-   ) {
-      stream = null;
-      subscriptionId = null;
-      errorCode = ErrorCode.None;
-      var paramsJsonObj = req.Params?.AsObject();
-      if (paramsJsonObj is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "'params' object not found";
-         return false;
-      }
-      var subReq = paramsJsonObj.Deserialize<SubscribeRequest>();
-      if (subReq is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "'params' object invalid";
-         return false;
-      }
-      subscriptionId = subReq.SubscriptionId ?? string.Empty;
-      var query = subReq.Query;
-      if (query is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "'params.query' field not found (expected a string)";
-         return false;
-      }
-      errorMessage = null;
-      var exp = DataStreamExpressionsInterpreter.Evaluate(query);
-      if (exp is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "'params.query' expression has invalid syntax";
-         return false;
-      }
-      string? exceptionMsg = null;
-      try {
-         stream = exp.getView(new Dictionary<string, StatementExpression>(0), primitiveManager);
-      } catch (Exception ex) {
-         exceptionMsg = ex.ToString();
-      }
-      if (stream is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
-         if (exceptionMsg is not null) {
-            errorMessage += ". Error" + exceptionMsg;
-         }
-         return false;
-      }
-      return true;
-   }
-}
@@ -1,18 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-using System.Text.Json.Serialization;
-
-enum SubscriptionStatus {
-   None,
-   Subscribed,
-   Unsubscribed,
-}
-
-class SubscriptionStatusMessage {
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-
-   [JsonPropertyName("status")]
-   [JsonConverter(typeof(JsonStringEnumConverter))]
-   public SubscriptionStatus Status { get; set; } = SubscriptionStatus.None;
-}
@@ -1,8 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-using System.Text.Json.Serialization;
-
-public class UnsubscribeRequest {
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-}
@@ -1,64 +0,0 @@
-using Bts.DataStreamCore.Services;
-using Bts.DataStreamWebApp.Server.Services;
-using Bts.PubSub.Services;
-
-using Spark.Api.Services;
-using Spark.Core.Services;
-
-using System.Reactive.Linq;
-
-var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
-   Args = args,
-   ApplicationName = "Bts.DataStreamWebApp.Server",
-});
-ConfigureApp(builder.Configuration);
-ConfigureServices(builder.Services, builder.Configuration);
-
-var app = builder.Build();
-if (app.Environment.IsDevelopment()) {
-   app.UseDirectoryBrowser();
-}
-var wsOpts = new WebSocketOptions {
-   KeepAliveInterval = TimeSpan.FromMinutes(2)
-};
-app.UseWebSockets(wsOpts);
-app.UseDefaultFiles();
-app.UseStaticFiles();
-if (app.Environment.IsDevelopment()) {
-   app.UseSwagger();
-   app.UseSwaggerUI();
-}
-app.UseHttpsRedirection();
-app.UseAuthorization();
-app.MapControllers();
-try {
-   Console.Write($"Starting Application");
-   app.Run();
-   Console.Write($"Application exited gracefully.");
-} catch (Exception ex) {
-   Console.Error.Write($"Application crashed: {ex}");
-   return 1;
-}
-return 0;
-
-static void ConfigureApp(ConfigurationManager configBuilder) {
-   configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
-   var cfgRoot = configBuilder.GetSection("config");
-   if (cfgRoot?.Value is not null) {
-      configBuilder.AddJsonFile(cfgRoot.Value);
-   }
-}
-
-static void ConfigureServices(IServiceCollection services, IConfiguration config) {
-   services.AddDirectoryBrowser();
-   services.AddControllers();
-   services.AddEndpointsApiExplorer();
-   services.AddSwaggerGen();
-   services.AddMasterWorker(config.GetSection("MasterWorker"));
-   services.AddViewStreamDatabase(config.GetSection("ViewStreamDatabase"));
-   services.AddPubSubClient(config.GetSection("PubSub"));
-   services.AddPrimitiveManager();
-   services.AddSpark(config.GetSection("Spark"));
-   services.AddSparkd(config.GetSection("SparkdCredentials"));
-   services.AddHostedService<DataStreamWebAppService>();
-}
@@ -1,26 +0,0 @@
-{
-  "$schema": "http://json.schemastore.org/launchsettings.json",
-  "profiles": {
-    "http": {
-      "commandName": "Project",
-      "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "launchUrl": "swagger",
-      "applicationUrl": "http://localhost:5000",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-    "https": {
-      "commandName": "Project",
-      "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "applicationUrl": "http://localhost:5000;https://localhost:5001",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-  },
-}
@@ -1,65 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Services;
-
-using BCG;
-using BCG.Utilities.General;
-
-using bts;
-using bts.api.core;
-
-using Bts.DataStreamExpressions;
-
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Options;
-
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Text.Json;
-using System.Threading;
-using System.Threading.Tasks;
-
-/// <summary>
-/// An IHostedService wrapper for ViewStream publishers. It provides a way for the Application to
-/// start/stop the view publishers.
-/// </summary>
-class DataStreamWebAppService : IHostedService {
-   readonly DataStreamWebAppServerConfig config;
-   readonly ViewStreamDatabase vsd;
-   readonly TaskCompletionSource taskCompletionSource = new();
-   readonly CancellationTokenSource cancellationTokenSource = new();
-
-   public DataStreamWebAppService(
-      ViewStreamDatabase vsd,
-      IOptions<DataStreamWebAppServerConfig> config
-   ) {
-      this.vsd = vsd;
-      this.config = config.Value;
-   }
-
-   public Task StartAsync(CancellationToken cancellationToken) {
-      // Task.Factory.StartNew(Run, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Default);
-      return Task.CompletedTask;
-
-      // void Run() {
-      //    Log.Gen.info("DataStreamWebAppService STARTED");
-      //    var defaultWaitTime = TimeSpan.FromMilliseconds(100);
-      //    var innerCancellationToken = cancellationTokenSource.Token;
-      //    var psConfig = config.PubSub ?? throw new InvalidOperationException("PubSub config is null");
-      //    while (!innerCancellationToken.IsCancellationRequested) {
-      //       vsd.doWork();
-      //       masterWorker.doWork();
-      //       Thread.Sleep(defaultWaitTime);
-      //    }
-      //    Log.Gen.info("DataStreamWebAppService STOPPED");
-      //    taskCompletionSource.SetResult();
-      // }
-   }
-
-   public Task StopAsync(CancellationToken cancellationToken) {
-      Log.Gen.info("Stopping DataStreamWebAppService");
-      cancellationTokenSource.Cancel();
-      return Task.CompletedTask;
-      // return taskCompletionSource.Task;
-   }
-}
@@ -1,6 +0,0 @@
-{
-  "WebServer": {
-    "ContentRootPath": "build/Bts.DataStreamWebApp.Client/production",
-    "WebRootPath": "dist"
-  }
-}
@@ -1,40 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  },
-  "AllowedHosts": "*",
-  "Spark": {
-    "ConfigServer": "http://localhost:8000/config"
-  },
-  "SparkdCredentials": {
-    "User": "bts",
-    "Password": ""
-  },
-  "MasterWorker": {
-    "Interval": "00:00:00.1"
-  },
-  "PubSub": {
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:12345",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:12345",
-          "TagRegExp": ".*"
-        }
-      ]
-    }
-  },
-  "ViewStreamDatabase": {
-    "Interval": "00:00:01"
-  }
-}
@@ -1,168 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <title></title>
-    <style>
-        table {
-            border: 0
-        }
-
-        .commslog-data {
-            font-family: Consolas, Courier New, Courier, monospace;
-        }
-
-        .commslog-server {
-            background-color: red;
-            color: white
-        }
-
-        .commslog-client {
-            background-color: green;
-            color: white
-        }
-    </style>
-</head>
-<body>
-    <h1>ALT WebSocket Sample Application</h1>
-    <p id="stateLabel">Ready to connect...</p>
-    <div>
-        <label for="connectionUrl">WebSocket Server URL:</label>
-        <input id="connectionUrl" />
-        <button id="connectButton" type="submit">Connect</button>
-    </div>
-    <p></p>
-    <div>
-        <label for="sendMessage">Message to send:</label>
-        <input id="sendMessage" disabled />
-        <button id="sendButton" type="submit" disabled>Send</button>
-        <button id="closeButton" disabled>Close Socket</button>
-    </div>
-
-    <h2>Communication Log</h2>
-    <table style="width: 800px">
-        <thead>
-            <tr>
-                <td style="width: 100px">From</td>
-                <td style="width: 100px">To</td>
-                <td>Data</td>
-            </tr>
-        </thead>
-        <tbody id="commsLog">
-        </tbody>
-    </table>
-
-    <script>
-        var connectionUrl = document.getElementById("connectionUrl");
-        var connectButton = document.getElementById("connectButton");
-        var stateLabel = document.getElementById("stateLabel");
-        var sendMessage = document.getElementById("sendMessage");
-        var sendButton = document.getElementById("sendButton");
-        var commsLog = document.getElementById("commsLog");
-        var closeButton = document.getElementById("closeButton");
-        var socket;
-
-        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
-        var port = document.location.port ? (":" + document.location.port) : "";
-
-        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
-
-        function updateState() {
-            function disable() {
-                sendMessage.disabled = true;
-                sendButton.disabled = true;
-                closeButton.disabled = true;
-            }
-            function enable() {
-                sendMessage.disabled = false;
-                sendButton.disabled = false;
-                closeButton.disabled = false;
-            }
-
-            connectionUrl.disabled = true;
-            connectButton.disabled = true;
-
-            if (!socket) {
-                disable();
-            } else {
-                switch (socket.readyState) {
-                    case WebSocket.CLOSED:
-                        stateLabel.innerHTML = "Closed";
-                        disable();
-                        connectionUrl.disabled = false;
-                        connectButton.disabled = false;
-                        break;
-                    case WebSocket.CLOSING:
-                        stateLabel.innerHTML = "Closing...";
-                        disable();
-                        break;
-                    case WebSocket.CONNECTING:
-                        stateLabel.innerHTML = "Connecting...";
-                        disable();
-                        break;
-                    case WebSocket.OPEN:
-                        stateLabel.innerHTML = "Open";
-                        enable();
-                        break;
-                    default:
-                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
-                        disable();
-                        break;
-                }
-            }
-        }
-
-        closeButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            socket.close(1000, "Closing from client");
-        };
-
-        sendButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            var data = sendMessage.value;
-            socket.send(data);
-            commsLog.innerHTML += '<tr>' +
-                '<td class="commslog-client">Client</td>' +
-                '<td class="commslog-server">Server</td>' +
-                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
-        };
-
-        connectButton.onclick = function() {
-            stateLabel.innerHTML = "Connecting";
-            socket = new WebSocket(connectionUrl.value);
-            socket.onopen = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
-                '</tr>';
-            };
-            socket.onclose = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
-                '</tr>';
-            };
-            socket.onerror = updateState;
-            socket.onmessage = function (event) {
-                commsLog.innerHTML += '<tr>' +
-                    '<td class="commslog-server">Server</td>' +
-                    '<td class="commslog-client">Client</td>' +
-                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
-            };
-        };
-
-        function htmlEscape(str) {
-            return str.toString()
-                .replace(/&/g, '&amp;')
-                .replace(/"/g, '&quot;')
-                .replace(/'/g, '&#39;')
-                .replace(/</g, '&lt;')
-                .replace(/>/g, '&gt;');
-        }
-    </script>
-</body>
-</html>
@@ -1,176 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <title></title>
-    <style>
-        table {
-            border: 0
-        }
-
-        .commslog-data {
-            font-family: Consolas, Courier New, Courier, monospace;
-        }
-
-        .commslog-server {
-            background-color: red;
-            color: white
-        }
-
-        .commslog-client {
-            background-color: green;
-            color: white
-        }
-    </style>
-</head>
-<body>
-    <h1>DataStreams via WebSocket</h1>
-    <p id="stateLabel">Ready to connect...</p>
-    <div>
-        <label for="connectionUrl">WebSocket Server URL:</label>
-        <input id="connectionUrl" />
-        <button id="connectButton" type="submit">Connect</button>
-    </div>
-    <p></p>
-    <div>
-        <label for="sendMessage">Query:</label>
-        <input id="sendMessage" disabled />
-        <button id="sendButton" type="submit" disabled>Send</button>
-        <button id="closeButton" disabled>Close Socket</button>
-    </div>
-
-    <h2>Communication Log</h2>
-    <table style="width: 800px">
-        <thead>
-            <tr>
-                <td style="width: 100px">From</td>
-                <td style="width: 100px">To</td>
-                <td>Data</td>
-            </tr>
-        </thead>
-        <tbody id="commsLog">
-        </tbody>
-    </table>
-
-    <script>
-        var connectionUrl = document.getElementById("connectionUrl");
-        var connectButton = document.getElementById("connectButton");
-        var stateLabel = document.getElementById("stateLabel");
-        var sendMessage = document.getElementById("sendMessage");
-        var sendButton = document.getElementById("sendButton");
-        var commsLog = document.getElementById("commsLog");
-        var closeButton = document.getElementById("closeButton");
-        var socket;
-
-        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
-        var port = document.location.port ? (":" + document.location.port) : "";
-
-        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
-
-        function updateState() {
-            function disable() {
-                sendMessage.disabled = true;
-                sendButton.disabled = true;
-                closeButton.disabled = true;
-            }
-            function enable() {
-                sendMessage.disabled = false;
-                sendButton.disabled = false;
-                closeButton.disabled = false;
-            }
-
-            connectionUrl.disabled = true;
-            connectButton.disabled = true;
-
-            if (!socket) {
-                disable();
-            } else {
-                switch (socket.readyState) {
-                    case WebSocket.CLOSED:
-                        stateLabel.innerHTML = "Closed";
-                        disable();
-                        connectionUrl.disabled = false;
-                        connectButton.disabled = false;
-                        break;
-                    case WebSocket.CLOSING:
-                        stateLabel.innerHTML = "Closing...";
-                        disable();
-                        break;
-                    case WebSocket.CONNECTING:
-                        stateLabel.innerHTML = "Connecting...";
-                        disable();
-                        break;
-                    case WebSocket.OPEN:
-                        stateLabel.innerHTML = "Open";
-                        enable();
-                        break;
-                    default:
-                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
-                        disable();
-                        break;
-                }
-            }
-        }
-
-        closeButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            socket.close(1000, "Closing from client");
-        };
-
-        sendButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            const rpcReq = {
-              jsonrpc: "2.0",
-              method: "subscribe",
-              params: {
-                query: sendMessage.value,
-              },
-              id: 1,
-            };
-            const rpcReqStr = JSON.stringify(rpcReq);
-            socket.send(rpcReqStr);
-            commsLog.innerHTML += '<tr>' +
-                '<td class="commslog-client">Client</td>' +
-                '<td class="commslog-server">Server</td>' +
-                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
-        };
-
-        connectButton.onclick = function() {
-            stateLabel.innerHTML = "Connecting";
-            socket = new WebSocket(connectionUrl.value);
-            socket.onopen = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
-                '</tr>';
-            };
-            socket.onclose = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
-                '</tr>';
-            };
-            socket.onerror = updateState;
-            socket.onmessage = function (event) {
-                commsLog.innerHTML += '<tr>' +
-                    '<td class="commslog-server">Server</td>' +
-                    '<td class="commslog-client">Client</td>' +
-                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
-            };
-        };
-
-        function htmlEscape(str) {
-            return str.toString()
-                .replace(/&/g, '&amp;')
-                .replace(/"/g, '&quot;')
-                .replace(/'/g, '&#39;')
-                .replace(/</g, '&lt;')
-                .replace(/>/g, '&gt;');
-        }
-    </script>
-</body>
-</html>
@@ -1,13 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
-  </ItemGroup>
-
-</Project>
@@ -1,11 +0,0 @@
-namespace Bts.Extensions.DependencyInjection;
-
-/// <summary>
-/// This interface provides a way to move dependencies outside of the constructor (and accept them
-/// via <see cref="IDependOn{T}.Reset(T?)"/> instead). It was designed to resolve circular
-/// dependency issues among startup services.
-/// </summary>
-/// <typeparam name="T">The type of dependency</typeparam>
-public interface IDependOn<T> {
-   void Reset(T? value);
-}
@@ -1,6 +0,0 @@
-#nullable enable
-namespace Bts.Extensions.DependencyInjection;
-
-public interface IDependOnKeyed<T> : IDependOn<T> {
-   string Key { get; }
-}
@@ -1,130 +0,0 @@
-namespace Bts.Extensions.DependencyInjection;
-
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-
-using System;
-using System.Linq;
-
-public static class ServiceCollectionExtensions {
-   /// <summary>
-   /// Adds a singleton service only if a service of type <typeparamref name="T"/> hasn't been added
-   /// already.
-   /// </summary>
-   /// <typeparam name="T"></typeparam>
-   /// <param name="services"></param>
-   /// <returns></returns>
-   public static IServiceCollection EnsureSingleton<T>(this IServiceCollection services) where T : class =>
-      services.HasService<T>()
-         ? services
-         : services.AddSingleton<T>();
-
-   /// <summary>
-   /// Adds a singleton service only if a service of type <typeparamref name="T"/> hasn't been added
-   /// already.
-   /// </summary>
-   /// <typeparam name="T"></typeparam>
-   /// <param name="services"></param>
-   /// <returns></returns>
-   public static IServiceCollection EnsureSingleton<T>(this IServiceCollection services, Func<IServiceProvider, T> implementationFactory) where T : class =>
-      services.HasService<T>()
-         ? services
-         : services.AddSingleton(implementationFactory);
-
-   /// <summary>
-   /// Adds a singleton service only if a service of type <typeparamref name="T"/> hasn't been added
-   /// already.
-   /// </summary>
-   /// <typeparam name="T">The type of the service to add.</typeparam>
-   /// <typeparam name="TImplementation">The type of the implementation to use.</typeparam>
-   /// <param name="services">The <see cref="IServiceCollection"/> to add the service to.</param>
-   /// <returns>A reference to this instance after the operation has completed.</returns>
-   public static IServiceCollection EnsureSingleton<T, TImplementation>(this IServiceCollection services) where T : class where TImplementation : class, T =>
-      services.HasService<T>()
-         ? services
-         : services.AddSingleton<T, TImplementation>();
-
-   /// <summary>
-   /// Adds a singleton service only if a service of type <typeparamref name="T"/> hasn't been added
-   /// already.
-   /// </summary>
-   /// <typeparam name="T">The type of the service to add.</typeparam>
-   /// <param name="services">The <see cref="IServiceCollection"/> to add the service to.</param>
-   /// <returns>A reference to this instance after the operation has completed.</returns>
-   public static IServiceCollection EnsureSingleton<T>(this IServiceCollection services, Func<IServiceProvider, object?, T> implementationFactory) where T : class =>
-      services.HasService<T>()
-         ? services
-         : services.AddSingleton(implementationFactory);
-
-   /// <summary>
-   /// Adds a singleton service only if a service of type <typeparamref name="T"/> hasn't been added
-   /// already.
-   /// </summary>
-   /// <typeparam name="T">The type of the service to add.</typeparam>
-   /// <typeparam name="TImplementation">The type of the implementation to use.</typeparam>
-   /// <param name="services">The <see cref="IServiceCollection"/> to add the service to.</param>
-   /// <returns>A reference to this instance after the operation has completed.</returns>
-   public static IServiceCollection EnsureSingleton<T, TImplementation>(this IServiceCollection services, Func<IServiceProvider, TImplementation> implementationFactory) where T : class where TImplementation : class, T =>
-      services.HasService<T>()
-         ? services
-         : services.AddSingleton(implementationFactory);
-
-   /// <summary>
-   /// Adds a singleton service only if a service of type <typeparamref name="T"/> hasn't been added
-   /// already.
-   /// </summary>
-   /// <typeparam name="T">The type of the service to add.</typeparam>
-   /// <param name="services">The <see cref="IServiceCollection"/> to add the service to.</param>
-   /// <returns>A reference to this instance after the operation has completed.</returns>
-   public static IServiceCollection EnsureKeyedSingleton<T>(this IServiceCollection services, object? key) where T : class =>
-      services.HasKeyedService<T>(key)
-         ? services
-         : services.AddKeyedSingleton<T>(key);
-
-   /// <summary>
-   /// Adds a singleton service only if a service of type <typeparamref name="T"/> hasn't been added
-   /// already.
-   /// </summary>
-   /// <typeparam name="T">The type of the service to add.</typeparam>
-   /// <param name="services">The <see cref="IServiceCollection"/> to add the service to.</param>
-   /// <param name="implementationFactory">The factory that creates the service.</param>
-   /// <returns>A reference to this instance after the operation has completed.</returns>
-   public static IServiceCollection EnsureKeyedSingleton<T>(this IServiceCollection services, object? key, Func<IServiceProvider, object?, T> implementationFactory) where T : class =>
-      services.HasKeyedService<T>(key)
-         ? services
-         : services.AddKeyedSingleton(key, implementationFactory);
-
-   /// <summary>
-   /// Adds a singleton service only if a service of type <typeparamref name="T"/> hasn't been added
-   /// already.
-   /// </summary>
-   /// <typeparam name="T">The type of the service to add.</typeparam>
-   /// <param name="services">The <see cref="IServiceCollection"/> to add the service to.</param>
-   /// <returns>A reference to this instance after the operation has completed.</returns>
-   public static IServiceCollection EnsureHostedService<T>(this IServiceCollection services) where T : class, IHostedService =>
-      services.HasService<T>()
-         ? services
-         : services.AddHostedService<T>();
-
-   /// <summary>
-   /// Adds a singleton service only if a service of type <typeparamref name="T"/> hasn't been added
-   /// already.
-   /// </summary>
-   /// <typeparam name="T">The type of the service to add.</typeparam>
-   /// <param name="services">The <see cref="IServiceCollection"/> to add the service to.</param>
-   /// <param name="implementationFactory">The factory that creates the service.</param>
-   /// <returns>A reference to this instance after the operation has completed.</returns>
-   public static IServiceCollection EnsureHostedService<T>(this IServiceCollection services, Func<IServiceProvider, T> implementationFactory) where T : class, IHostedService =>
-      services.HasService<T>()
-         ? services
-         : services.AddHostedService(implementationFactory);
-
-   public static bool HasService<T>(this IServiceCollection services) =>
-      services.Any(s => s.ServiceType.IsAssignableTo(typeof(T)));
-
-   public static bool HasServiceImplementation<T, TImplementation>(this IServiceCollection services) =>
-      services.Any(s => s.ServiceType.IsAssignableTo(typeof(T)) && s.ImplementationType == typeof(TImplementation));
-
-   public static bool HasKeyedService<T>(this IServiceCollection services, object? key) =>
-      services.Any(s => Equals(s.ServiceKey, key) && s.ServiceType.IsAssignableTo(typeof(T)));
-}
@@ -14,7 +14,6 @@
    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
    <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-    <ProjectReference Include="..\Bts.Extensions.DependencyInjection\Bts.Extensions.DependencyInjection.csproj" />
  </ItemGroup>

  <ItemGroup>
@@ -1,35 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using System;
-using System.Collections.Generic;
-
-public class BoundImportAttribute {
-   
-}
-
-public class DataStreamActivator {
-   Dictionary<string, Type> myTypeLookupTable = new();
-
-   /// <summary>
-   /// Constructs and returns a ViewStream of type corresponding to the friendly name
-   /// </summary>
-   /// <param name="friendlyName">friendly primitive name </param>
-   /// <param name="args">args can be null or an empty ICollection if the paramterless constructor
-   /// is the desired one</param>
-   /// <returns></returns>
-   public ViewStream Activate(string friendlyName, IList<object>? args) {
-      object[]? constructorArgs = null;
-      if (args != null && args.Count > 0) {
-         constructorArgs = new object[args.Count];
-         args.CopyTo(constructorArgs, 0);
-      }
-      if (!myTypeLookupTable.TryGetValue(friendlyName, out var type)) {
-         throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
-      }
-      return constructorArgs is null ?
-         Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type) as ViewStream :
-         Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream;
-   }
-}
@@ -1,7 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-public interface IViewStreamFactory {
-   public ViewStream? Create();
-}
@@ -11,8 +11,11 @@ public class PandasScriptStream : ViewStream {
   public PandasScriptStream(ViewStream vs, string file)
      : this(vs, file, null) { }

-   public PandasScriptStream(ViewStream vs, string file, string? pythonExe) {
-      mySetRealSource(AttachStream<PandasStream>(new(vs, readFile(), pythonExe)));
+   public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
+      : this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
+
+   public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
+      mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));

      string readFile() {
         try {
@@ -1,6 +1,7 @@
 namespace Bts.DataStream;

 using BCG;
+using BCG.Extensions;
 using BCG.Utilities.General;

 using bts;
@@ -22,6 +23,7 @@ public class PandasStream : ViewStream, IWorker {
   readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
   readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
   readonly string pythonExe;
+   readonly ViewStreamDatabase vsd;
   readonly CSVSaverStream inputCsvStream;
   readonly Lazy<string> pythonCode;
   IDictionary<int, IBlueRow>? lastRowMap = null;
@@ -32,11 +34,12 @@ public class PandasStream : ViewStream, IWorker {
   IDisposable? dispose;

   public PandasStream(ViewStream vs, string dataFrameExpression)
-      : this(vs, dataFrameExpression, null) { }
+      : this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }

-   public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe) {
+   public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
      pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
      this.pythonExe = pythonExe;
+      this.vsd = vsd;
      // This should fire an update when it writes to the output file
      inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
      pythonCode = new(getPythonCode);
@@ -56,7 +59,7 @@ import sys
   }

   protected override void myOnFirstSubscriber() {
-      base.myOnFirstSubscriber();
+      vsd.addWorker(this);
      inputCsvStream.onChange += onInputChanged;
      this.dispose = Disposable.Create(dispose);

@@ -65,6 +68,7 @@ import sys

      void dispose() {
         inputCsvStream.onChange -= onInputChanged;
+         vsd.removeWorker(this);
         deleteFile(inputFile);
         deleteFile(outputFile);
      }
@@ -81,10 +85,8 @@ import sys
      }
   }

-   protected override void myOnLastSubscriber() {
-      base.myOnLastSubscriber();
+   protected override void myOnLastSubscriber() =>
      Disposable.dispose(ref dispose);
-   }

   static void executePythonCode(string pythonCode, string? pythonExe = null) {
      pythonExe ??= "python";
@@ -10,6 +10,7 @@ namespace BCG {
        }
        private readonly string _ip;
        private readonly int _port;
+        private readonly ViewStreamDatabase _vsd;
        State _state = new State();
        private readonly int _msg_size;

@@ -18,10 +19,13 @@ namespace BCG {
        private ITCPConnection _connection = null;
        private byte[] _buff;

-        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) {
+        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
+        this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
+        public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
            _ip = ip;
            _port = port;
            _msg_size = msg_size;
+            _vsd = vsd;
            _schema = BlueDataUtility.form_schema("TCPClient",
            Tuple.Create("id", TypeCode.Int32),
            Tuple.Create("n_sends", TypeCode.UInt32),
@@ -72,4 +76,4 @@ namespace BCG {


    }
-}
+}
@@ -2,14 +2,13 @@ using System;
 using System.Collections.Generic;

 using Bts.DataStreamExpressions;
+using BCG.Utilities.General;
 using bts;
-using Bts.DataStreamCore;
-using Bts.PubSub;

 namespace BCG {
   // listens to the requests posted on "cloud_streams" request table, and 
   // if unfulfilled starts a ViewPubsliher
-   public class RemoteQueryManager : IParameterizedWorker {
+   public class RemoteQueryManager : IWorker {
      public class StreamPubUnit {
         public string name;
         public string stream_expression;
@@ -25,13 +24,12 @@ namespace BCG {
            view_publisher = view_publisher_;
         }
      }
-      static ViewStream? build_stream(string stream_expression, PrimitiveManager primitiveManager, IServiceProvider serviceProvider)
+      static ViewStream? build_stream(string stream_expression)
      {
         try
         {
            var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
-            var rv = exp.getView(new Dictionary<string, StatementExpression>(0), primitiveManager);
-            rv?.Reset(serviceProvider);
+            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
            return rv;

         } catch (Exception ex) {
@@ -42,25 +40,17 @@ namespace BCG {
      private IDictionary<string, StreamPubUnit>
         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
      private ViewStream _stream = new TableStream("cloud_streams");
-      readonly IPubTableBroker _broker;
-      private readonly ManagedSubscriber _sub;
-      readonly PrimitiveManager primitiveManager;
-      readonly IServiceProvider serviceProvider;
+      private PubTableBroker _broker = new PubTableBroker();
+      private ManagedSubscriber _sub = ManagedSubscriber.instance;

-      public RemoteQueryManager(ManagedSubscriber sub, IPubTableBroker broker, PrimitiveManager primitiveManager, IServiceProvider serviceProvider) {
-         _sub = sub;
-         _broker = broker;
-         this.primitiveManager = primitiveManager;
-         this.serviceProvider = serviceProvider;
+
+      public RemoteQueryManager() {
         _stream.onChange += _on_str;
         _sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
         });
      }
-
-      TimeSpan IParameterizedWorker.Interval => TimeSpan.FromSeconds(1);
-
      private void _on_str(object obj)
      {
         if (obj is not DataPacket dp) {
@@ -79,11 +69,7 @@ namespace BCG {

      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
-         var stream = build_stream(stream_expression, primitiveManager, serviceProvider);
-         if (stream is null) {
-            Log.Gen.error($"Failed to spawn remote stream. stream_expression: {stream_expression}");
-            return;
-         }
+         ViewStream stream = build_stream(stream_expression);
         _expose_to_ws(name, stream);
         if (persist) {
            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
@@ -1,6 +1,6 @@
-using bts;
+using BCG;

-using Microsoft.Extensions.DependencyInjection;
+using bts;

 using System;

@@ -9,7 +9,7 @@ namespace BCG {
   [DataStreamUserRespresentation("remoteQuery")]
   public class RemoteQueryStream : ViewStream {
      private readonly string _name, _stream_exp;
-      IPublisherCore? _publisher_core_backing;
+      private PublisherCore _publisher_core = new PublisherCore();
      private ViewStream _source_str;

      public RemoteQueryStream(params string[] lines) {
@@ -20,14 +20,7 @@ namespace BCG {

      }

-      IPublisherCore? _publisher_core =>
-         _publisher_core_backing ??= ServiceProvider?.GetService<IPublisherCore>();
-
      protected override void myOnFirstSubscriber() {
-         if (_publisher_core is null) {
-            throw new InvalidOperationException(
-               "Unable to initialize PublisherStream because PublisherCore is null");
-         }
         RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
         byte[] rem_req_bytes = rem_req.serialize();
         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
@@ -15,7 +15,7 @@ namespace BCG {

      public AddColumnsStream(ViewStream vs, string col_def) {
         _new_col = parse_exp(col_def);
-         ViewStream rs = AttachStream(new MapStream(vs, _get_schema, _map));
+         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
         mySetRealSource(rs);
      }

@@ -91,14 +91,10 @@ namespace BCG
         }
         _increment_values = increment_values_del;
         _get_column_prefix = get_column_prefix_del;
-         if(only_fire_once) {
-            ViewStream vs = AttachStream(new OnlyFireOnceStream(stream));
-            vs = AttachStream(new NonIncGroupByStream(vs, this));
-            mySetRealSource(vs);
-         } else {
-            ViewStream vs = AttachStream(new NonIncGroupByStream(stream, this));
-            mySetRealSource(vs);
-         }
+         if(only_fire_once)
+            mySetRealSource(new NonIncGroupByStream(new OnlyFireOnceStream(stream), this));
+         else
+            mySetRealSource(new NonIncGroupByStream(stream, this));
      }

      private void _set_schema(List<string> tc_prefixes, Pair<string, Type>[] group_by_col_info, string table_name)
@@ -245,4 +241,4 @@ namespace BCG
         return values;
      }
   }
-}
+}
@@ -14,7 +14,7 @@ namespace BCG
      public CaseChangeStream(ViewStream input, bool toUpper, params string[] columnsToChangeCase)
      {
         myCaseChangingColumns = columnsToChangeCase;         
-         mySetRealSource(AttachStream(new SuperMapStream(input, this)));
+         mySetRealSource(new SuperMapStream(input, this));
         myIsUpperCase = toUpper;
      }

@@ -32,10 +32,10 @@ namespace BCG
         
      {
         mySetRealSource (
-            AttachStream(new MapStream (sourceStream, 
+            new MapStream (sourceStream, 
                           isConcatenatedDataConfined ? new ConcatenateColumnsMapperForSameDataSet (columnDefs) :
                                                         new ConcatenateColumnsMapper (columnDefs)
-                        ))
+                        )
         );
      }
   }
@@ -10,7 +10,7 @@ namespace BCG
        public AverageCollectorStream(ViewStream stream, params string[] columnsToAverage)
        {        
            myAveragingColumns = columnsToAverage;
-            mySetRealSource(AttachStream(new CrushingCollectorStream(stream, this)));
+            mySetRealSource(new CrushingCollectorStream(stream, this));
        }
        
        private string[] myAveragingColumns;
@@ -1,23 +1,24 @@
+using bts;
+
 using System;
 using System.Collections;

-namespace BCG
-{
+namespace BCG {
 	/// <summary>
 	/// Provides a stream that with distinct rows with respect to the columns
 	/// the user specifies in the argument
 	/// </summary>
-	[DataStreamUserRespresentation ("distinct")]
-	public class DistinctRowStream : ViewStream
-	{
-		public DistinctRowStream (ViewStream stream, params string[] cols)
-		{
-         ArrayList groupByCols = new ArrayList ();
-         foreach (string col in cols)
-            foreach (string c in col.Split (' '))
-               groupByCols.Add (c);
-         mySetRealSource (AttachStream(new GroupByStream (stream, 
-            new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false)));
+	[DataStreamUserRespresentation("distinct")]
+	public class DistinctRowStream : ViewStream {
+		public DistinctRowStream(ViewStream stream, params string[] cols) {
+			var g_by_eval = new DistinctRowEvaluator(cols);
+			ViewStream r = new GroupByStream(stream, g_by_eval);
+			var proj_exp = String.Empty;
+			foreach (var c in cols)
+				proj_exp += $"{c} ";
+			proj_exp = proj_exp.Trim();
+			r = new NewProjectStream(r, proj_exp);
+			mySetRealSource(r);
 		}
 	}
 }
@@ -1,49 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
-
-// using System.Runtime.Serialization.Formatters.Binary;
-
-// namespace BCG
-// {
-//    public class DumpLoaderStream : ViewStream, ISideEffectStream
-//    {
-//       //wtf
-//       //public class Exception : System.Exception
-//       //{public Exception(string msg) : base(msg) { }}
-
-//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
-//       {
-//          mySetRealSource(
-//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-//                ViewStreamDatabase.instance));
-//       }
-
-//       public DumpLoaderStream(string fileName)
-//       {
-//         if (fileName == null)
-//            throw new LoaderStream.Exception("File name cannot be null");
-
-//         for (int i = 0; i < 5; i++)
-//            try
-//            {
-//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-//               {
-//                  IClientDeserializer deser = new ClientUniversalDeserializer();
-//                  byte[] the_ops;
-//                  Array the_rows;
-//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
-
-//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
-//               }
-//               break;
-//            }
-//            catch (IOException)
-//            {
-//               System.Threading.Thread.Sleep(50);
-//            }
-
-//       }
-//    }
-// }
@@ -1,6 +1,7 @@
 using BCG;

 namespace bts {
+   // This is the other side of the SaverStream, its "loads" the stream and fires once
   [DataStreamUserRespresentation("loader")]
   public class LoaderStream : ViewStream {

@@ -5,10 +5,11 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using bts;
+using BCG;

-namespace BCG {
-   // Writes all the udpates to a stream in a file.
+namespace bts {
+   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
+   // which just writes one update, as a create)
   // File naming:
   // token_epoch_yyyyMMdd_HHmmss
   // File location: directory(passed)\token\
@@ -94,10 +95,10 @@ namespace BCG {
      void _on_change(object new_value) {
         DataPacket dp = new_value as DataPacket;
         byte[] serialized_update = null;
-         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
+         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
         _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (_write_create) {
-            // new files always starts with create
+         if (first_write) {
+            // first file always starts with create
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
         } else
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -105,8 +106,8 @@ namespace BCG {
         long utc_epoch = epoch_now();

         //time, length of data, data
-         bw.Write(utc_epoch);
-         bw.Write(serialized_update.Length);
+         //bw.Write(utc_epoch);
+         //bw.Write(serialized_update.Length);
         bw.Write(serialized_update);
         _out_fs.Flush();
         myFireEvent(dp);
@@ -1,190 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
-
-
-// namespace BCG {
-
-//    public class PlaybackStream : ViewStream, IWorker {
-//       private readonly PlaybackType _type;
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       // these are the packets already downloaded from the current file
-//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
-//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
-//       private IEnumerable<string> _file_names;
-//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
-//       private DateTime _playback_t0, _playback_t1;
-//       private bool _all_files_read = false;
-//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
-
-//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
-//          _type = PlaybackType.Streaming;
-//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
-//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
-//          List<string> files_in_range = new List<string>();
-//          if (Directory.Exists(token_directory_name)) {
-//             _file_names =
-//          Directory.GetFiles(token_directory_name).Where(f => {
-//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
-//             return file_time > _playback_t0 && file_time < _playback_t1;
-//          });
-//          }
-//       }
-
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
-//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _snapshot_data.packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
-
-
-//       private void _load_packets_from_current_file() {
-//          int curr_idx = _current_fs.Item1;
-//          FileStream curr_fs = _current_fs.Item2;
-//          _packets = new List<PlaybackPacket>();
-//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
-//       }
-
-//       private bool current_file_all_replayed {
-//          get {
-//             bool rv = true;
-//             _packets.ForEach(p => rv = rv && p.replayed);
-//             return rv;
-//          }
-//       }
-
-//       protected override void myOnLastSubscriber() {
-//          var fs = _current_fs.Item2;
-//          if (fs != null) {
-//             fs.Close();
-//          }
-//          ViewStreamDatabase.instance.removeWorker(this);
-//       }
-
-//       // public event PlaybackStreamProgress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          //else
-//          {
-//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
-//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
-//             if (onProgress != null)
-//                onProgress(myFileReader.Position, myFileReader.Length);
-//          }
-//       }
-
-//       #region IWorker Members
-
-//       public void doWork() {
-//          if (_all_files_read)
-//             return;
-//          if (current_file_all_replayed) {
-//             _load_next_file();
-//          }
-//          PlaybackPacket curr_packet = _locate_packet_to_replay();
-//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
-//             byte[] ops;
-//             Array row_array;
-//             // what a stupid interface!
-//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//             _fire_packet(ops, row_array);
-//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//          } else {
-//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
-//             // this is the time we need to fire (replay) this packet
-//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
-//             long now_epoch = PersistorStream.epoch_now();
-//             // fire when we are past the 
-//             if (now_epoch >= time_to_fire) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//                _fire_packet(ops, row_array);
-//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//             }
-//          }
-//       }
-
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
-
-//          }
-//       }
-
-//       private void _load_next_file() {
-//          var new_idx = _current_fs.Item1 + 1;
-//          if (new_idx >= _file_names.Count()) {
-//             _all_files_read = true;
-//             bts.Log.Gen.info("All files read");
-//             return;
-//          }
-//          var prev_fs = _current_fs.Item2;
-//          if (prev_fs != null)
-//             prev_fs.Close();
-//          var new_filename = _file_names.ElementAt(new_idx);
-//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
-//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
-//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
-//          _load_packets_from_current_file();
-//       }
-
-//       #endregion
-//    }
-// }
@@ -1,68 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-
-// namespace BCG
-// {
-//    public class RecorderStream : ViewStream
-//    {
-//       private ViewStream mySource;
-//       private DateTime myStartTime;
-//       private FileStream myOutFile;
-//       private string myFileName;
-//       private IBlueDataSerializer mySerializer;
-
-//       public RecorderStream(ViewStream src, string recordFile)
-//       {
-//          mySource = src;
-//          myFileName = recordFile;
-
-//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-//       }
-
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStartTime = DateTime.Now;         
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//          myOutFile.Close();
-//       }
-
-//       void mySource_onChange(object newValue)
-//       {
-//          DataPacket dp = (DataPacket)newValue;         
-
-//          BlueTable toWrite = new BlueTable(dp.schema);
-
-//          if (mySerializer == null)
-//          {
-//             mySerializer = new UniversalSerializer(dp.schema);
-//          }
-//          foreach (IBlueRow r in dp.newRows)
-//             toWrite.addRow(r);
-
-//          byte[] result;
-
-//          if (myLastValue == null)
-//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-//          else
-//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
-
-//          BinaryWriter bw = new BinaryWriter(myOutFile);
-//          //write down what time this happened
-//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-//          bw.Write(result.Length);
-//          bw.Write(result);         
-//          myOutFile.Flush();
-
-//          myFireEvent(dp);
-//       }
-//    }
-// }
@@ -5,7 +5,8 @@ using System.IO;
 using BCG;

 namespace bts {
-   // this saves the total state in the filr provided in the cons
+   // this saves the updates as a Create on the same file, each update calls -save, which
+   // reqrites the whole data to the packet. The other side of this stream is the Loader
   [DataStreamUserRespresentation("saver")]
   public class SaverStream : ViewStream {
      private ViewStream _stream;
@@ -31,7 +32,6 @@ namespace bts {
         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
         StreamSerializationUtils.write_to_file(str_bytes, _filename);
-
      }
   }
 }
@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
+using bts;


 namespace BCG {
@@ -3,20 +3,25 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using BCG.Utilities.General;
-
 namespace BCG
 {
    public class DateBasedFilter: IDynamicStreamFilter, IWorker
    {
+        private ViewStreamDatabase myVSD;
        private string myDateColumn;
        private int myNDays;       
        private DateTime myReference;
        private DateTime myTodaysDate;
        
        public DateBasedFilter(string dateColumn, int nDays)
+            : this(dateColumn, nDays, ViewStreamDatabase.instance) { }
+
+        public DateBasedFilter(string dateColumn, int nDays, ViewStreamDatabase vsd)
        {
            myDateColumn = dateColumn;
            myNDays = nDays;
+            myVSD = vsd;
+            myVSD.addWorker(this);
            myTodaysDate = DateTime.Today;
            myReference = myTodaysDate.AddDays(-1 * myNDays);                     
        }       
@@ -1,12 +1,7 @@
-using BCG.Utilities.General;
-
 using Bts.DataStream;
-using Bts.PubSub;

 using mbts.config;

-using Spark.Core.Disposables;
-
 using System;
 using System.Text.RegularExpressions;

@@ -146,7 +141,6 @@ namespace BCG
      private BlueTable myLastFiredRows;
      private IBlueRowCollection myLastReceivedRows;
      private IStreamFilter myFilter;
-      private IDisposable? filterWorkerSubscription;
      private ViewStream myParentStream;
      private string myFilterString;
      private BCG.StreamWorld.Addition.FilterStream myFilterStream;
@@ -174,7 +168,7 @@ namespace BCG
         if (myUseNewFilter)
         {
            myFilterStream = new BCG.StreamWorld.Addition.FilterStream(vs, filter);
-            mySetRealSource(AttachStream(myFilterStream));
+            mySetRealSource(myFilterStream);
         }
         else
         {
@@ -266,7 +260,6 @@ namespace BCG
            }
            else
            {
-               Disposable.dispose(ref filterWorkerSubscription);
               myFilter = value;
               if (numberOfSubscribers > 0)
               {
@@ -281,9 +274,6 @@ namespace BCG
                       BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
                       myLastFiredRows = nRows;
                       myFireEvent(new DataPacket(nRows, aRows, dRows));
-                       if (myFilter is IWorker filterWorker) {
-                           filterWorkerSubscription = ViewStreamDatabase.Subscribe(filterWorker);
-                       }
                   }
               }
            }
@@ -304,14 +294,6 @@ namespace BCG
         return GetType().ToString() + " " + myFilterString;
      }

-      protected override void OnViewStreamDatabase(IWorkScheduler viewStreamDatabase) {
-         base.OnViewStreamDatabase(viewStreamDatabase);
-         var w = this as IWorker;
-         if (w is not null) {
-            filterWorkerSubscription = viewStreamDatabase.Subscribe(w);
-         }
-      }
-
      private void myHandleFilterChange()
      {
         filter = myFilter;
@@ -4,13 +4,11 @@ using System.Linq;
 using System.Text;
 using BCG.Utilities.General;

-using Bts.PubSub;
-
 namespace BCG
 {    
    public class SlidingWindowFilter: IDynamicStreamFilter, IWorker
    {
-        private IWorkScheduler workScheduler;
+        private ViewStreamDatabase myVSD;
        private string myDateColumn;
        private int myNSecs;     
        private DateTime myReference;
@@ -19,9 +17,14 @@ namespace BCG
        private DateTime myLastTimeFilteredApplied;
        
        public SlidingWindowFilter(string dateColumn, int nSecs)
+            : this(dateColumn, nSecs, ViewStreamDatabase.instance) { }       
+        
+        public SlidingWindowFilter(string dateColumn, int nSecs, ViewStreamDatabase vsd)
        {
            myDateColumn = dateColumn;
            myNSecs = nSecs;
+            myVSD = vsd;
+            myVSD.addWorker(this);
            myCurrentTime = DateTime.Now;
            myReference = myCurrentTime.AddSeconds(-1 * myNSecs);                  
        }
@@ -52,9 +55,5 @@ namespace BCG
                
            }           
        }
-        
-        public void Dispose() {
-            workScheduler.Remove(this);
-        }
    }
 }
@@ -4,7 +4,7 @@ using System.Text.RegularExpressions;
 using BCG.Utilities.General;

 namespace BCG {
-   public class TimeBasedFilter : IDynamicStreamFilter, IWorker {
+   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable {

      public static TimeSpan parse_duration(string input) {
         int days = 0, hours = 0, minutes = 0, seconds = 0;
@@ -42,15 +42,28 @@ namespace BCG {
      : this(col_name, parse_duration(row_time_to_live)) {

      }
+
      public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
+      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
      }

-      public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
+      public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
+      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) {
+      }
+
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
+      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
+      }
+
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
+         myVsd = vsd;
         _time_col = rowTimeColumnName;
         _ttl = rowTimeToLive;
         myFilterFrequency = filterFrequency;
         myNextFilter = DateTime.MinValue;
+
+         // add worker
+         myVsd.addWorker(this);
      }
      #endregion

@@ -84,11 +97,29 @@ namespace BCG {
      }
      #endregion

+      #region IDisposable
+      public void Dispose() {
+         // remove worker
+         myVsd.removeWorker(this);
+      }
+      #endregion
+
      #region state
+      private ViewStreamDatabase myVsd;
      private string _time_col;
      private TimeSpan _ttl;
      private TimeSpan myFilterFrequency;
      private DateTime myNextFilter;
      #endregion
   }
+   [DataStreamUserRespresentation("timefilter")]
+   public class TimeBasedFilterStream : ViewStream {
+      public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
+      :
+      this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
+      public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
+         TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
+         mySetRealSource(new FilterStream(base_stream, filter));
+      }
+   }
 }
@@ -1,14 +0,0 @@
-namespace BCG;
-
-using System;
-
-[DataStreamUserRespresentation("timefilter")]
-public class TimeBasedFilterStream : ViewStream {
-   public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
-      : this(base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
-
-   public TimeBasedFilterStream(ViewStream base_stream, string time_col, TimeSpan time_span) {
-      var filter = new TimeBasedFilter(time_col, time_span);
-      mySetRealSource(AttachStream<FilterStream>(new(base_stream, filter)));
-   }
-}
@@ -5,8 +5,8 @@ namespace BCG {
        public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
        }
        public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
-            mySetRealSource(AttachStream(new GroupByStream(stream,
-                                                new MaxGroupByEvaluator(outputCol, inputCols))));
+            mySetRealSource(new GroupByStream(stream,
+                                                new MaxGroupByEvaluator(outputCol, inputCols)));
        }

        public MaxStream(ViewStream stream, string outputCol)
@@ -1,15 +1,13 @@
-namespace BCG {
-   public class MinStream : ViewStream {
-      public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
-         mySetRealSource(AttachStream(new GroupByStream(stream,
-                           new MinGroupByEvaluator(outputCol, inputCol))));
-      }
+using System;

-      public MinStream(ViewStream stream, string outputCol)
-         : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) {
-            ViewStream vs = AttachStream(new AddColumnsStream(stream, "fakeCol=0"));
-            vs = AttachStream(new GroupByStream(vs, new MinGroupByEvaluator(outputCol, new[] { "fakeCol" })));
-            mySetRealSource(vs);
-      }
-   }
+namespace BCG {
+    public class MinStream : ViewStream {
+        public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
+            mySetRealSource(new GroupByStream(stream,
+                              new MinGroupByEvaluator(outputCol, inputCol)));
+        }
+
+        public MinStream(ViewStream stream, string outputCol)
+                       : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
 }
@@ -26,22 +26,22 @@ namespace BCG {
        public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-            mySetRealSource(AttachStream(new GroupByStream(stream,
+            mySetRealSource(new GroupByStream(stream,
                                              outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
-                                              new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))));
+                                              new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
        }

        public SumStream(ViewStream stream, string[] output_columns) {
-            ViewStream rs = AttachStream(new AddColumnsStream(stream, "fake_col=0"));
-            rs = AttachStream(new SumStream(rs, output_columns, new string[] { "fake_col" }, false));
-            rs = AttachStream(new ProjectStream(rs, "!fake_col"));
+            ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
+            rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
+            rs = new ProjectStream(rs, "!fake_col");
            mySetRealSource(rs);
        }

        public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
            string[] output_columns = space_seperated_output_columns.Split(' ');
            string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
-            mySetRealSource(AttachStream(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns)));
+            mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
        }

        public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
@@ -52,13 +52,12 @@ namespace BCG {
                // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
                // rs = new ProjectStream(rs, "!fake_col");
            } else {
-                rs = AttachStream(new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns)));
+                rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
            }
            if (force_null_into_other_columns) {
-                ArgumentNullException.ThrowIfNull(rs);
                List<string> valid_cols = new List<string>(output_columns);
                valid_cols.AddRange(groupby_columns);
-                rs = AttachStream(new MapStream(rs, new NullMapper(valid_cols.ToArray())));
+                rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
            }
            mySetRealSource(rs);
        }
@@ -72,9 +71,9 @@ namespace BCG {
        public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-            mySetRealSource(AttachStream(new GroupByStream(stream,
+            mySetRealSource(new GroupByStream(stream,
                                              outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
-                                              new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))));
+                                              new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
        }

    }
@@ -83,8 +82,8 @@ namespace BCG {
    public class SumStream<T> : ViewStream {
        public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
            string[] inputCols = spaceSeparatedInputCols.Split(' ');
-            mySetRealSource(AttachStream(new GroupByStream(stream,
-                                                new SumGroupByEvaluator<T>(outputCol, inputCols))));
+            mySetRealSource(new GroupByStream(stream,
+                                                new SumGroupByEvaluator<T>(outputCol, inputCols)));
        }
    }
 }
@@ -22,9 +22,9 @@ namespace BCG
      {
         if (myUseSuperJoin)
         {
-            ViewStream rs = AttachStream(this is LeftJoinStream
-               ? new SuperLeftJoinStream(v1, joinCol1, cm1.columnExpression, v2, joinCol2, cm2.columnExpression)
-               : new SuperJoinStream(v1, joinCol1, cm1.columnExpression, v2, joinCol2, cm2.columnExpression));
+            ViewStream rs = this is LeftJoinStream ?
+               new SuperLeftJoinStream(v1, joinCol1, cm1.columnExpression, v2, joinCol2, cm2.columnExpression) as ViewStream :
+               new SuperJoinStream(v1, joinCol1, cm1.columnExpression, v2, joinCol2, cm2.columnExpression) as ViewStream;
            mySetRealSource(rs);
         }
         else
@@ -65,9 +65,9 @@ namespace BCG
      {
         if (myUseSuperJoin)
         {
-            ViewStream rs = AttachStream(this is LeftJoinStream
-               ? new SuperLeftJoinStream(v1, joinCol1, v2, joinCol2)
-               : new SuperJoinStream(v1, joinCol1, v2, joinCol2));
+            ViewStream rs = this is LeftJoinStream ?
+               new SuperLeftJoinStream(v1, joinCol1, v2, joinCol2) as ViewStream :
+               new SuperJoinStream(v1, joinCol1, v2, joinCol2) as ViewStream;
            mySetRealSource(rs);
         }
         else
@@ -27,10 +27,10 @@ namespace BCG
                  rightJoinColArray = rightJoinColumns.Split(' ', ',', ';');
         if (leftJoinColArray.Length == 1 && rightJoinColArray.Length == 1)
            /// so that this handles regular joins too
-            mySetRealSource(AttachStream(leftJoin ? new SuperLeftJoinStream(leftStream, leftJoinColArray[0], leftProject,
+            mySetRealSource(leftJoin ? new SuperLeftJoinStream(leftStream, leftJoinColArray[0], leftProject,
                                                               rightStream, rightJoinColArray[0], rightProject) :
                                       new SuperJoinStream(leftStream, leftJoinColArray[0], leftProject,
-                                                              rightStream, rightJoinColArray[0], rightProject)));
+                                                              rightStream, rightJoinColArray[0], rightProject));
         else
         {
            string leftCols = "ljc:";
@@ -41,12 +41,12 @@ namespace BCG
            foreach (string c in rightJoinColArray)
               rightCols += c + " ";
            rightCols = rightCols.Trim();
-            leftStream = AttachStream(new ConcatenateColumnsStream(leftStream, leftCols));
-            rightStream = AttachStream(new ConcatenateColumnsStream(rightStream, rightCols));
-            ViewStream res = AttachStream(leftJoin
-               ? new SuperLeftJoinStream(leftStream, "ljc", leftProject, rightStream, "rjc", rightProject)
-               : new SuperJoinStream(leftStream, "ljc", leftProject, rightStream, "rjc", rightProject));
-            res = AttachStream(new ProjectStream(res, "!ljc rjc"));
+            ViewStream res =
+               leftJoin ? new SuperLeftJoinStream(new ConcatenateColumnsStream(leftStream, leftCols), "ljc", leftProject,
+                                                   new ConcatenateColumnsStream(rightStream, rightCols), "rjc", rightProject) :
+                           new SuperJoinStream(new ConcatenateColumnsStream(leftStream, leftCols), "ljc", leftProject,
+                                                   new ConcatenateColumnsStream(rightStream, rightCols), "rjc", rightProject);
+            res = new ProjectStream(res, "!ljc rjc");
            mySetRealSource(res);
         }
      }
@@ -15,6 +15,7 @@ namespace BCG
      private readonly INonIncJoinInlineMapper _output_mapper;
      private readonly BlueRowHasher _row_hasher;
      private readonly TimeSpan _throttle_period;
+      private readonly ViewStreamDatabase _vsd;

      private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
      private IBlueTableSchema _output_schema = null;
@@ -34,17 +35,22 @@ namespace BCG
         : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
      { }

-      public NonIncJoinStream(INonIncJoinInlineMapper output_mapper, int throttles_ms,
+      public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
                              params Triplet<ViewStream, string, string>[] join_specs)
-         : this(NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
+         : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
      { }

      public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
+         : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
+      { }
+
+      public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
      {
         _join_specs = join_specs;
         _output_mapper = output_mapper;
         _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
         _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
+         _vsd = vsd;
      }

      protected override void myOnFirstSubscriber()
@@ -59,7 +65,7 @@ namespace BCG
         }
         if (_output_mapper != null)
            _output_mapper.onMapperChange += _on_mapper_change;
-         base.myOnFirstSubscriber();
+         _vsd.addWorker(this);
      }

      protected override void myOnLastSubscriber()
@@ -68,7 +74,7 @@ namespace BCG
            d.Dispose();
         if (_output_mapper != null)
            _output_mapper.onMapperChange -= _on_mapper_change;
-         base.myOnLastSubscriber();
+         _vsd.removeWorker(this);
      }

      private void _stream_callback()
@@ -1,7 +1,9 @@
-#nullable disable
-using System;
+using System;
 using System.Collections;
 using System.Collections.Generic;
+using System.Text;
+using BCG.Utilities.General;
+using BCG.Utilities;

 namespace BCG
 {
@@ -52,7 +52,7 @@ namespace BCG {
               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
            }

-         mySetRealSource(AttachStream(new BlueRowCollectionStaticStream(t)));
+         mySetRealSource(new BlueRowCollectionStaticStream(t));
      }

      public LiteralStream(string schema, string value) : this("literal", schema, value) { }
@@ -1,10 +1,6 @@
-using bts;
-
 using Bts.DataStream;
-using Bts.DataStreamCore;

 using mbts.config;
-using mbts.utils;

 using System;

@@ -20,8 +16,8 @@ namespace BCG {
   [DataStreamUserRespresentation (false)]
   public class MapStream : MethodDrivenStream
   {
-      protected IStreamMapper? myMapper;
-      protected IChangeableStreamMapper? myChangeableMapper;
+      protected IStreamMapper myMapper;
+      protected IChangeableStreamMapper myChangeableMapper = null;
      private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;

      public class FuncWrapper : IStreamMapper {
@@ -44,7 +40,7 @@ namespace BCG {

      public MapStream(ViewStream vs, IStreamMapper mapper) : base(vs) {
         if (myUseNewMap)
-            mySetRealSource(AttachStream(new SuperMapStream(vs, mapper)));
+            mySetRealSource(new SuperMapStream(vs, mapper));
         else
            myMapper = mapper;
      }
@@ -57,7 +53,7 @@ namespace BCG {

      public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this(vs, (IStreamMapper)mapper) {
         if (myUseNewMap)
-            mySetRealSource(AttachStream(new SuperMapStream(vs, mapper)));
+            mySetRealSource(new SuperMapStream(vs, mapper));
         else {
            mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
            myChangeableMapper = mapper;
@@ -76,7 +72,7 @@ namespace BCG {


      protected override void myHandleFirstCall() {
-         ArgumentNullException.ThrowIfNull(myMapper);
+
         IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
         myLastFiredPacket = new DataPacket();
         myLastFiredPacket.newRows = new BlueTable(outputSchema);
@@ -87,34 +83,28 @@ namespace BCG {
      }

      protected override void myHandleIncrementalAdd() {
-         if (myLastArrivedPacket.addedRows.Count == 0) {
-            Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
-            return;
-         }
-         ArgumentNullException.ThrowIfNull(myMapper);
-         var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
-         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
+         //t.start ();
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
-            var newRow = myMapper.map(row);
-            newRows.addRow(newRow);
-            addedRows.addRow(newRow);
+            IBlueRow newRow = myMapper.map(row);
+            (myLastFiredPacket.newRows as BlueTable).addRow(newRow);
+            (myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
         }
-         myLastFiredPacket.addedRows = addedRows;
+         //t.stop ();
+         //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
+         //System.Diagnostics.Trace.WriteLine (" ");
      }

      protected override void myHandleIncrementalDelete() {
-         if (myLastArrivedPacket.deletedRows.Count == 0) {
-            Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
-            return;
-         }
-         var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
-         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
+         //t.start ();
+         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
-            var delRow = newRows[row.Id()];
-            deletedRows.addRow(delRow);
-            newRows.deleteRow(row);
+            IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
+            (myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
+            (myLastFiredPacket.newRows as BlueTable).deleteRow(row);
         }
-         myLastFiredPacket.deletedRows = deletedRows;
+         //t.stop ();
+         //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
      }
Author	SHA1	Message	Date
Adeel Khan	11e6d25001	Add: Incremental Books	2024-10-04 17:05:02 +00:00
Mohammad Arham	83d66ae9d7	Remove: code to remove secuties	2024-10-03 18:03:10 +05:00
Mohammad Arham	e5e74b7b42	Add: code to connect to Live databento	2024-10-03 18:03:08 +05:00
Mohammad Arham	6ce2fb2cc5	Add: databento repllay filename to cofig	2024-10-03 18:01:49 +05:00
Mohammad Arham	9d63a004ec	Add: added onix fix to lib to build dir	2024-10-03 18:01:49 +05:00
Mohammad Arham	ccd53d71ea	Add: entry for onix fix	2024-10-03 18:01:49 +05:00
Mohammad Arham	74a0235227	Add databento to external	2024-10-03 18:01:49 +05:00
Taha Afzal	42bb101b43	Adding comments	2024-10-03 18:01:49 +05:00
Taha Afzal	c53561c6ef	Creating scaffolding for incremental book updates in GeneralMarketDataListener	2024-10-03 18:01:49 +05:00
Taha	3701a70bd9	Fixing build relted to BCG vs bts namespace	2024-10-03 18:01:49 +05:00
Taha	df56de3dd2	Cleaning up FileStreams	2024-10-03 18:01:49 +05:00
Taha Afzal	c9a418ce4b	spaces, newlines?	2024-10-03 18:01:49 +05:00
Taha	aaad404507	Added benzinga streamd to MDPub's test config	2024-10-03 18:01:49 +05:00
Taha Afzal	45acaa20b7	PubSubSingleTagState is not needed any more	2024-10-03 18:01:49 +05:00
Taha Afzal	96a08500fb	Fixing problems where PSS's client socket's IP was different than what the client was sending in PubSubClientSubscription	2024-10-03 18:01:49 +05:00
Taha Afzal	510e7b0050	Fixing PubsubEchoTestStream	2024-10-03 18:01:47 +05:00
Taha Afzal	2ab76c753e	Merging	2024-10-03 17:57:34 +05:00
Taha Afzal	a02c6b77d4	src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs	2024-10-03 17:57:32 +05:00
Taha	6a289a5561	Icomplete SharedStreamsConfig was breaking build, fixing	2024-10-03 17:55:30 +05:00
Taha	fddd310a03	Added new benzinga streams	2024-10-03 17:55:30 +05:00
Taha	d2a2d5fb37	Adding TableMetaDataStream	2024-10-03 17:55:30 +05:00
Taha Afzal	4b76442a8f	Cleaning Config 1.0 (not completely done)	2024-10-03 17:55:30 +05:00
Taha	09b863f2ab	Added pubsub routes to the Title of DSQA	2024-10-03 17:55:30 +05:00
Taha	dd221aba10	Fixing the bug where messages were being sent by the pubsubserver out of order	2024-10-03 17:55:28 +05:00
Taha	5ef862b026	some buf gixes + cleanup	2024-10-03 17:54:28 +05:00
Taha	cc43e9d3a4	PubSuMessageReactor empties the message queue (instead oif one elem ata a time) in the main loop	2024-10-03 17:52:47 +05:00
Taha	b6eb727bc9	PubSubServer's linux issues fixed	2024-10-03 17:52:38 +05:00
John Colvin	8ace69ab74	Add CBOE to spark admin webapp	2024-10-03 17:46:51 +05:00
John Colvin	dc88b4ee10	Add TAIFEX to spark admin webapp	2024-10-03 17:46:51 +05:00
Taha	dee32dead4	Fixing implied vols for SPX (market data was getting divide dby the fwd)	2024-10-03 17:46:51 +05:00
John Colvin	b2e022505f	fix build: Add ".list" to SymbolLists config For new exchanges, TAIFEX and CBOE	2024-10-03 17:46:51 +05:00
John Colvin	ac9b3ec365	fix (build): Add break statement to new venue switch cases Namely, TAIFEX and CBOE	2024-10-03 17:46:51 +05:00
John Colvin	ed89da7f38	fix: Preserve vs exp sec type (Option) when computing implied vol Because the exp keys in the data map have sec type Option.	2024-10-03 17:46:51 +05:00
John Colvin	a334d9be88	ui: Highlight launcher buttons when selected Namely, the expandable menu buttons in Analytics shell module	2024-10-03 17:46:51 +05:00
John Colvin	aee56569d1	Replace trivial refs to ThreadedBlackboard w/ IBus	2024-10-03 17:46:51 +05:00
John Colvin	7fdd3f6b1b	fix: Move accounts subscription to dispatcher thread We update a collection that is bound to the view in the callback (mutation ops throw if performed on a different thread).	2024-10-03 17:46:51 +05:00
John Colvin	3e1949d94f	fix: Check 'HasTimestamp' before accessing prop in man fill row Pending adds (i.e., unpersisted new rows) don't have timestamps. In such cases, the 'Timestamp' getter throws an exception.	2024-10-03 17:46:51 +05:00
John Colvin	4cb3dbc313	Cache/index options for faster lookups in SystemContractData	2024-10-03 17:46:51 +05:00
John Colvin	0268bd476f	fix: Include venue w/ symbol when publishing interp greeks	2024-10-03 17:46:51 +05:00
John Colvin	2511b0d966	Fix typo in comment	2024-10-03 17:46:50 +05:00
John Colvin	92456649df	fix: SelectorList.RemoveAt doesn't need 'convert_back' delegate So don't throw an exception if it is null.	2024-10-03 17:46:50 +05:00
John Colvin	ec32078dd5	fix: Don't assume venue is in spread def string Pass it as an argument to the parse fn. This should be fine for now, but obviously a singular venue param won't work for cross exchange spreads.	2024-10-03 17:46:50 +05:00
John Colvin	6bcce832ac	improvement: Move system contracts sub off of dispatcher thread Also add 200ms throttle. Contracts shouldn't change that often, so we don't need lightening speed here. Furthermore, such changes may trigger expensive cpu ops. Note: this change also replaces refs to ThreadedBlackboard w/ IBus (the former is somewhat tethered to the dispatcher thread, particularly by its timer_callback).	2024-10-03 17:46:50 +05:00
John Colvin	85529b2232	fix: Pass symbol/venue from vs usage cfg to AnalyticsService proto	2024-10-03 17:46:50 +05:00
John Colvin	c3cb8995bf	cleanup: Remove unused code	2024-10-03 17:46:50 +05:00
John Colvin	ab2f0f975e	refactor: Index vol and theos data by symbol/venue tuple Rather than just symbol, to avoid conflicts when the same symbol exists at multiple exchanges	2024-10-03 17:46:50 +05:00
John Colvin	2e133ea182	cleanup: Remove redundant type VolSurfaceSymbolStructure There is an identical type (in terms of prop types and corresponding proto indices) called VolSurfaceSymbolRelationship that is used in more places.	2024-10-03 17:46:50 +05:00
John Colvin	30615f3429	Move nested vm classes into own files VolSurfaceSettingsEditor has too much code (still). It is hard to navigate.	2024-10-03 17:46:50 +05:00
John Colvin	61371804a2	Simplify driver symbol structure in vs settings vm The code was confusing and difficult to follow.	2024-10-03 17:46:50 +05:00
John Colvin	02d82e39bc	fix: Raise prop changed event when is_selected changes	2024-10-03 17:46:50 +05:00
John Colvin	50188e6688	Add venue to vs calc fns	2024-10-03 17:46:50 +05:00
John Colvin	5137785f7f	fix: Provide venue param w/ symbol when computing fut slide	2024-10-03 17:46:50 +05:00
John Colvin	def021612c	diag: Log added/removed exps when syncing vs input w/ system exps This shouldn't happen often, but if it does, we should know about it. PS: We recently fixed a bug caused by inequalities between VolSurfaceExpirations in vs input, and in SystemContractData. User inputs to the vs grid would be immediately reverted, as the exps associated with the inputs were not found in the SystemContractData. This bug would have been easier to track down with this logging in place.	2024-10-03 17:46:50 +05:00
John Colvin	217c5be131	Add venue to VolSurface(DisplayGrid)Expiration So we can handle symbol/exp date conflicts for securities that differ only by venue (exchange)	2024-10-03 17:46:50 +05:00
John Colvin	a75441cd07	Clean up VolSurfaceExpiration * Enable nullable lang feat * Use file-scoped namespace * Sort members * Include sec type in Compare impl	2024-10-03 17:46:50 +05:00
John Colvin	78eb096d6f	Add null check in SelectorList ctr	2024-10-03 17:46:50 +05:00
John Colvin	2b66f46db1	cleanup: Remove unused code	2024-10-03 17:46:50 +05:00
John Colvin	ae3029d9ed	refactor: Replace ThreadedBlackboard w/ IBus In SingleVolSurfaceCommittedOutputRetriever	2024-10-03 17:46:50 +05:00
John Colvin	d4a0603933	Clean up VolSurfaceTimeParameters * Enable nullable lang feat * Use file-scoped namespace * Implement ICloneable	2024-10-03 17:46:50 +05:00
John Colvin	db6e96cf00	cleanup: Remove unnecessary ternary op	2024-10-03 17:46:50 +05:00
John Colvin	371db4afd2	fix: Actually use (sec) type arg in to_security_expiration	2024-10-03 17:46:50 +05:00
John Colvin	08484a0954	fix: Check security_type in vs exp Equals impl	2024-10-03 17:46:50 +05:00
John Colvin	39db44ed63	fix: Maintain sec type when converting SecurityExpiration to vs exp Avoids data loss	2024-10-03 17:46:50 +05:00
John Colvin	b4dca8e23f	refactor: Get rid of sec type 'Option' default (magic const) Better to be explict	2024-10-03 17:46:50 +05:00
John Colvin	b4bf06d17a	Fix selection binding issues in vs settings editor This specifically addresses issues in the driver symbol grids. Unfortunately, we can't bind directly to ComboCheckBox.SelectedItems because the view is contained in TabControl. When the selected tab changes, the previously selected tab clears its ItemsSource (and consequently the SelectedItems) as a side effect of being unloaded.	2024-10-03 17:46:50 +05:00
John Colvin	8c17705f02	refactor: Enable nullable (ref types) in ImmutableListExtensions	2024-10-03 17:46:50 +05:00
John Colvin	551f26e5f3	Fix typo in namespace bts.collection -> bts.collections	2024-10-03 17:46:50 +05:00
John Colvin	f4adf35e49	fix: Use ComboCheckBoxItem.IsSelectedProperty Instead of Selector.IsSelectedProperty. Unlike ComboBoxItem, ComboCheckItem's IsSelectedProperty isn't linked to Selector.IsSelectedProperty.	2024-10-03 17:46:50 +05:00
John Colvin	90128863d9	Add currency column to mview	2024-10-03 17:46:50 +05:00
John Colvin	300831c97a	fix: Use SetCurrentValue instead of SetValue So we propagate values to/from the binding data source, rather than overwrite the data source itself.	2024-10-03 17:46:50 +05:00
John Colvin	f3f6a82d9a	Add venue, sec type dropdown filters to exp params ui	2024-10-03 17:46:50 +05:00
John Colvin	7c8a41b2b7	fix: Add null check in ComboCheckBox.SelectedItemsCollectionChanged	2024-10-03 17:46:50 +05:00
John Colvin	408114c1f6	fix: Disconnect old INotifyCollectionChanged source when dp changes	2024-10-03 17:46:50 +05:00
John Colvin	5c9be0bfb9	fix: Index opt/fut exps by venue in vs analytics	2024-10-03 17:46:50 +05:00
John Colvin	c0f970ab89	fix: Support multiple exchanges in exp params view/vm	2024-10-03 17:46:50 +05:00
John Colvin	25e1804286	cleanup: Remove unused 'option_symbol' param from compute_atm_data	2024-10-03 17:46:50 +05:00
John Colvin	3c84dc48f6	fix: Filter by exch when getting atm data for sheets This is necessary now that we support exchanges with overlapping securities	2024-10-03 17:46:50 +05:00
John Colvin	9ae20f0c4d	fix: Remove sec id prop from SecurityExpiration This class isn't designed to represent a particular contract; it's basically a key to get option theos.	2024-10-03 17:46:50 +05:00
John Colvin	e8a72e7bd9	refactor: Add 'onDispatcher' param to 'ThreadedBlackboard.subscribe' This will allow us to gradually move subscriptions off of the dispatcher (render) thread. When the render thread is pegged, the ui is laggy and unresponsive. The new param defaults to true, so it won't change anything until we explicitly set it to false in 'subscribe' calls.	2024-10-03 17:46:50 +05:00
John Colvin	e99e2d817a	fix: Remove redundant SparkTimer threads This addresses some slowness on the front end. Prior to this commit, each ThreadedBlackboard would start its own timer thread. Since multiple ThreadedBlackboard run concurrently, this multiplied the frequency of the SparkTimer callbacks by the number of active ThreadedBlackboards. The fix is to start this thread in an IShellDaemon rather than the ThreadedBlackboard. This avoids duplicate threads because the former is guarenteed to be instantiated only once, via dependency injection.	2024-10-03 17:46:50 +05:00
John Colvin	7f4468c812	refactor: Simplify FakeMDDaemon code that delays post Use Task.Delay/ContinueWith instead of subscribing to the SparkTimer tag.	2024-10-03 17:46:50 +05:00
John Colvin	6206cbc425	refactor: Use IBus instead of ThreadedBlackboard This should address some slowness/mouse lag in the front end (recently reported by Brandon at Immutable Cap)	2024-10-03 17:46:50 +05:00
John Colvin	21e50745e2	fix: Detect and log ccapi depth msg parse errors This fixes a crash that would happen if a depth item's price or quantity were an empty string (now we log an error and keep going).	2024-10-03 17:46:50 +05:00
John Colvin	990e21e387	fix: Actually store currency in shared mem This field has always been in the Security proto, though it until now it had been omitted from the shared memory representation (FixedSecurity).	2024-10-03 17:46:50 +05:00
Mohammad Arham	357be39db3	Merge branch 'development.new_exchanges' of bitbucket.org:bluetradesys/spark into development.new_exchanges	2024-09-24 07:34:09 -04:00
Mohammad Arham	68e04cb3f5	FIX: CHnaged databanto symbol source to asset	2024-09-24 11:31:49 +00:00
Mohammad Arham	b4a5d5c832	Added cpp to security common	2024-09-24 10:53:57 +00:00
Adeel Khan	242fd758f4	Add: Taifex Trades	2024-09-16 01:20:44 +05:00
Mohammad Arham	3c9755f0dd	ADD: Added delay to databento file reader	2024-09-12 15:10:34 +00:00
Mohammad Arham	61ed847d7d	Add: Replay speed added to mdreplayer	2024-09-11 06:50:37 +00:00
Mohammad Arham	fc9094fea8	Added: CBOE working on databento	2024-09-11 06:17:26 +00:00
Mohammad Arham	e36c0efc4b	Added: GLBX working on databento	2024-09-11 06:16:42 +00:00
Mohammad Arham	4158a14cff	Added: option to send timestamp in onbook IMDListener	2024-09-11 06:15:40 +00:00
Mohammad Arham	201a45cba0	taifex deo	2024-09-05 00:59:00 +00:00
Mohammad Arham	ab31b9b253	Add: Multiple files working taifex	2024-09-03 15:15:55 +00:00
Adeel Khan	780ddc79a7	Add: Taifex I083 Message	2024-09-03 20:06:48 +05:00
Mohammad Arham	d5a2adaed6	fix: removed errernous code	2024-09-03 13:09:54 +00:00
Mohammad Arham	008ab3f068	Merge branch 'development.new_exchanges' of bitbucket.org:bluetradesys/spark into development.new_exchanges	2024-09-02 16:31:14 -04:00
Mohammad Arham	7d926f588d	FIX: typo, changed date to day	2024-09-02 16:31:10 -04:00
Mohammad Arham	707cd07876	Add: CBOE L1 books and contracts working	2024-08-28 16:22:51 +00:00
Mohammad Arham	f7ecb9090c	add: code to start working on cboe	2024-08-27 12:33:23 +00:00
Mohammad Arham	a19e8a7315	change: changes to make taifex work	2024-08-27 07:45:59 -04:00
John Colvin	cf189ecf81	refactor: Start cboe and taifex md 'start_databento' Similar to how we start all ccapi implemented exchanges in "_start_cc" (in MarketDataGateway)	2024-08-26 19:13:59 +00:00
John Colvin	4405b7adcb	refactor: Rename cc_md_listeners -> venue_listeners They are general purpose, not specific to ccapi.	2024-08-26 19:09:57 +00:00
John Colvin	e51ee7b7b9	refactor: Wrap IDGenerator and IMarketDataListeners in shared_ptrs Our various market data providers (ccapi, emumba, replay, onixs) dereference pointers to common instances of these types concurrently on different threads. Although we've correctly managed memory ownership thus far, we really should be using smart pointers (this is why they were made).	2024-08-26 19:03:29 +00:00
John Colvin	dfaa74f426	build: Add databento dep via FetchContent Now we actually build libdatabento in the devcontainer along with our code (as we do with the vcpkg deps, though this is not part of vcpkg)	2024-08-26 17:13:51 +00:00
Mohammad Arham	1d2ce59009	add: added taifex entries	2024-08-22 23:13:05 +00:00
Adeel Khan	3602cc05bb	added: TaifexDataHandler, temp	2024-08-23 02:06:47 +05:00
Adeel Khan	ac88a81ac6	Security Extensions	2024-08-23 01:42:24 +05:00
Adeel Khan	01fa8e0c63	added: files for Taifex, Cboe	2024-08-23 00:57:40 +05:00
John Colvin	59547e0f37	fix build: Lock fmt version to 10.2.1 in vcpkg This fixes the front-end (Windows) libspark build	2024-08-21 13:45:43 -04:00
John Colvin	3ce100a9cf	build: Add databento to sparkd * Install databento deps in docker - libzstd-dev * Add databento cmake module to externals	2024-08-21 13:49:53 +00:00
John Colvin	6513fb0cd5	cleanup: Remove unused include 'date.h' from MDSource This was an external header, and may interfere with a dependency in the databento lib (which we are about to add).	2024-08-21 13:49:24 +00:00
Taha Afzal	fd0fb1f760	Adding incremental book updates to IMarketDataListener	2024-08-12 22:49:27 -04:00
				`@@ -1 +0,0 @@`
				<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>