fix use dep injection in bts.datastreamwebapp.server

fix Add spark service modules designed for DependencyInjection
Fix mismatched types in PubsubHeartbeatStream schema
2024-07-11 11:06:29 -04:00 · 2024-07-11 11:05:23 -04:00 · 2024-07-11 10:59:41 -04:00 · 2024-07-11 10:43:24 -04:00 · 2024-07-11 10:38:30 -04:00 · 2024-07-11 10:37:29 -04:00
520 changed files with 32543 additions and 21538 deletions
@@ -66,6 +66,7 @@ RUN \

 # Dotnet
 RUN apt-get update && apt-get install -y dotnet7
+RUN apt-get update && apt-get install -y dotnet8

 # Node
 ARG node_major=18
@@ -19,13 +19,23 @@
    "--shm-size",
    "2gb",
    "-p",
+    "5000:5000",
+    "-p",
+    "5001:5001",
+    "-p",
+    "5173:5173",
+    "-p",
+    "5174:5174",
+    "-p",
    "8000:8000",
    "-p",
    "8001:8001",
    "-p",
    "8002:8002",
    "-p",
-    "8003:8003"
+    "8003:8003",
+    "-p",
+    "12345:12345",
  ],
  "mounts": [
    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
@@ -207,17 +207,23 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private

+# local symbols
+dotnet_naming_symbols.local_symbols.applicable_kinds = local
+
+# parameter symbols
+dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
+
 # Naming styles

-dotnet_naming_style.snake_case.required_prefix =
-dotnet_naming_style.snake_case.required_suffix =
-dotnet_naming_style.snake_case.word_separator = _
-dotnet_naming_style.snake_case.capitalization = all_lower
+dotnet_naming_style.pascal_case.required_prefix =
+dotnet_naming_style.pascal_case.required_suffix =
+dotnet_naming_style.pascal_case.word_separator =
+dotnet_naming_style.pascal_case.capitalization = pascal_case

-dotnet_naming_style.private_snake_case.required_prefix = _
-dotnet_naming_style.private_snake_case.required_suffix =
-dotnet_naming_style.private_snake_case.word_separator = _
-dotnet_naming_style.private_snake_case.capitalization = all_lower
+dotnet_naming_style.camel_case.required_prefix =
+dotnet_naming_style.camel_case.required_suffix =
+dotnet_naming_style.camel_case.word_separator =
+dotnet_naming_style.camel_case.capitalization = camel_case

 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -229,6 +235,10 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case

+dotnet_naming_style.local.capitalization = camel_case
+
+dotnet_naming_style.parameter.capitalization = camel_case
+
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -238,14 +248,18 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion

-dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
-dotnet_naming_rule.public_members_snake_case.style = snake_case
-dotnet_naming_rule.public_members_snake_case.severity = suggestion
+dotnet_naming_rule.public_members.symbols = public_symbols
+dotnet_naming_rule.public_members.style = pascal_case
+dotnet_naming_rule.public_members.severity = suggestion

-dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
-dotnet_naming_rule.private_members_snake_case.style = private_snake_case
-dotnet_naming_rule.private_members_snake_case.severity = suggestion
+dotnet_naming_rule.private_members.symbols = private_symbols
+dotnet_naming_rule.private_members.style = camel_case
+dotnet_naming_rule.private_members.severity = suggestion

-dotnet_naming_rule.parameters.symbols = parameters
-dotnet_naming_rule.parameters.style = snake_case
+dotnet_naming_rule.variables_are_camel_case.severity = suggestion
+dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
+dotnet_naming_rule.variables_are_camel_case.style = local
+
+dotnet_naming_rule.parameters.symbols = parameter_symbols
+dotnet_naming_rule.parameters.style = parameter
 dotnet_naming_rule.parameters.severity = suggestion
@@ -0,0 +1,3 @@
+*.cmd text=auto eol=crlf
+*.cs text=auto eol=crlf
+*.sh text=auto eol=lf
@@ -8,7 +8,9 @@ cmake-build-debug/
 .idea/
 .dir-locals.el
 *.pyc
-**.vscode
+**.vscode/*
+!.vscode/launch.json
+!.vscode/tasks.json
 .vs
 obj/
 *.csproj.user
@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = git@bitbucket.org:bluetradesys/externals.git
+	url = https://bitbucket.org/bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git
@@ -0,0 +1,213 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": ".NET Core Attach",
+      "type": "coreclr",
+      "request": "attach"
+    },
+    {
+      "name": "Bts.DataStreamWebApp.Server",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream WebApp (Debug)",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    {
+      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
+      ],
+      "env": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    {
+      "name": "DataStreamPublisher",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "DataStreamPublisher",
+      },
+      "windows": {
+        "program": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DataStreamPublisher (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "DataStreamPublisher",
+      },
+      "windows": {
+        "program": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
+      ],
+      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "DataStreamQueryAnalyzer",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
+      "program": "",
+      "windows": {
+        "program": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
+        ],
+        "cwd": "build/Debug/bin/net7.0-windows",
+        "env": {
+          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
+        },
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal",
+    },
+    {
+      "name": "PubSubServer",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build DataStream (Debug)",
+      "program": "",
+      "linux": {
+        "program": "PubSubServer",
+      },
+      "windows": {
+        "program": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "PubSubServer (Skip Build)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "linux": {
+        "program": "PubSubServer",
+      },
+      "windows": {
+        "program": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
+      "cwd": "build/Debug/bin/net7.0",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "",
+      "windows": {
+        "preLaunchTask": "Build Spark (Debug)",
+        "program": "Spark.exe",
+        "cwd": "build/Debug/bin/net7.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Spark (Parameterized)",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build Spark (Debug)",
+      "program": "",
+      "windows": {
+        "program": "Spark.exe",
+        "args": [
+          "--sparkd",
+          "${input:sparkConfigServer}"
+        ],
+        "cwd": "build/Debug/bin/net7.0-windows",
+      },
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Sparkd",
+      "type": "coreclr",
+      "request": "launch",
+      "preLaunchTask": "Build Sparkd (Debug | Clang)",
+      "program": "",
+      "linux": {
+        "program": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "cwd": "build/sparkd-debug-clang/server",
+      "stopAtEntry": false,
+      "console": "integratedTerminal"
+    }
+  ],
+  "inputs": [
+    {
+      "id": "dataStreamPublisherConfig",
+      "type": "pickString",
+      "description": "Publisher Configuration",
+      "options": [
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
+      ],
+      "default": "MarketDataPublisher",
+    },
+    {
+      "id": "pubSubServerConfig",
+      "type": "pickString",
+      "description": "PubSub Configuration",
+      "options": [
+        "PubSubServer",
+        "SimpleEchoClient",
+      ],
+      "default": "PubSubServer",
+    },
+    {
+      "id": "sparkConfigServer",
+      "type": "promptString",
+      "description": "Config Server URL",
+      "default": "http://localhost:8000/config",
+    },
+  ]
+}
@@ -0,0 +1,935 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "Build Api Server",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_api_server.sh",
+      },
+      "windows": {
+        "command": "tools/build_api_server.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Api Server (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_api_server.sh",
+      },
+      "windows": {
+        "command": "tools/build_api_server.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Api Server (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_api_server.cmd",
+      },
+      "linux": {
+        "command": "tools/build_api_server.sh",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStreamQueryAnalyzer (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_data_stream_query_analyzer.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build DataStream WebApp (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/build_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/build_data_stream_web_app.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark.cmd",
+      },
+      "args": [
+        "--configuration",
+        "${input:buildConfiguration}",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark (Debug)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Debug",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Spark (Release)",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/build_spark.cmd",
+      },
+      "args": [
+        "--configuration",
+        "Release",
+      ],
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--preset",
+          "${input:sparkdCmakePreset}",
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Debug)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Debug",
+          "--compiler",
+          "${input:sparkdCompiler}"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Debug | Gcc)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Debug",
+          "--compiler",
+          "gcc"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Debug | Clang)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Debug",
+          "--compiler",
+          "clang"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Release)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Release",
+          "--compiler",
+          "${input:sparkdCompiler}"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Release | Gcc)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Release",
+          "--compiler",
+          "gcc"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Build Sparkd (Release | Clang)",
+      "group": "build",
+      "type": "shell",
+      "command": "",
+      "linux": {
+        "command": "tools/build_sparkd.sh",
+        "args": [
+          "--build-type",
+          "Release",
+          "--compiler",
+          "clang"
+        ],
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Publish DataStreamWebApp",
+      "group": "build",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/publish_data_stream_web_app.sh",
+      },
+      "windows": {
+        "command": "tools/publish_data_stream_web_app.cmd",
+      },
+      "icon": {
+        "id": "terminal",
+        "color": "terminal.ansiCyan"
+      },
+    },
+    {
+      "label": "Edit DataStream Config",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "editor",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/conf/spark.conf"
+        ],
+      },
+      "windows": {
+        "command": "vim",
+        "args": [
+          "${env:APPDATA}/bts/DataStream/conf/spark.conf"
+        ],
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+    {
+      "label": "Edit Spark Config",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "vim",
+        "args": [
+          "${env:APPDATA}/bts/conf/spark.conf"
+        ],
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "dedicated",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+    {
+      "label": "Run DataStreamQueryAnalyzer",
+      "dependsOn": "Build DataStreamQueryAnalyzer",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamQueryAnalyzer (Debug)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamQueryAnalyzer (Release)",
+      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "DataStreamQueryAnalyzer.exe",
+        "args": [
+          "--config",
+          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
+        ],
+        "options": {
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamPublisher",
+      "dependsOn": "Build DataStream",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "DataStreamPublisher",
+      },
+      "windows": {
+        "command": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamPublisher (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "DataStreamPublisher",
+      },
+      "windows": {
+        "command": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run DataStreamPublisher (Release)",
+      "dependsOn": "Build DataStream (Release)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "DataStreamPublisher",
+      },
+      "windows": {
+        "command": "DataStreamPublisher.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer",
+      "dependsOn": "Build DataStream",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Debug)",
+      "dependsOn": "Build DataStream (Debug)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Debug | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Release)",
+      "dependsOn": "Build DataStream (Release)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run PubSubServer (Release | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Spark",
+      "dependsOn": "Build Spark",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark (Debug)",
+      "dependsOn": "Build Spark (Debug)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark (Release)",
+      "dependsOn": "Build Spark (Release)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "Spark.exe",
+        "options": {
+          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
+        },
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      }
+    },
+    {
+      "label": "Run Spark Config Query",
+      "dependsOn": "Build Spark",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "Spark.Console",
+      },
+      "windows": {
+        "command": "Spark.Console.exe",
+      },
+      "args": [
+        "config",
+        "--query",
+        "${input:sparkConfigQuery}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Sparkd",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "run"
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "Run Sparkd Config Query",
+      "dependsOn": "Build Sparkd",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "sparkd",
+      },
+      "args": [
+        "config",
+        "--query",
+        "${input:sparkdConfigQuery}",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
+    {
+      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "DataStreamQueryAnalyzer.exe",
+          "'Starting DataStreamQueryAnalyzer'"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+    {
+      "label": "View PubSubServer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/logging/show_last_run.sh",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
+          "Starting PubSubServer"
+        ]
+      },
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "PubSubServer.exe",
+          "Starting PubSubServer"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
+  ],
+  "inputs": [
+    {
+      "id": "buildConfiguration",
+      "type": "pickString",
+      "description": "Configuration",
+      "options": [
+        "Debug",
+        "Release",
+      ],
+      "default": "Debug",
+    },
+    {
+      "id": "sparkdCmakePreset",
+      "type": "pickString",
+      "description": "Configuration",
+      "options": [
+        "sparkd-debug-clang",
+        "sparkd-debug-gcc",
+        "sparkd-release-clang",
+        "sparkd-release-gcc",
+      ],
+      "default": "sparkd-debug-clang",
+    },
+    {
+      "id": "sparkdCompiler",
+      "type": "pickString",
+      "description": "Configuration",
+      "options": [
+        "clang",
+        "gcc",
+      ],
+      "default": "sparkd-debug-clang",
+    },
+    {
+      "id": "dataStreamPublisherConfig",
+      "type": "pickString",
+      "description": "Publisher Configuration",
+      "options": [
+        "MarketDataPublisher",
+        "SimpleEchoClient",
+        "SimpleTestDataPublisher",
+        "TestDataPublisher",
+      ],
+      "default": "MarketDataPublisher",
+    },
+    {
+      "id": "pubSubServerConfig",
+      "type": "pickString",
+      "description": "PubSub Configuration",
+      "options": [
+        "PubSubServer",
+        "SimpleEchoClient",
+      ],
+      "default": "PubSubServer",
+    },
+    {
+      "id": "sparkConfigQuery",
+      "type": "promptString",
+      "description": "Config Type(s)"
+    },
+    {
+      "id": "sparkdConfigQuery",
+      "type": "promptString",
+      "description": "Config Type(s)"
+    }
+  ]
+}
@@ -14,12 +14,17 @@ set(
  CACHE
  STRING
  "External dependencies root folder (should contain folders: lib, bin, include")
+
+# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
+# can run simultaneously without fighting over the same shm files (e.g., sparkd,
+# api server, etc.).
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
 set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
 set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
-set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
+set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
 if (WIN32)
  set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
  set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
@@ -31,13 +36,13 @@ endif()
 include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
 link_directories(${SPARK_LIB})

-if(BUILD_API_SERVER)
-  add_compile_definitions(SPARK_ENABLE_API_SERVER)
+add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
+add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
+if(DEFINED APP_DATA_DIR_OVERRIDE)
+  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
 endif()
-
+add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
 if(WIN32)
-  add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
-  add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
  # subdirs
  add_subdirectory(src/lib)
 else()
@@ -47,13 +52,7 @@ else()
  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")

-  if(BUILD_API_SERVER)
-    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
-    add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
-  elseif(BUILD_SPARKD)
-    add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
-    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
-
+  if(BUILD_SPARKD)
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
    add_compile_definitions(SPARK_ENABLE_CCAPI)
@@ -15,11 +15,13 @@
        "rhs": "Windows"
      },
      "generator": "Visual Studio 17 2022",
-      "binaryDir": "${sourceDir}/build",
+      "binaryDir": "${sourceDir}/build/${presetName}",
      "environment": {
        "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
      },
      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
        "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
@@ -29,8 +31,12 @@
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "spark"
      }
    },

@@ -46,6 +52,8 @@
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
        "platform": "",
+        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
          "value": "ON"
@@ -62,7 +70,8 @@
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "ON"
-        }
+        },
+        "SHARED_MEMORY_PREFIX": "sparkd"
      }
    },

@@ -95,98 +104,72 @@
    {
      "name": "sparkd-debug-clang",
      "inherits": "sparkd-debug",
-      "displayName": "Debug (clang)",
+      "displayName": "Sparkd (Debug | Clang)",
      "description": "Debug build compiled w/ clang",
-      "binaryDir": "${sourceDir}/build/clang/debug",
      "cacheVariables": {
        "CMAKE_C_COMPILER": "clang",
        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
      }
    },

    {
      "name": "sparkd-debug-gcc",
      "inherits": "sparkd-debug",
-      "displayName": "Debug (gcc)",
+      "displayName": "Sparkd (Debug | Gcc)",
      "description": "Debug build compiled w/ gcc",
-      "binaryDir": "${sourceDir}/build/gcc/debug",
      "cacheVariables": {
        "CMAKE_C_COMPILER": "gcc",
        "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
      }
    },

    {
      "name": "sparkd-release-clang",
      "inherits": "sparkd-release",
-      "displayName": "Release (clang)",
+      "displayName": "Sparkd (Release | Clang)",
      "description": "Release build compiled w/ clang",
-      "binaryDir": "${sourceDir}/build/clang/release",
      "cacheVariables": {
        "CMAKE_C_COMPILER": "clang",
        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
      }
    },

    {
      "name": "sparkd-release-gcc",
      "inherits": "sparkd-release",
-      "displayName": "Release (gcc)",
+      "displayName": "Sparkd (Release | Gcc)",
      "description": "Release build compiled w/ gcc",
-      "binaryDir": "${sourceDir}/build/gcc/release",
      "cacheVariables": {
        "CMAKE_C_COMPILER": "gcc",
        "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
-      }
-    },
-
-    {
-      "name": "spark-api-server-base",
-      "hidden": true,
-      "binaryDir": "${sourceDir}/build/${presetName}",
-      "cacheVariables": {
-        "BUILD_API_SERVER": {
-          "type": "bool",
-          "value": "ON"
-        },
-        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
      }
    },

    {
      "name": "spark-api-server-windows",
      "inherits": "spark",
-      "displayName": "Spark Api Server",
+      "displayName": "Spark Api Server (Windows)",
      "cacheVariables": {
-        "BUILD_API_SERVER": {
-          "type": "bool",
-          "value": "ON"
-        }
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
+        "SHARED_MEMORY_PREFIX": "spark_api_server"
      }
    },

    {
      "name": "spark-api-server-linux",
      "inherits": "sparkd",
-      "displayName": "Spark Api Server",
+      "displayName": "Spark Api Server (Linux)",
      "generator": "Ninja Multi-Config",
      "cacheVariables": {
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "OFF"
        },
-        "BUILD_API_SERVER": {
-          "type": "bool",
-          "value": "ON"
-        },
        "BUILD_SPARKD": {
          "type": "BOOL",
          "value": "OFF"
@@ -197,7 +180,11 @@
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        }
      }
    },

@@ -220,13 +207,63 @@
    },

    {
-      "name": "data-stream-suite",
+      "name": "data-stream-suite-windows",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
      "inherits": "spark",
-      "displayName": "Data Stream Suite",
+      "displayName": "Data Stream Suite (Windows)",
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
+        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows;${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "SHARED_MEMORY_PREFIX": "data_stream"
+      }
+    },
+
+    {
+      "name": "data-stream-suite-linux",
+      "displayName": "Data Stream Suite (Linux)",
+      "description": "Data Stream utils ported over from Edge",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Linux"
+      },
+      "inherits": "sparkd",
+      "generator": "Ninja Multi-Config",
+      "cacheVariables": {
+        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
+        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
+        "BUILD_ADMIN_WEB_APP": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SDK": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "BUILD_SPARKD": {
+          "type": "BOOL",
+          "value": "OFF"
+        },
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
+        "EXPORT_MD_CONSUMER": {
+          "type": "BOOL",
+          "value": "ON"
+        },
+        "SHARED_MEMORY_PREFIX": "data_stream"
      }
    }
  ],
@@ -244,7 +281,7 @@

    {
      "name": "spark-debug",
-      "displayName": "Debug",
+      "displayName": "Spark (Debug)",
      "inherits": "spark",
      "configurePreset": "spark",
      "configuration": "Debug"
@@ -252,7 +289,7 @@

    {
      "name": "spark-release",
-      "displayName": "Release",
+      "displayName": "Spark (Release)",
      "inherits": "spark",
      "configurePreset": "spark",
      "configuration": "Release"
@@ -280,28 +317,28 @@
    {
      "name": "sparkd-debug-clang",
      "configurePreset": "sparkd-debug-clang",
-      "displayName": "Debug (clang)",
+      "displayName": "Sparkd (Debug | Clang)",
      "inherits": "sparkd"
    },

    {
      "name": "sparkd-debug-gcc",
      "configurePreset": "sparkd-debug-gcc",
-      "displayName": "Debug (gcc)",
+      "displayName": "Sparkd (Debug | Gcc)",
      "inherits": "sparkd"
    },

    {
      "name": "sparkd-release-clang",
      "configurePreset": "sparkd-release-clang",
-      "displayName": "Release (clang)",
+      "displayName": "Sparkd (Release | Clang)",
      "inherits": "sparkd"
    },

    {
      "name": "sparkd-release-gcc",
      "configurePreset": "sparkd-release-gcc",
-      "displayName": "Release (gcc)",
+      "displayName": "Sparkd (Release | Gcc)",
      "inherits": "sparkd"
    },

@@ -318,14 +355,14 @@

    {
      "name": "spark-api-server-windows-debug",
-      "displayName": "Spark Api Server (Debug)",
+      "displayName": "Spark Api Server (Windows | Debug)",
      "inherits": "spark-api-server-windows",
      "configuration": "Debug"
    },

    {
      "name": "spark-api-server-windows-release",
-      "displayName": "Spark Api Server (Debug)",
+      "displayName": "Spark Api Server (Windows | Release)",
      "inherits": "spark-api-server-windows",
      "configuration": "Release"
    },
@@ -344,14 +381,14 @@
    {
      "name": "spark-api-server-linux-debug",
      "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Debug)",
+      "displayName": "Spark Api Server (Linux | Debug)",
      "configuration": "Debug"
    },

    {
      "name": "spark-api-server-linux-release",
      "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Release)",
+      "displayName": "Spark Api Server (Linux | Release)",
      "configuration": "Release"
    },

@@ -371,16 +408,30 @@
    },

    {
-      "name": "data-stream-suite-debug",
-      "displayName": "DataStreamSuite (Debug)",
-      "configurePreset": "data-stream-suite",
+      "name": "data-stream-suite-windows-debug",
+      "displayName": "Data Stream Suite (Windows | Debug)",
+      "configurePreset": "data-stream-suite-windows",
      "configuration": "Debug"
    },

    {
-      "name": "data-stream-suite-release",
-      "displayName": "DataStreamSuite (Release)",
-      "configurePreset": "data-stream-suite",
+      "name": "data-stream-suite-windows-release",
+      "displayName": "Data Stream Suite (Windows | Release)",
+      "configurePreset": "data-stream-suite-windows",
+      "configuration": "Release"
+    },
+
+    {
+      "name": "data-stream-suite-linux-debug",
+      "displayName": "Data Stream Suite (Linux | Debug)",
+      "configurePreset": "data-stream-suite-linux",
+      "configuration": "Debug"
+    },
+
+    {
+      "name": "data-stream-suite-linux-release",
+      "displayName": "Data Stream Suite (Linux | Release)",
+      "configurePreset": "data-stream-suite-linux",
      "configuration": "Release"
    }
  ]
@@ -8,7 +8,6 @@
   public int Integer;
   public double Real;
   public string Identifier;
-   public DateTime DateTime; 
   public ImmutableList<IArithmeticExpression> Arguments;
   public ImmutableList<Assignment> Assignments;
   public IArithmeticExpression Argument;
@@ -1,41 +1,22 @@
-using mbts.spark;
+using System.Diagnostics;

-using System;
-using System.Runtime;
+namespace BCG.Utilities.General;

-namespace BCG.Utilities.General
-{
-	/// <summary>
-	/// Summary description for Crono.
-	/// </summary>
-	public class Crono
-	{
-      public Crono()
-      {
-         myTimer = new HighPerformanceTimer();
-      }
+/// <summary>
+/// Summary description for Crono.
+/// </summary>
+public class Crono {
+   readonly Stopwatch myTimer = new();

-      public void start() 
-      {myTimer.start();}
+   public void start() =>
+      myTimer.Start();

-      public void stop()
-      {myTimer.stop();}
+   public void stop() =>
+      myTimer.Stop();

-      public TimeSpan time
-      {
-         get 
-         {
-            return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
-         }
-      }
-
-      public double miliseconds
-      {
-         get
-         {return myTimer.totalMilliseconds;}
-      }
-
-      private HighPerformanceTimer myTimer;
-	}
+   public TimeSpan time =>
+      myTimer.Elapsed;

+   public double miliseconds =>
+      myTimer.Elapsed.TotalMilliseconds;
 }
@@ -1,192 +1,192 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Collections.Concurrent;
-using System.Reflection;
-using Microsoft.Win32;
-
-namespace BCG
-{
-   public enum RegistryRoot { HKCU, HKLM }
-
-   [Serializable]
-   public class BlueRegistry
-   {
-      public const string BlueSubKey = @"Software\Blue Capital Group\";
-      public static readonly string ApplicationPrefix;
-      protected string myFullPath;
-
-      public BlueRegistry(string subPath)
-      {
-         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
-      }
-
-      public BlueRegistry(BlueRegistry parent, string subPath)
-      {
-         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
-      }
-
-      static BlueRegistry()
-      {
-         string app = "Applications";
-
-         Assembly ass = Assembly.GetEntryAssembly();
-         if (ass != null)
-         {
-            app = ass.GetName().Name;
-
-            object[] attrs = ass.GetCustomAttributes(
-               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
-
-            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
-            {
-               app = ((AssemblyTitleAttribute)attrs[0]).Title;
-            }
-         }
-
-         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
-      }
-
-      public object getValue(string name)
-      {
-         return getValue(RegistryRoot.HKCU, name);
-      }
-
-      public object getValue(string name, object defaultValue)
-      {
-         return getValue(RegistryRoot.HKCU, name, defaultValue);
-      }
-
-      public void setValue(string name, object value)
-      {
-         setValue(RegistryRoot.HKCU, name, value);
-      }
-
-      public void deleteValue(string name)
-      {
-         deleteValue(RegistryRoot.HKCU, name);
-      }
-
-      private RegistryKey myGetRegistryKey(RegistryRoot root)
-      {
-         if (root == RegistryRoot.HKCU)
-            return Registry.CurrentUser.CreateSubKey(myFullPath);
-         else
-            return Registry.LocalMachine.CreateSubKey(myFullPath);
-      }
-
-      #region Generic Methods
-      internal virtual void setValue(RegistryRoot root, string name, object value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.SetValue(name, value);
-         }
-      }
-
-      internal virtual object getValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name);
-         }
-      }
-
-      internal virtual void deleteValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.DeleteValue(name);
-         }
-      }
-
-      internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name, defaultValue);
-         }
-      }
-      #endregion
-
-      #region Double
-      internal virtual void setDouble(RegistryRoot root, string name, double value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            DoubleRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual double getDouble(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return DoubleRegistryHandler.getValue(rk, name);
-         }
-      }
-      #endregion
-
-      #region Enum
-      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            EnumRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return EnumRegistryHandler.getValue(rk, name, enumType);
-         }
-      }
-      #endregion
-
-      #region Array
-      internal virtual void setArray(RegistryRoot root, string name, Array array)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            ArrayRegistryHandler.setValue(rk, name, array);
-         }
-      }
-
-      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return ArrayRegistryHandler.getValue(rk, name, elementType);
-         }
-      }
-      #endregion
-
-      #region SortedList
-      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.setValue(rk, name, slist);
-         }
-      }
-
-      internal virtual SortedList getSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return SortedListRegistryHandler.getValue(rk, name);
-         }
-      }
-
-      internal virtual void deleteSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.deleteValue(rk, name);
-         }
-      }
-      #endregion
-   }
-}
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Reflection;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public enum RegistryRoot { HKCU, HKLM }
+
+   [Serializable]
+   public class BlueRegistry
+   {
+      public const string BlueSubKey = @"Software\Blue Capital Group\";
+      public static readonly string ApplicationPrefix;
+      protected string myFullPath;
+
+      public BlueRegistry(string subPath)
+      {
+         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
+      }
+
+      public BlueRegistry(BlueRegistry parent, string subPath)
+      {
+         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
+      }
+
+      static BlueRegistry()
+      {
+         string app = "Applications";
+
+         Assembly ass = Assembly.GetEntryAssembly();
+         if (ass != null)
+         {
+            app = ass.GetName().Name;
+
+            object[] attrs = ass.GetCustomAttributes(
+               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
+
+            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
+            {
+               app = ((AssemblyTitleAttribute)attrs[0]).Title;
+            }
+         }
+
+         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
+      }
+
+      public object getValue(string name)
+      {
+         return getValue(RegistryRoot.HKCU, name);
+      }
+
+      public object getValue(string name, object defaultValue)
+      {
+         return getValue(RegistryRoot.HKCU, name, defaultValue);
+      }
+
+      public void setValue(string name, object value)
+      {
+         setValue(RegistryRoot.HKCU, name, value);
+      }
+
+      public void deleteValue(string name)
+      {
+         deleteValue(RegistryRoot.HKCU, name);
+      }
+
+      private RegistryKey myGetRegistryKey(RegistryRoot root)
+      {
+         if (root == RegistryRoot.HKCU)
+            return Registry.CurrentUser.CreateSubKey(myFullPath);
+         else
+            return Registry.LocalMachine.CreateSubKey(myFullPath);
+      }
+
+      #region Generic Methods
+      public virtual void setValue(RegistryRoot root, string name, object value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.SetValue(name, value);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name);
+         }
+      }
+
+      internal virtual void deleteValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.DeleteValue(name);
+         }
+      }
+
+      public virtual object getValue(RegistryRoot root, string name, object defaultValue)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name, defaultValue);
+         }
+      }
+      #endregion
+
+      #region Double
+      internal virtual void setDouble(RegistryRoot root, string name, double value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            DoubleRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual double getDouble(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return DoubleRegistryHandler.getValue(rk, name);
+         }
+      }
+      #endregion
+
+      #region Enum
+      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            EnumRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return EnumRegistryHandler.getValue(rk, name, enumType);
+         }
+      }
+      #endregion
+
+      #region Array
+      internal virtual void setArray(RegistryRoot root, string name, Array array)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            ArrayRegistryHandler.setValue(rk, name, array);
+         }
+      }
+
+      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return ArrayRegistryHandler.getValue(rk, name, elementType);
+         }
+      }
+      #endregion
+
+      #region SortedList
+      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.setValue(rk, name, slist);
+         }
+      }
+
+      internal virtual SortedList getSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return SortedListRegistryHandler.getValue(rk, name);
+         }
+      }
+
+      internal virtual void deleteSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.deleteValue(rk, name);
+         }
+      }
+      #endregion
+   }
+}
@@ -1,10 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0-windows</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
-    <UseWindowsForms>true</UseWindowsForms>
  </PropertyGroup>

 </Project>
@@ -1,4 +1,6 @@
 using System;
+using System.Runtime.Versioning;
+
 using Microsoft.Win32;

 namespace BCG
@@ -10,6 +12,7 @@ namespace BCG
         rk.SetValue (name, value.ToString ());
      }

+      [SupportedOSPlatform("windows")]
      public static double getValue (RegistryKey rk, string name)
      {
         object v = rk.GetValue (name);
@@ -0,0 +1,20 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.3.1' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
+  <PropertyGroup>
+    <StartupCommand>npm run dev</StartupCommand>
+    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
+    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
+    <!-- Allows the build (or compile) script located on package.json to run on Build -->
+    <ShouldRunBuildScript>true</ShouldRunBuildScript>
+    <!-- Folder where production build objects will be placed -->
+    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
+  </PropertyGroup>
+</Project>
@@ -0,0 +1,8 @@
+# React + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/ico" href="/logo.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>BTS Cloud Streams</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>
@@ -0,0 +1,38 @@
+{
+  "name": "bts.datastreamwebapp.client",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@angular/material": "^18.0.4",
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@fontsource/noto-sans": "^5.0.22",
+    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
+    "@mui/icons-material": "^5.15.20",
+    "@mui/material": "^5.15.20",
+    "@mui/x-data-grid": "^7.7.1",
+    "bulma": "^1.0.1",
+    "react": "^18.3.1",
+    "react-data-table-component": "^7.6.2",
+    "react-dom": "^18.3.1",
+    "react-use-websocket": "^4.8.1",
+    "rpc-websockets": "^7.11.2"
+  },
+  "devDependencies": {
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^8.57.0",
+    "eslint-plugin-react": "^7.34.2",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-react-refresh": "^0.4.7",
+    "vite": "^5.2.13"
+  }
+}
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
@@ -0,0 +1,28 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+h1#main-header {
+  padding: 4px 8px 4px 8px;
+  text-align: center;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}
@@ -0,0 +1,24 @@
+import { createTheme, ThemeProvider, CssBaseline } from '@mui/material';
+import useMediaQuery from '@mui/material/useMediaQuery';
+import QueryControl from './QueryControl';
+import getTheme from './Theme';
+import '@fontsource/noto-sans';
+import './App.css';
+
+const App = () => {
+  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
+  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
+
+  return (
+    <ThemeProvider theme={theme}>
+      <CssBaseline />
+      <h1 id='main-header' style={{
+        color: theme.palette.text.primary,
+        fontFamily: 'Noto Sans',
+      }}>BTS Cloud Streams</h1>
+      <QueryControl websocketUrl="ws://localhost:5000/ws" theme={theme} />
+    </ThemeProvider>
+  );
+};
+
+export default App;
@@ -0,0 +1,32 @@
+@use "@material/checkbox";
+@use "@material/icon-button";
+@use "@material/data-table/data-table";
+
+th {
+  font-size: large;  
+  padding: 15px;
+}
+
+td {
+  padding: 15px;
+}
+
+td:hover {
+  background-color: cadetblue;
+}
+
+td#cell-decimal {
+  text-align: right;
+}
+
+td#cell-integer {
+  text-align: right;
+}
+
+td#cell-string {
+  text-align: center;
+}
+
+td#cell-date-time {
+  text-align: center;
+}
@@ -0,0 +1,52 @@
+import PropTypes from 'prop-types';
+import { DataGrid } from '@mui/x-data-grid';
+
+function getGridColDef(col) {
+  return {
+    field: col.name,
+    headerName: col.name,
+  };
+}
+
+export default function DataTable({ columns, rows }) {
+  return (
+    <DataGrid
+      columns={columns.map(getGridColDef)}
+      rows={rows}
+      sx={{
+        borderColor: 'primary.main',
+        backgroundColor: 'primary.dark',
+        color: 'text.primary',
+        '& .MuiDataGrid-columnHeader': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-filler': {
+          color: 'primary.main',
+          fontWeight: 700,
+          backgroundColor: 'primary.dark'
+        },
+        '& .MuiDataGrid-cell': {
+          color: 'text.primary',
+        },
+        '& .MuiDataGrid-cell:hover': {
+          color: 'primary.main',
+        },
+        '& .MuiDataGrid-row': {
+          '--DataGrid-rowBorderColor': 'secondary.dark'
+        }
+      }}
+    />
+  );
+}
+
+DataTable.propTypes = {
+  columns: PropTypes.arrayOf(
+    PropTypes.shape({
+      name: PropTypes.string.isRequired,
+      type: PropTypes.string.isRequired,
+    })
+  ).isRequired,
+  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
+};
@@ -0,0 +1,100 @@
+import PropTypes from 'prop-types';
+import DataTable from './DataTable';
+import QueryInputForm from './QueryInputForm';
+import Container from '@mui/material/Container';
+import React from 'react';
+
+var rpcReqIdCounter = 1;
+
+function subscribeReq(query) {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "subscribe",
+    params: {
+      subscription_id: "1",
+      query: query
+    }
+  };
+}
+
+function unsubscribeReq() {
+  return {
+    jsonrpc: "2.0",
+    id: rpcReqIdCounter++,
+    method: "unsubscribe",
+    params: {
+      subscription_id: "1",
+    }
+  };
+}
+
+class QueryControl extends React.Component {
+  constructor({ websocketUrl, theme }) {
+    super(websocketUrl);
+    console.log('Connecting to WebSocket:', websocketUrl);
+    this.state = {
+      rows: null,
+      ws: new WebSocket(websocketUrl),
+      columns: null,
+      isRunning: false,
+      theme: theme,
+    };
+    this.state.ws.onopen = () => {
+      console.info('WebSocket open');
+    };
+    this.state.ws.onclose = () => {
+      console.info('WebSocket closed');
+    };
+    this.state.ws.onmessage = e => {
+      console.info(`Received message: ${e.data}`);
+      const msg = JSON.parse(e.data);
+      if (msg.id != null) {
+        let newState = { ...this.state };
+        newState.isRunning = msg.result.status == 'Subscribed';
+        this.setState(newState);
+      } else {
+        let newState = { ...this.state };
+        newState.rows = msg.params.all;
+        console.info(`Received rows: ${newState.rows}`);
+        newState.columns = msg.params.columns;
+        this.setState(newState);
+      }
+    };
+
+    this.start = (query) => {
+      this.state.ws.send(JSON.stringify(subscribeReq(query)));
+    };
+
+    this.stop = () => {
+      this.state.ws.send(JSON.stringify(unsubscribeReq()));
+    };
+  }
+
+  componentWillUnmount() {
+    if (this.state.ws) {
+      console.warn('Closing websocket because component is unmounting');
+      this.state.ws.close();
+    }
+  }
+
+  render() {
+    console.info('Rendering');
+    return (
+      <Container>
+        <QueryInputForm isRunning={this.state.isRunning} onStart={this.start} onStop={this.stop} />
+        {
+          this.state.rows == null ? <p style={{ color: this.state.theme.palette.warning.main }}>No data received</p> :
+            <DataTable columns={this.state.columns} rows={this.state.rows} />
+        }
+      </Container>
+    );
+  }
+}
+
+QueryControl.propTypes = {
+  websocketUrl: PropTypes.string.isRequired,
+  theme: PropTypes.object.isRequired,
+};
+
+export default QueryControl;
@@ -0,0 +1,93 @@
+import { useRef } from 'react';
+import Box from '@mui/material/Box';
+import Button from '@mui/material/Button';
+import Grid from '@mui/material/Grid';
+import Paper from '@mui/material/Paper';
+import PlayArrowIcon from '@mui/icons-material/PlayArrow';
+import PropTypes from 'prop-types';
+import StopIcon from '@mui/icons-material/Stop';
+import TextField from '@mui/material/TextField';
+import Tooltip from '@mui/material/Tooltip';
+
+const QueryInputForm = ({ isRunning, onStart, onStop }) => {
+  let textInput = useRef(null);
+
+  const onClearClick = () => {
+    textInput.current.value = '';
+  };
+
+  const onRunClick = () => {
+    onStart(textInput.current.value);
+  };
+
+  const onStopClick = () => {
+    onStop();
+  };
+
+  const onKeyDown = (e) => {
+    // Detect keyboard shortcuts.
+    //   Ctrl+Enter : Run
+    if (e.ctrlKey && e.key === 'Enter') {
+      onRunClick();
+    }
+  };
+
+  return (
+    <Box sx={{ p: 2 }}>
+      <Paper elevation={3} sx={{ color: 'primary.main' }}>
+        <Box sx={{ p: 2 }}>
+          <TextField
+            fullWidth
+            multiline
+            rows={10}
+            variant="outlined"
+            onKeyDown={onKeyDown}
+            inputRef={textInput}
+            placeholder="Enter Query..."
+          />
+        </Box>
+        <Box sx={{ p: 2, borderTop: 1 }}>
+          <Grid container spacing={2} justifyContent="flex-start">
+            <Grid item xs>
+              <Tooltip title="Clear Query Text">
+                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
+                  Clear
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Run (Ctrl+Enter)">
+                <Button onClick={onRunClick} sx={{
+                  color: `query_start_button.${isRunning ? 'active' : 'inactive'}.color`,
+                  bgcolor: `query_start_button.${isRunning ? 'active' : 'inactive'}.background_color`,
+                  padding: '11px'
+                }}>
+                  <PlayArrowIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+            <Grid item>
+              <Tooltip title="Stop">
+                <Button onClick={onStopClick} sx={{
+                  color: `query_stop_button.${isRunning ? 'inactive' : 'active'}.color`,
+                  bgcolor: `query_stop_button.${isRunning ? 'inactive' : 'active'}.background_color`,
+                  padding: '11px'
+                }}>
+                  <StopIcon />
+                </Button>
+              </Tooltip>
+            </Grid>
+          </Grid>
+        </Box>
+      </Paper>
+    </Box>
+  );
+};
+
+QueryInputForm.propTypes = {
+  isRunning: PropTypes.bool.isRequired,
+  onStart: PropTypes.func.isRequired,
+  onStop: PropTypes.func.isRequired,
+};
+
+export default QueryInputForm;
@@ -0,0 +1,88 @@
+import { createTheme } from '@mui/material/styles';
+import PropTypes from 'prop-types';
+import '@fontsource/noto-sans';
+import './App.css';
+
+const getLightPalette = () => {
+  return {
+    mode: 'light',
+    primary: {
+      light: '#6573c3',
+      main: '#3f51b5',
+      dark: '#2c387e',
+      contrastText: '#ffffff',
+    },
+    secondary: {
+      light: '#f73378',
+      main: '#f50057',
+      dark: '#ab003c',
+      contrastText: '#ffffff',
+    },
+    text: {
+      primary: '#000000c4',
+      secondary: '#00000086',
+      disabled: '#0000005b',
+      hint: '#0000005b',
+    }
+  };
+};
+
+const getDarkPalette = () => {
+  return {
+    mode: 'dark',
+    primary: {
+      light: '#bcc5ff',
+      main: '#33baf9',
+      dark: '#202334',
+      contrastText: '#78a2ac',
+    },
+    secondary: {
+      light: '#9fd0b8',
+      main: '#5eb289',
+      dark: '#ffffff12',
+      contrastText: '#c8d4f7',
+    },
+    text: {
+      primary: '#eaf8f8',
+      dark: '#eaf8f8a0'
+    },
+    query_start_button: {
+      active: {
+        color: '#33baf9',
+        background_color: '#ffffff40',
+      },
+      inactive: {
+        color: '#ffffff40',
+        background_color: '#ffffff00',
+      }
+    },
+    warning: {
+      main: '#c4823f'
+    },
+    query_stop_button: {
+      active: {
+        color: '#ff4444a0',
+        background_color: '#ffffff40',
+      },
+      inactive: {
+        color: '#ff444450',
+        background_color: '#ffffff00',
+      }
+    },
+  };
+};
+
+const getTheme = ({ mode }) => {
+  return createTheme({
+    typography: {
+      fontFamily: '"Helvetica"'
+    },
+    palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
+  });
+};
+
+export default getTheme;
+
+getTheme.propTypes = {
+  mode: PropTypes.string.isRequired,
+};
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>
@@ -0,0 +1,51 @@
+.dark-high-contrast {
+  --md-sys-color-primary: rgb(251 250 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 0 0);
+  --md-sys-color-primary-container: rgb(173 204 255);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(250 250 255);
+  --md-sys-color-on-secondary: rgb(0 0 0);
+  --md-sys-color-secondary-container: rgb(159 208 255);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(255 249 251);
+  --md-sys-color-on-tertiary: rgb(0 0 0);
+  --md-sys-color-tertiary-container: rgb(222 193 231);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 249 249);
+  --md-sys-color-on-error: rgb(0 0 0);
+  --md-sys-color-error-container: rgb(255 186 177);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(255 255 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(251 250 255);
+  --md-sys-color-outline: rgb(200 202 212);
+  --md-sys-color-outline-variant: rgb(200 202 212);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(0 0 0);
+  --md-sys-color-inverse-primary: rgb(0 42 83);
+  --md-sys-color-primary-fixed: rgb(219 231 255);
+  --md-sys-color-on-primary-fixed: rgb(0 0 0);
+  --md-sys-color-primary-fixed-dim: rgb(173 204 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
+  --md-sys-color-secondary-fixed: rgb(214 233 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 0 0);
+  --md-sys-color-secondary-fixed-dim: rgb(159 208 255);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
+  --md-sys-color-tertiary-fixed: rgb(249 222 255);
+  --md-sys-color-on-tertiary-fixed: rgb(0 0 0);
+  --md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}
@@ -0,0 +1,51 @@
+.dark-medium-contrast {
+  --md-sys-color-primary: rgb(173 204 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 22 49);
+  --md-sys-color-primary-container: rgb(112 146 198);
+  --md-sys-color-on-primary-container: rgb(0 0 0);
+  --md-sys-color-secondary: rgb(159 208 255);
+  --md-sys-color-on-secondary: rgb(0 24 42);
+  --md-sys-color-secondary-container: rgb(100 149 193);
+  --md-sys-color-on-secondary-container: rgb(0 0 0);
+  --md-sys-color-tertiary: rgb(222 193 231);
+  --md-sys-color-on-tertiary: rgb(33 14 42);
+  --md-sys-color-tertiary-container: rgb(162 136 171);
+  --md-sys-color-on-tertiary-container: rgb(0 0 0);
+  --md-sys-color-error: rgb(255 186 177);
+  --md-sys-color-on-error: rgb(55 0 1);
+  --md-sys-color-error-container: rgb(255 84 73);
+  --md-sys-color-on-error-container: rgb(0 0 0);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(251 250 255);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(200 202 212);
+  --md-sys-color-outline: rgb(160 163 171);
+  --md-sys-color-outline-variant: rgb(128 131 139);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(40 42 47);
+  --md-sys-color-inverse-primary: rgb(35 73 120);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 17 40);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 18 34);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(28 9 37);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}
@@ -0,0 +1,51 @@
+.dark {
+  --md-sys-color-primary: rgb(165 200 255);
+  --md-sys-color-surface-tint: rgb(165 200 255);
+  --md-sys-color-on-primary: rgb(0 49 94);
+  --md-sys-color-primary-container: rgb(34 72 118);
+  --md-sys-color-on-primary-container: rgb(212 227 255);
+  --md-sys-color-secondary: rgb(154 203 250);
+  --md-sys-color-on-secondary: rgb(0 51 83);
+  --md-sys-color-secondary-container: rgb(12 74 114);
+  --md-sys-color-on-secondary-container: rgb(206 229 255);
+  --md-sys-color-tertiary: rgb(218 189 226);
+  --md-sys-color-on-tertiary: rgb(61 41 70);
+  --md-sys-color-tertiary-container: rgb(85 63 93);
+  --md-sys-color-on-tertiary-container: rgb(247 216 255);
+  --md-sys-color-error: rgb(255 180 171);
+  --md-sys-color-on-error: rgb(105 0 5);
+  --md-sys-color-error-container: rgb(147 0 10);
+  --md-sys-color-on-error-container: rgb(255 218 214);
+  --md-sys-color-background: rgb(17 19 24);
+  --md-sys-color-on-background: rgb(225 226 233);
+  --md-sys-color-surface: rgb(17 19 24);
+  --md-sys-color-on-surface: rgb(225 226 233);
+  --md-sys-color-surface-variant: rgb(67 71 78);
+  --md-sys-color-on-surface-variant: rgb(195 198 207);
+  --md-sys-color-outline: rgb(141 145 153);
+  --md-sys-color-outline-variant: rgb(67 71 78);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(225 226 233);
+  --md-sys-color-inverse-on-surface: rgb(46 48 53);
+  --md-sys-color-inverse-primary: rgb(60 96 144);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(17 19 24);
+  --md-sys-color-surface-bright: rgb(55 57 62);
+  --md-sys-color-surface-container-lowest: rgb(12 14 19);
+  --md-sys-color-surface-container-low: rgb(25 28 32);
+  --md-sys-color-surface-container: rgb(29 32 36);
+  --md-sys-color-surface-container-high: rgb(40 42 47);
+  --md-sys-color-surface-container-highest: rgb(50 53 58);
+}
@@ -0,0 +1,51 @@
+.light-high-contrast {
+  --md-sys-color-primary: rgb(0 34 70);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(29 68 114);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(0 36 61);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(3 70 110);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(46 26 55);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(81 59 89);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(78 0 2);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(140 0 9);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(0 0 0);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(32 36 43);
+  --md-sys-color-outline: rgb(63 67 74);
+  --md-sys-color-outline-variant: rgb(63 67 74);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(255 255 255);
+  --md-sys-color-inverse-primary: rgb(228 236 255);
+  --md-sys-color-primary-fixed: rgb(29 68 114);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(0 45 88);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(3 70 110);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(0 47 77);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(81 59 89);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}
@@ -0,0 +1,51 @@
+.light-medium-contrast {
+  --md-sys-color-primary: rgb(29 68 114);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(83 118 167);
+  --md-sys-color-on-primary-container: rgb(255 255 255);
+  --md-sys-color-secondary: rgb(3 70 110);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(71 121 163);
+  --md-sys-color-on-secondary-container: rgb(255 255 255);
+  --md-sys-color-tertiary: rgb(81 59 89);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(133 108 141);
+  --md-sys-color-on-tertiary-container: rgb(255 255 255);
+  --md-sys-color-error: rgb(140 0 9);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(218 52 46);
+  --md-sys-color-on-error-container: rgb(255 255 255);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(63 67 74);
+  --md-sys-color-outline: rgb(91 95 103);
+  --md-sys-color-outline-variant: rgb(119 123 131);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(83 118 167);
+  --md-sys-color-on-primary-fixed: rgb(255 255 255);
+  --md-sys-color-primary-fixed-dim: rgb(58 93 141);
+  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-secondary-fixed: rgb(71 121 163);
+  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
+  --md-sys-color-secondary-fixed-dim: rgb(43 96 137);
+  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed: rgb(133 108 141);
+  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
+  --md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}
@@ -0,0 +1,51 @@
+.light {
+  --md-sys-color-primary: rgb(60 96 144);
+  --md-sys-color-surface-tint: rgb(60 96 144);
+  --md-sys-color-on-primary: rgb(255 255 255);
+  --md-sys-color-primary-container: rgb(212 227 255);
+  --md-sys-color-on-primary-container: rgb(0 28 58);
+  --md-sys-color-secondary: rgb(46 98 140);
+  --md-sys-color-on-secondary: rgb(255 255 255);
+  --md-sys-color-secondary-container: rgb(206 229 255);
+  --md-sys-color-on-secondary-container: rgb(0 29 50);
+  --md-sys-color-tertiary: rgb(110 86 118);
+  --md-sys-color-on-tertiary: rgb(255 255 255);
+  --md-sys-color-tertiary-container: rgb(247 216 255);
+  --md-sys-color-on-tertiary-container: rgb(39 20 48);
+  --md-sys-color-error: rgb(186 26 26);
+  --md-sys-color-on-error: rgb(255 255 255);
+  --md-sys-color-error-container: rgb(255 218 214);
+  --md-sys-color-on-error-container: rgb(65 0 2);
+  --md-sys-color-background: rgb(249 249 255);
+  --md-sys-color-on-background: rgb(25 28 32);
+  --md-sys-color-surface: rgb(249 249 255);
+  --md-sys-color-on-surface: rgb(25 28 32);
+  --md-sys-color-surface-variant: rgb(224 226 236);
+  --md-sys-color-on-surface-variant: rgb(67 71 78);
+  --md-sys-color-outline: rgb(116 119 127);
+  --md-sys-color-outline-variant: rgb(195 198 207);
+  --md-sys-color-shadow: rgb(0 0 0);
+  --md-sys-color-scrim: rgb(0 0 0);
+  --md-sys-color-inverse-surface: rgb(46 48 53);
+  --md-sys-color-inverse-on-surface: rgb(240 240 247);
+  --md-sys-color-inverse-primary: rgb(165 200 255);
+  --md-sys-color-primary-fixed: rgb(212 227 255);
+  --md-sys-color-on-primary-fixed: rgb(0 28 58);
+  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
+  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
+  --md-sys-color-secondary-fixed: rgb(206 229 255);
+  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
+  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
+  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
+  --md-sys-color-tertiary-fixed: rgb(247 216 255);
+  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
+  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
+  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
+  --md-sys-color-surface-dim: rgb(217 218 224);
+  --md-sys-color-surface-bright: rgb(249 249 255);
+  --md-sys-color-surface-container-lowest: rgb(255 255 255);
+  --md-sys-color-surface-container-low: rgb(243 243 250);
+  --md-sys-color-surface-container: rgb(237 237 244);
+  --md-sys-color-surface-container-high: rgb(231 232 238);
+  --md-sys-color-surface-container-highest: rgb(225 226 233);
+}
@@ -0,0 +1,107 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+button#query-button {
+  height: 5em;
+  vertical-align: top;
+}
+
+button#play-button {
+  padding: '10px 20px';
+  font-size: '16px';
+  cursor: 'pointer';
+  background-color: '#28a745';
+  color: '#fff';
+  border: 'none';
+  border-radius: '5px';
+};
+
+button#pause-button {
+  background-color: #ffc107;
+}
+
+button#stop-button {
+  background-color: #dc3545;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+p#footer {
+  float: none;
+  clear: both;
+  vertical-align: bottom;
+}
+
+table#query-output {
+  float: none;
+  clear: both;
+}
+
+textarea#query-input {
+  height: 5em;
+  width: 50em;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}
@@ -0,0 +1,9 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App';
+
+ReactDOM.createRoot(document.getElementById('root')).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+)
@@ -0,0 +1,69 @@
+import { fileURLToPath, URL } from 'node:url';
+import { defineConfig } from 'vite';
+import plugin from '@vitejs/plugin-react';
+import fs from 'fs';
+import path from 'path';
+import child_process from 'child_process';
+import process, { env } from 'process';
+
+const baseFolder =
+  process.env.APPDATA !== undefined && process.env.APPDATA !== ''
+    ? `${process.env.APPDATA}/ASP.NET/https`
+    : `${process.env.HOME}/.aspnet/https`;
+
+const certificateName = "bts.datastreamwebapp.client";
+const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
+const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
+
+if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
+  if (0 !== child_process.spawnSync('dotnet', [
+    'dev-certs',
+    'https',
+    '--export-path',
+    certFilePath,
+    '--format',
+    'Pem',
+    '--no-password',
+  ], { stdio: 'inherit', }).status) {
+    throw new Error("Could not create certificate.");
+  }
+}
+
+const target = env.ASPNETCORE_HTTPS_PORT
+  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
+  : env.ASPNETCORE_URLS
+    ? env.ASPNETCORE_URLS.split(';')[0]
+    : 'https://localhost:5001';
+
+export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
+  return {
+    plugins: [plugin()],
+    resolve: {
+      alias: {
+        '@': fileURLToPath(new URL('./src', import.meta.url))
+      }
+    },
+    build: {
+      minify: false,
+      sourcemap: true,
+      outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
+    },
+    server: {
+      proxy: {
+        '^/api': {
+          target,
+          secure: false
+        },
+        '^/ws': {
+          target,
+          secure: false
+        }
+      },
+      port: 5173,
+      https: {
+        key: fs.readFileSync(keyFilePath),
+        cert: fs.readFileSync(certFilePath),
+      }
+    }
+  };
+});
@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <InvariantGlobalization>true</InvariantGlobalization>
+    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
+    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
+    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
+    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+    </ProjectReference>
+    <ProjectReference Include="..\DataStream\DataStream.csproj" />
+    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
+    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
+    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
+  </ItemGroup>
+
+  <!-- <ItemGroup>
+    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
+  </ItemGroup> -->
+</Project>
@@ -0,0 +1,106 @@
+{
+  "MasterWorker": {
+    "Interval": "00:00:00.1"
+  },
+  "ViewStreamDatabase": {
+    "Interval": "00:00:00.1"
+  },
+  "Sparkd": {
+    "Config": {
+      "ConfigServer": "http://localhost:8000/config"
+    },
+    "Credentials": {
+      "User": "bts",
+      "Password": ""
+    },
+    "EnableAdmin": true,
+    "EnableOrders": true,
+    "EnablePosition": true
+  },
+  "PubSub": {
+    "MillisecondsBetweenSubInfoUpdates": 1000,
+    "MillisecondsBetweenTimeSliceUpdates": 5000,
+    "MillisecondsBetweenNoBufferSpaceRetries": 100,
+    "RetriesOnNoBufferSpace": 1,
+    "DoItAllSecondsBetweenLogs": 0,
+    "HeartbeatInterval": 15000,
+    "MissingHeartbeatsBeforeGc": 50,
+    "UseManagedSockets": true,
+    "StateUpdatePeriod": 5000,
+    "StatisticsUpdatePeriod": 5000,
+    "DisposedConnectionCheckPeriod": 1000,
+    "MessageProcessingWarningThreshold": 1000,
+    "ServerPort": 12345,
+    "Routing": {
+      "PublisherRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ],
+      "SubscriberRoutes": [
+        {
+          "Name": "default",
+          "ServerAddress": "bts-cloud://127.0.0.1:12345",
+          "TagRegExp": ".*"
+        }
+      ]
+    },
+    "MaxPacketSize": 500000000,
+    "BlueCommMustDie": true,
+    "CrushableTags": [],
+    "TagTypeToAssemblyMap": {
+      "Entries": [
+        {
+          "HandlerClass": "BCG.PubSubServerTagHandler",
+          "Assembly": "DataStreamCore.dll"
+        }
+      ]
+    },
+    "ServerQueue": "PubSubQueue",
+    "PortRange": {
+      "First": 30400,
+      "Last": 31700
+    },
+    "BindToRandomPort": false,
+    "DebugTags": "^NOTAGS$",
+    "LatencyReportThresholdMs": 1500,
+    "UseSubstations": false,
+    "UseDynamicClients": false,
+    "IsSubstation": false,
+    "InternallyPublishSubInfo": true,
+    "UseManagedClientsInNativeCode": false,
+    "CompressedTags": "^NOTAGS$",
+    "TagsToReportCompressionRatio": "^NOTAGS$",
+    "DisableMsmq": false,
+    "UsePrepublishedServerState": false,
+    "IpTranslation": null,
+    "GcActivationThreshold": 0,
+    "WaitPeriodAfterGc": 1000,
+    "TagHandlerFactory": {
+      "Handlers": [
+        {
+          "TagType": "DataStream",
+          "Assembly": "DataStreamCore",
+          "Class": "bts.PubsubServerStreamTagHandler"
+        },
+        {
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
+        },
+        {
+          "TagType": ".*",
+          "Assembly": "DataStreamCore",
+         "Class": "bts.PubsubServerGenericDataHandler"
+        }
+      ]
+    }
+  }
+}
@@ -0,0 +1,4 @@
+{
+  "content_root_path": "build/Bts.DataStreamWebApp.Client/production",
+  "web_root_path": "dist"
+}
@@ -0,0 +1,30 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using Microsoft.AspNetCore.Mvc;
+
+[Route("/ws")]
+public class DataStreamSubscriptionController : ControllerBase {
+   readonly ILogger logger;
+
+   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
+      this.logger = logger;
+      logger.LogInformation("Creating {ControllerType}", GetType().Name);
+   }
+
+   [Route("/ws")]
+   public async Task Get(CancellationToken cancellationToken) {
+      if (!HttpContext.WebSockets.IsWebSocketRequest) {
+         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
+         return;
+      }
+      logger.LogInformation("Establishing websocket connection...");
+      try {
+         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
+         logger.LogInformation("Websocket connection established.");
+         using var server = new DataStreamWebSocketServer(webSocket, logger);
+         await server.Run(cancellationToken);
+      } catch (Exception ex) {
+         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
+      }
+   }
+}
@@ -0,0 +1,125 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamCore;
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamSubscription : IDisposable, IHostedService {
+   public readonly ViewStream ViewStream;
+   public DataPacket? LastFired;
+
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+
+   public DataStreamSubscription(string id, JsonRpcId requestId, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
+      ViewStream = viewStream;
+      RequestId = requestId;
+      Id = id;
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public string Id { get; }
+   public JsonRpcId RequestId { get; }
+   public SubscriptionStatus Status { get; private set; }
+
+   public void Start() {
+      if (Status == SubscriptionStatus.Subscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Subscribed;
+      ViewStream.onChange += onChange;
+   }
+
+   public Task StartAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Start();
+      return Task.CompletedTask;
+   }
+
+   public Task Clear() {
+      if (LastFired is null) {
+         return Task.CompletedTask;
+      }
+      logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
+      return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
+   }
+
+   public void Stop() {
+      if (Status == SubscriptionStatus.Unsubscribed) {
+         logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
+         return;
+      }
+      logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
+      Status = SubscriptionStatus.Unsubscribed;
+      ViewStream.onChange -= onChange;
+   }
+
+   public Task StopAsync(CancellationToken cancellationToken) {
+      cancellationToken.ThrowIfCancellationRequested();
+      Stop();
+      return Task.CompletedTask;
+   }
+
+   public void Dispose() =>
+      Stop();
+
+   async void onChange(object newValue) {
+      if (newValue is not DataPacket dp) {
+         logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
+         return;
+      }
+      if (webSocket.State != WebSocketState.Open) {
+         logger.LogError(
+            "Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
+         Dispose();
+         return;
+      }
+      await sendDataPacket("subscribe", dp);
+   }
+
+   async Task sendDataPacket(string method, DataPacket dp) {
+      var serializerOpts = new JsonSerializerOptions {
+         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
+      };
+      serializerOpts.Converters.Add(new DataPacketJsonConverter() {
+         IncludeCurrent = true,
+         UseObjectRows = true,
+      });
+      var res = JsonRpcRequest.CreateNotification(method, dp);
+      var str = serialize(res, serializerOpts);
+      if (str is null) {
+         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
+         Dispose();
+         return;
+      }
+      logger.LogInformation(
+         "Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
+         dp.newRows.Count,
+         dp.schema.columnNames.Length,
+         str.Length);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
+      LastFired = dp;
+   }
+
+   string? serialize<T>(T message, JsonSerializerOptions? opts) {
+      opts ??= JsonSerializerOptions.Default;
+      try {
+         return JsonSerializer.Serialize(message, opts);
+      } catch (Exception ex) {
+         logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
+      }
+      return null;
+   }
+}
@@ -0,0 +1,89 @@
+namespace Bts.DataStreamWebApp.Server;
+
+using Bts.PubSub;
+
+using CommandLine;
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+class DataStreamWebAppServerConfig {
+   /// <summary>
+   /// The login user for sparkd
+   /// </summary>
+   [JsonPropertyName("user")]
+   [Option("user", HelpText = "The sparkd login user")]
+   public string? User { get; set; }
+
+   /// <summary>
+   /// The login password for sparkd
+   /// </summary>
+   [JsonPropertyName("password")]
+   [Option("pass", HelpText = "The sparkd login password")]
+   public string? Password { get; set; }
+
+   /// <summary>
+   /// The config server url for sparkd
+   /// </summary>
+   [JsonPropertyName("sparkd")]
+   [Option("sparkd", HelpText = "The sparkd config server")]
+   public string? Sparkd { get; set; }
+
+   [JsonPropertyName("pubsub")]
+   public PubSubConfig? PubSub { get; set; }
+
+   [Option("content-root-path")]
+   [JsonPropertyName("content_root_path")]
+   public string? ContentRootPath { get; set; }
+
+   [Option("web-root-path")]
+   [JsonPropertyName("web_root_path")]
+   public string? WebRootPath { get; set; }
+
+   [Option("environment-name")]
+   [JsonPropertyName("environment_name")]
+   public string? EnvironmentName { get; set; }
+
+   public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
+
+   [Option(
+      "config",
+      MetaValue = "PATH",
+      HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
+   [JsonIgnore]
+   public string? ConfigFile { get; set; }
+
+   public static DataStreamWebAppServerConfig Default() =>
+      new() {
+         User = "bts",
+         PubSub = PubSubConfig.Default(),
+      };
+
+   public void MergeWith(DataStreamWebAppServerConfig other) {
+      User ??= other.User;
+      Password ??= other.Password;
+      PubSub ??= other.PubSub;
+      Sparkd ??= other.Sparkd;
+      ContentRootPath ??= other.ContentRootPath;
+      WebRootPath ??= other.WebRootPath;
+      ConfigFile ??= other.ConfigFile;
+   }
+
+   /// <summary>
+   /// Merges options with those defined in `ConfigFile` (does not overwrite
+   /// non-null existing values).
+   /// </summary>
+   /// <exception cref="InvalidOperationException">
+   /// JsonSerializer.Deserialize returns null
+   /// </exception>
+   public void MergeWithConfigFile() {
+      if (ConfigFile is null) {
+         return;
+      }
+      var cfgString = File.ReadAllText(ConfigFile);
+      var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
+         throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
+      MergeWith(parsedCfg);
+      ConfigFile = null;
+   }
+}
@@ -0,0 +1,121 @@
+namespace Bts.DataStreamWebApp.Server.Controllers;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamWebApp.Server.Messages;
+
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
+   readonly ILogger logger;
+   readonly WebSocket webSocket;
+   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
+
+   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
+      this.webSocket = webSocket;
+      this.logger = logger;
+   }
+
+   public async Task<int> Run(CancellationToken cancellationToken) {
+      logger.LogInformation("DataStreamWebSocketServer running");
+      await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger, cancellationToken)) {
+         await (req.Method switch {
+            "subscribe" => onSubscribe(req, cancellationToken),
+            "unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>(), cancellationToken),
+            _ => Task.CompletedTask,
+         });
+      }
+      logger.LogInformation("DataStreamWebSocketServer stopped");
+      return 0;
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions. To additionally notify the client of the subscription status
+   /// change(s), use <see cref="DisposeAsync"/>.
+   /// </summary>
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      disposeSubscriptions();
+   }
+
+   /// <summary>
+   /// Disposes internal subscriptions (like <see cref="Dispose"/>), then sends 'subscription status change'
+   /// notifications to the client (unlike <see cref="Dispose"/>).
+   /// </summary>
+   public async ValueTask DisposeAsync() {
+      GC.SuppressFinalize(this);
+      var finalSubs = subscriptions.Values.Select(s => (s.RequestId, s.Id)).ToList();
+      disposeSubscriptions();
+      // Send a 'subscription status change' msg to the client for each one.
+      foreach (var (requestId, id) in finalSubs) {
+         await sendSubscriptionStatus(requestId, id, "unsubscribe", SubscriptionStatus.Unsubscribed, CancellationToken.None);
+      }
+   }
+
+   async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      if (!SubscribeRequest.ParseQuery(req, out var vs, out var subId, out var errMsg, out var errCode)) {
+         logger.LogError("Invalid 'subscribe' request: {Reason}", errMsg);
+         var rpcError = new JsonRpcError<JsonNode>() {
+            Code = (long)errCode,
+            Message = $"{errCode}: {errMsg}",
+         };
+         await send(rpcError, cancellationToken);
+         return;
+      }
+      if (subscriptions.TryGetValue(subId, out var oldSub)) {
+         oldSub.Dispose();
+      }
+      var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
+      subscriptions[subId] = sub;
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
+      await sub.StartAsync(cancellationToken);
+   }
+
+   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
+      ArgumentNullException.ThrowIfNull(req.Id);
+      ArgumentException.ThrowIfNullOrEmpty(req.Method);
+      var subId = req.Params?.SubscriptionId ?? string.Empty;
+      if (!subscriptions.TryGetValue(subId, out var sub)) {
+         logger.LogInformation("Ignoring '{Method}' request because no subscription was found with id {Id}.", req.Method, subId);
+         return;
+      }
+      logger.LogInformation("Disposing subscription");
+      sub.Dispose();
+      subscriptions.Remove(subId);
+      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   async Task unsubscribe(JsonRpcId requestId, string subscriptionId, CancellationToken cancellationToken) {
+      await sendSubscriptionStatus(requestId, subscriptionId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
+   }
+
+   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
+      logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
+      var response = JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
+         SubscriptionId = subId,
+         Status = status,
+      });
+      return send(response, cancellationToken);
+   }
+
+   Task send<T>(T message, CancellationToken cancellationToken) {
+      if (message is null) {
+         return Task.CompletedTask;
+      }
+      var str = JsonSerializer.Serialize(message);
+      var bytes = Encoding.UTF8.GetBytes(str);
+      logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
+      return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken);
+   }
+
+   void disposeSubscriptions() {
+      logger.LogInformation("Disposing DataStreamWebSocketServer subscriptions. Count: {Count}", subscriptions.Count);
+      subscriptions.Values.DisposeAll();
+      subscriptions.Clear();
+   }
+}
@@ -0,0 +1,6 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+enum ErrorCode {
+   None = 0,
+   InvalidSubscriptionRequest,
+}
@@ -0,0 +1,67 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using BCG;
+
+using bts.websocket_core.json_rpc;
+
+using Bts.DataStreamExpressions;
+
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+public class SubscribeRequest {
+   [JsonPropertyName("query")]
+   public string? Query { get; set; }
+
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   internal static bool ParseQuery(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(true)] out string? subscriptionId, [NotNullWhen(false)] out string? errorMessage, out ErrorCode errorCode) {
+      stream = null;
+      subscriptionId = null;
+      errorCode = ErrorCode.None;
+      var paramsJsonObj = req.Params?.AsObject();
+      if (paramsJsonObj is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params' object not found";
+         return false;
+      }
+      var subReq = paramsJsonObj.Deserialize<SubscribeRequest>();
+      if (subReq is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params' object invalid";
+         return false;
+      }
+      subscriptionId = subReq.SubscriptionId ?? string.Empty;
+      var query = subReq.Query;
+      if (query is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params.query' field not found (expected a string)";
+         return false;
+      }
+      errorMessage = null;
+      var exp = DataStreamExpressionsInterpreter.Evaluate(query);
+      if (exp is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "'params.query' expression has invalid syntax";
+         return false;
+      }
+      string? exceptionMsg = null;
+      try {
+         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
+      } catch (Exception ex) {
+         exceptionMsg = ex.ToString();
+      }
+      if (stream is null) {
+         errorCode = ErrorCode.InvalidSubscriptionRequest;
+         errorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
+         if (exceptionMsg is not null) {
+            errorMessage += ". Error" + exceptionMsg;
+         }
+         return false;
+      }
+      return true;
+   }
+}
@@ -0,0 +1,18 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+enum SubscriptionStatus {
+   None,
+   Subscribed,
+   Unsubscribed,
+}
+
+class SubscriptionStatusMessage {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+
+   [JsonPropertyName("status")]
+   [JsonConverter(typeof(JsonStringEnumConverter))]
+   public SubscriptionStatus Status { get; set; } = SubscriptionStatus.None;
+}
@@ -0,0 +1,8 @@
+namespace Bts.DataStreamWebApp.Server.Messages;
+
+using System.Text.Json.Serialization;
+
+public class UnsubscribeRequest {
+   [JsonPropertyName("subscription_id")]
+   public string? SubscriptionId { get; set; }
+}
@@ -0,0 +1,59 @@
+using Bts.DataStreamCore.Services;
+using Bts.PubSub.Services;
+
+using Spark.Api.Services;
+
+using System.Reactive.Linq;
+
+var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
+   Args = args,
+   ApplicationName = "Bts.DataStreamWebApp.Server",
+});
+ConfigureApp(builder.Configuration);
+ConfigureServices(builder.Services, builder.Configuration);
+
+var app = builder.Build();
+if (app.Environment.IsDevelopment()) {
+   app.UseDirectoryBrowser();
+}
+var wsOpts = new WebSocketOptions {
+   KeepAliveInterval = TimeSpan.FromMinutes(2)
+};
+app.UseWebSockets(wsOpts);
+app.UseDefaultFiles();
+app.UseStaticFiles();
+if (app.Environment.IsDevelopment()) {
+   app.UseSwagger();
+   app.UseSwaggerUI();
+}
+app.UseHttpsRedirection();
+app.UseAuthorization();
+app.MapControllers();
+try {
+   Console.Write($"Starting Application");
+   app.Run();
+   Console.Write($"Application exited gracefully.");
+} catch (Exception ex) {
+   Console.Error.Write($"Application crashed: {ex}");
+   return 1;
+}
+return 0;
+
+static void ConfigureApp(ConfigurationManager configBuilder) {
+   configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
+   var cfgRoot = configBuilder.GetSection("config");
+   if (cfgRoot?.Value is not null) {
+      configBuilder.AddJsonFile(cfgRoot.Value);
+   }
+}
+
+static void ConfigureServices(IServiceCollection services, IConfiguration config) {
+   services.AddDirectoryBrowser();
+   services.AddControllers();
+   services.AddEndpointsApiExplorer();
+   services.AddSwaggerGen();
+   services.AddMasterWorker(config.GetSection("MasterWorker"));
+   services.AddViewStreamDatabase(config.GetSection("ViewStreamDatabase"));
+   services.AddPubSubClient(config.GetSection("PubSub"));
+   services.AddSparkd(config.GetSection("Sparkd"));
+}
@@ -0,0 +1,26 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "applicationUrl": "http://localhost:5000",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:5000;https://localhost:5001",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
+      }
+    },
+  },
+}
@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}
@@ -0,0 +1,168 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>ALT WebSocket Sample Application</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Message to send:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            var data = sendMessage.value;
+            socket.send(data);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>
@@ -0,0 +1,176 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <title></title>
+    <style>
+        table {
+            border: 0
+        }
+
+        .commslog-data {
+            font-family: Consolas, Courier New, Courier, monospace;
+        }
+
+        .commslog-server {
+            background-color: red;
+            color: white
+        }
+
+        .commslog-client {
+            background-color: green;
+            color: white
+        }
+    </style>
+</head>
+<body>
+    <h1>DataStreams via WebSocket</h1>
+    <p id="stateLabel">Ready to connect...</p>
+    <div>
+        <label for="connectionUrl">WebSocket Server URL:</label>
+        <input id="connectionUrl" />
+        <button id="connectButton" type="submit">Connect</button>
+    </div>
+    <p></p>
+    <div>
+        <label for="sendMessage">Query:</label>
+        <input id="sendMessage" disabled />
+        <button id="sendButton" type="submit" disabled>Send</button>
+        <button id="closeButton" disabled>Close Socket</button>
+    </div>
+
+    <h2>Communication Log</h2>
+    <table style="width: 800px">
+        <thead>
+            <tr>
+                <td style="width: 100px">From</td>
+                <td style="width: 100px">To</td>
+                <td>Data</td>
+            </tr>
+        </thead>
+        <tbody id="commsLog">
+        </tbody>
+    </table>
+
+    <script>
+        var connectionUrl = document.getElementById("connectionUrl");
+        var connectButton = document.getElementById("connectButton");
+        var stateLabel = document.getElementById("stateLabel");
+        var sendMessage = document.getElementById("sendMessage");
+        var sendButton = document.getElementById("sendButton");
+        var commsLog = document.getElementById("commsLog");
+        var closeButton = document.getElementById("closeButton");
+        var socket;
+
+        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+        var port = document.location.port ? (":" + document.location.port) : "";
+
+        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
+
+        function updateState() {
+            function disable() {
+                sendMessage.disabled = true;
+                sendButton.disabled = true;
+                closeButton.disabled = true;
+            }
+            function enable() {
+                sendMessage.disabled = false;
+                sendButton.disabled = false;
+                closeButton.disabled = false;
+            }
+
+            connectionUrl.disabled = true;
+            connectButton.disabled = true;
+
+            if (!socket) {
+                disable();
+            } else {
+                switch (socket.readyState) {
+                    case WebSocket.CLOSED:
+                        stateLabel.innerHTML = "Closed";
+                        disable();
+                        connectionUrl.disabled = false;
+                        connectButton.disabled = false;
+                        break;
+                    case WebSocket.CLOSING:
+                        stateLabel.innerHTML = "Closing...";
+                        disable();
+                        break;
+                    case WebSocket.CONNECTING:
+                        stateLabel.innerHTML = "Connecting...";
+                        disable();
+                        break;
+                    case WebSocket.OPEN:
+                        stateLabel.innerHTML = "Open";
+                        enable();
+                        break;
+                    default:
+                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
+                        disable();
+                        break;
+                }
+            }
+        }
+
+        closeButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            socket.close(1000, "Closing from client");
+        };
+
+        sendButton.onclick = function () {
+            if (!socket || socket.readyState !== WebSocket.OPEN) {
+                alert("socket not connected");
+            }
+            const rpcReq = {
+              jsonrpc: "2.0",
+              method: "subscribe",
+              params: {
+                query: sendMessage.value,
+              },
+              id: 1,
+            };
+            const rpcReqStr = JSON.stringify(rpcReq);
+            socket.send(rpcReqStr);
+            commsLog.innerHTML += '<tr>' +
+                '<td class="commslog-client">Client</td>' +
+                '<td class="commslog-server">Server</td>' +
+                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
+        };
+
+        connectButton.onclick = function() {
+            stateLabel.innerHTML = "Connecting";
+            socket = new WebSocket(connectionUrl.value);
+            socket.onopen = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
+                '</tr>';
+            };
+            socket.onclose = function (event) {
+                updateState();
+                commsLog.innerHTML += '<tr>' +
+                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
+                '</tr>';
+            };
+            socket.onerror = updateState;
+            socket.onmessage = function (event) {
+                commsLog.innerHTML += '<tr>' +
+                    '<td class="commslog-server">Server</td>' +
+                    '<td class="commslog-client">Client</td>' +
+                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
+            };
+        };
+
+        function htmlEscape(str) {
+            return str.toString()
+                .replace(/&/g, '&amp;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;');
+        }
+    </script>
+</body>
+</html>
@@ -0,0 +1,26 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Reactive.Linq;
+
+public static class DataStreamExtensions {
+   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
+      source as ViewStream ?? new ObservableViewStream(source);
+
+   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
+
+      IDisposable Subscribe(IObserver<DataPacket> observer) {
+         source.onChange += OnChange;
+         return Disposable.Create(() => source.onChange -= OnChange);
+
+         void OnChange(object? newValue) =>
+            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
+               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
+      }
+   }
+}
@@ -0,0 +1,29 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using System;
+using System.IO;
+
+public class PandasScriptStream : ViewStream {
+   public PandasScriptStream(ViewStream vs, string file)
+      : this(vs, file, null) { }
+
+   public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
+      : this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
+
+   public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
+      mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
+
+      string readFile() {
+         try {
+            return File.ReadAllText(file);
+         } catch (Exception ex) {
+            Log.Gen.error($"PandasScriptStream error reading file: {ex}");
+            throw;
+         }
+      }
+   }
+}
@@ -1,62 +1,130 @@
-// using System;
-// using System.Diagnostics;
-
-// using BCG;
-// using BCG.Utilities.General;
-
-
-// namespace bts {
-
-//     public class PandasStream : ViewStream, IWorker {
-//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
-
-//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
-
-//         }
-
-//         public PandasStream(ViewStream vs, string pandas_code) :
-//         this(vs, ViewStreamDatabase.instance, pandas_code) {
-//         }
-
-
-//         protected override void myOnFirstSubscriber() {
-
-//             // string python_code = @"
-//             //    import winsound
-//             //    frequency = 2500  # Set frequency (in Hz)
-//             //    duration = 1000   # Set duration (in milliseconds)
-//             //    winsound.Beep(frequency, duration)
-//             // ";
-//             string python_code = @"
-//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
-//             with open(file_path, \"w\") as f:
-//                 f.write(\"Hello, world!\n\")
-//             ";
-
-//             execute_python_code(python_code);
-//         }
-
-//         static void execute_python_code(string python_code) {
-//             ProcessStartInfo start = new ProcessStartInfo();
-//             start.FileName = "python"; // Python executable
-//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
-//             start.UseShellExecute = false;
-//             start.RedirectStandardOutput = true;
-
-//             using (Process process = Process.Start(start)) {
-//                 using (System.IO.StreamReader reader = process.StandardOutput) {
-//                     string result = reader.ReadToEnd();
-//                     Console.Write(result);
-//                 }
-//             }
-//         }
-
-//         protected override void myOnLastSubscriber() {
-//         }
-
-//         public void doWork() {
-
-//         }
-
-//     }
-// }
+namespace Bts.DataStream;
+
+using BCG;
+using BCG.Extensions;
+using BCG.Utilities.General;
+
+using bts;
+using bts.collections;
+
+using Bts.DataStreamCore;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+
+public class PandasStream : ViewStream, IWorker {
+   readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
+   readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
+   readonly string pythonExe;
+   readonly ViewStreamDatabase vsd;
+   readonly CSVSaverStream inputCsvStream;
+   readonly Lazy<string> pythonCode;
+   IDictionary<int, IBlueRow>? lastRowMap = null;
+   int inputVersion = 0;
+   int outputVersion = 0;
+   DateTime lastFireTime = DateTime.MinValue;
+   TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
+   IDisposable? dispose;
+
+   public PandasStream(ViewStream vs, string dataFrameExpression)
+      : this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }
+
+   public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
+      pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
+      this.pythonExe = pythonExe;
+      this.vsd = vsd;
+      // This should fire an update when it writes to the output file
+      inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
+      pythonCode = new(getPythonCode);
+
+      string getPythonCode() {
+         var imports =
+@"import numpy as np
+import pandas as pd
+import sys
+";
+         var bd = new StringBuilder(imports);
+         bd.AppendLine($"df=pd.read_csv('{inputFile}')");
+         bd.AppendLine(dataFrameExpression);
+         bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
+         return bd.ToString();
+      }
+   }
+
+   protected override void myOnFirstSubscriber() {
+      vsd.addWorker(this);
+      inputCsvStream.onChange += onInputChanged;
+      this.dispose = Disposable.Create(dispose);
+
+      void onInputChanged(object newValue) =>
+         inputVersion++;
+
+      void dispose() {
+         inputCsvStream.onChange -= onInputChanged;
+         vsd.removeWorker(this);
+         deleteFile(inputFile);
+         deleteFile(outputFile);
+      }
+
+      void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
+         if (!File.Exists(file)) {
+            return;
+         }
+         try {
+            File.Delete(file);
+         } catch (Exception ex) {
+            Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
+         }
+      }
+   }
+
+   protected override void myOnLastSubscriber() =>
+      Disposable.dispose(ref dispose);
+
+   static void executePythonCode(string pythonCode, string? pythonExe = null) {
+      pythonExe ??= "python";
+      var start = new ProcessStartInfo {
+         FileName = pythonExe,
+         Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
+         UseShellExecute = false,
+         RedirectStandardOutput = true
+      };
+      using var process = Process.Start(start)
+         ?? throw new InvalidOperationException("Failed to launch python");
+      process.WaitForExit();
+   }
+
+   void IWorker.doWork() {
+      if (outputVersion == inputVersion) {
+         return;
+      }
+      var now = DateTime.Now;
+      var elapsedSinceLastFire = now - lastFireTime;
+      if (elapsedSinceLastFire < fireThrottle) {
+         return;
+      }
+      outputVersion = inputVersion;
+      executePythonCode(pythonCode.Value, pythonExe);
+      var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
+      var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
+      var comp = BlueRowEqualityComparer.Instance;
+      var deletedRows = new BlueTable(schema);
+      var addedRows = new BlueTable(schema);
+      if (lastRowMap is not null) {
+         deletedRows.SetItems(lastRowMap.Where(oldRow =>
+            !(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
+         addedRows.SetItems(rowMap.Where(newRow =>
+            !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
+      }
+      lastRowMap = rowMap;
+      lastFireTime = DateTime.Now;
+      myFireEvent(new DataPacket(rows, addedRows, deletedRows));
+   }
+}
@@ -39,8 +39,8 @@ namespace BCG {

        protected override void myOnFirstSubscriber() {
            MasterWorker.getInstance(false).addWorker(this, _send_period);
-            var server_address = new BlutoSubAddress(_ip, _port);
-            bts.AsyncTCPConnector_APM.connect_to(server_address, this);
+            var server_rci = new RemoteConnectionInfo(_ip, _port);
+            bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
        }

        protected override void myOnLastSubscriber() {
@@ -0,0 +1,37 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using System;
+
+public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
+   readonly IObservable<DataPacket> source;
+   IDisposable? subscription;
+
+   public ObservableViewStream(IObservable<DataPacket> source) =>
+      this.source = source;
+
+   public void Dispose() {
+      GC.SuppressFinalize(this);
+      if (subscription is not null) {
+         subscription.Dispose();
+         subscription = null;
+      }
+   }
+
+   protected override void myOnFirstSubscriber() {
+      Log.isnull(subscription);
+      subscription = source.Subscribe(myFireEvent);
+   }
+
+   protected override void myOnLastSubscriber() {
+      Log.notnull(subscription);
+      subscription.Dispose();
+      subscription = null;
+   }
+
+   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
+      source.Subscribe(observer);
+}
@@ -1,32 +0,0 @@
-using BCG;
-
-namespace BCG
-{
-   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
-   [DataStreamUserRespresentation("add_to_cloud")]
-   public class AddToCloudStream : ViewStream
-   {
-      private readonly string _name, _stream_exp;
-      private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
-      public AddToCloudStream(string name, params string[] lines)
-      : this(name, 0, lines)
-      {
-      }
-
-      public AddToCloudStream(string name, int persist, params string[] lines)
-      {
-         _name = name;
-         _stream_exp = getExpression(lines);
-         _requestor.add_to_cloud(_name, _stream_exp, persist == 1);
-         mySetRealSource(new TableStream("cloud_streams"));
-      }
-
-      static string getExpression(params string[] lines) {
-         for (int i = 0; i < lines.Length; i++) {
-            // unescape quotes
-            lines[i] = lines[i].Replace("\\\"", "\"");
-         }
-         return string.Join("\n", lines);
-      }
-   }
-}
@@ -1,31 +1,24 @@
-using BCG;
-
 using System;
 using System.Collections.Generic;

-using mbts.config;
-using bts.utils;
-using System.Threading.Tasks;
 using Bts.DataStreamExpressions;
-using System.Collections.Generic;
 using BCG.Utilities.General;
+using bts;
+using Bts.PubSub;
+using Bts.DataStreamCore;

-namespace BCG
-{
+namespace BCG {
   // listens to the requests posted on "cloud_streams" request table, and 
   // if unfulfilled starts a ViewPubsliher
-   public class CloudStreamManager : IWorker
-   {
-      public class StreamPubUnit
-      {
+   public class RemoteQueryManager : IWorker {
+      public class StreamPubUnit {
         public string name;
         public string stream_expression;
         public ViewStream stream;
         public bool persist;
         public ViewPublisher view_publisher;

-         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
-         {
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
            name = name_;
            stream_expression = stream_expression_;
            stream = stream_;
@@ -33,7 +26,7 @@ namespace BCG
            view_publisher = view_publisher_;
         }
      }
-      static ViewStream build_stream(string stream_expression)
+      static ViewStream? build_stream(string stream_expression)
      {
         try
         {
@@ -41,9 +34,7 @@ namespace BCG
            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
            return rv;

-         }
-         catch (Exception ex)
-         {
+         } catch (Exception ex) {
            bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
            return null;
         }
@@ -51,35 +42,40 @@ namespace BCG
      private IDictionary<string, StreamPubUnit>
         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
      private ViewStream _stream = new TableStream("cloud_streams");
-      private PubTableBroker _broker = new PubTableBroker();
+      readonly IPubTableBroker _broker;
+      readonly IManagedSubscriber _sub;

-      public CloudStreamManager()
-      {
+
+      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
+         _sub = managedSubscriber;
+         _broker = pubTableBroker;
         _stream.onChange += _on_str;
-
+         _sub.Subscribe("RemoteRequest", (string tag, byte[] data) => {
+            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
+            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
+         });
      }
      private void _on_str(object obj)
      {
-         DataPacket dp = obj as DataPacket;
+         if (obj is not DataPacket dp) {
+            return;
+         }
         foreach (IBlueRow r in dp.newRows)
         {
            string name = r.get<string>("name");
            bool persist = r.get<bool>("persist");
            string stream_exp = r.get<string>("stream_exp");
-            if (!_stream_pub_map.ContainsKey(name))
-            {
-               _assign(name, stream_exp, persist);
+            if (!_stream_pub_map.ContainsKey(name)) {
+               _spawn_remote_stream(name, stream_exp, persist);
            }
         }
      }

-      private void _assign(string name, string stream_expression, bool persist)
-      {
+      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
         ViewStream stream = build_stream(stream_expression);
         _expose_to_ws(name, stream);
-         if (persist)
-         {
+         if (persist) {
            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
         }
         ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
@@ -87,14 +83,12 @@ namespace BCG
         _stream_pub_map.Add(name, u);
      }

-      private void _expose_to_ws(string name, ViewStream stream)
-      {
+      private void _expose_to_ws(string name, ViewStream stream) {
         // TODO: john
      }

-      public void doWork()
-      {
+      public void doWork() {

      }
   }
-}
+}
@@ -0,0 +1,48 @@
+using BCG;
+
+using bts;
+
+using System;
+
+namespace BCG {
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("remoteQuery")]
+   public class RemoteQueryStream : ViewStream {
+      private readonly string _name, _stream_exp;
+      private PublisherCore _publisher_core = new PublisherCore();
+      private ViewStream _source_str;
+
+      public RemoteQueryStream(params string[] lines) {
+         _stream_exp = get_expression(lines);
+         _name = Guid.NewGuid().ToString();
+         _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
+         //_name.Replace("-", "_");
+
+      }
+
+      protected override void myOnFirstSubscriber() {
+         RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
+         byte[] rem_req_bytes = rem_req.serialize();
+         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
+         _source_str = new TableStream(_name);
+         _source_str.onChange += new ChangeDelegate(_handle_data);
+      }
+
+      protected override void myOnLastSubscriber() {
+         //maybe there should be a way to turn remote requests off ?
+         _source_str.onChange -= new ChangeDelegate(_handle_data);
+      }
+
+      static string get_expression(params string[] lines) {
+         for (int i = 0; i < lines.Length; i++) {
+            // unescape quotes
+            lines[i] = lines[i].Replace("\\\"", "\"");
+         }
+         return string.Join("\n", lines);
+      }
+
+      private void _handle_data(object o) {
+         myFireEvent(o);
+      }
+   }
+}
@@ -0,0 +1,31 @@
+namespace bts {
+   using System.IO;
+   using BCG;
+
+   using System.IO;
+   using System.Threading.Tasks.Dataflow;
+
+   public class RemoteRequest {
+      public readonly string stream_expression;
+      public readonly string published_table_name;
+      public RemoteRequest(string stream_expression_, string published_table_name_) {
+         stream_expression = stream_expression_;
+         published_table_name = published_table_name_;
+      }
+      public byte[] serialize() {
+         var str = new MemoryStream();
+         var bw = new BinaryWriter(str);
+         bw.write_string_cpp(stream_expression);
+         bw.write_string_cpp(published_table_name);
+         return str.ToArray();
+      }
+
+      public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
+         BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
+         string stream_expression_ = br.read_string_cpp();
+         string published_table_name_ = br.read_string_cpp();
+         return new RemoteRequest(stream_expression_, published_table_name_);
+
+      }
+   }
+}
@@ -1,59 +1,54 @@
-using BCG;
+// using BCG;

-using System.Linq;
+// using bts;

-namespace BCG
-{
-   public class StreamOffloadRequestor
-   {
-      public class OffloadOp
-      {
-         string name;
-         string stream_exp;
-      }
-      private IBlueTableSchema _schema;
-      private IBlueRowCollection _rows;
-      private PubTableBroker _broker = new PubTableBroker();
-      ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
-      public StreamOffloadRequestor()
-      {
-         _schema = new BlueTableSchema("cloud_streams",
-                                   new ColumnInfo("id", 0, typeof(int)),
-                                   new ColumnInfo("name", 1, typeof(string)),
-                                   new ColumnInfo("persist", 2, typeof(bool)),
-                                   new ColumnInfo("stream_exp", 3, typeof(string)));
-         _existing_offloads_vs.onChange += _on_str;
-      }
-      private void _on_str(object obj)
-      {
-         DataPacket dp = obj as DataPacket;
-         _rows = dp.newRows;
-         // this is needed so the broker knows the state of the table
-         _broker.createTable(_rows);
-      }
-      public void add_to_cloud(string name, string stream_exp, bool persist)
-      {
-         // check if it exists
-         if (_rows == null)
-         {
-            BlueTable table = new BlueTable(_schema);
-            IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
-            table.addRow(nr);
-            _broker.createTable(table);
-            // create table  
-         }
-         else
-         {
-            if (exists(name))
-               return;
-            IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
-            _broker.addRow(nr);
-         }
-      }
+// using System.Linq;

-      bool exists(string name)
-      {
-         return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
-      }
-   }
-}
+// namespace BCG {
+//    // used on the client
+//    public class StreamOffloadRequestor {
+//       public class OffloadOp {
+//          string name;
+//          string stream_exp;
+//       }
+//       private IBlueTableSchema _schema;
+//       private IBlueRowCollection _rows;
+//       private PubTableBroker _broker = new PubTableBroker();
+//       ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
+//       public StreamOffloadRequestor() {
+//          _schema = new BlueTableSchema("cloud_streams",
+//                                    new ColumnInfo("id", 0, typeof(int)),
+//                                    new ColumnInfo("name", 1, typeof(string)),
+//                                    new ColumnInfo("persist", 2, typeof(bool)),
+//                                    new ColumnInfo("stream_exp", 3, typeof(string)));
+//          _existing_offloads_vs.onChange += _on_str;
+//       }
+//       private void _on_str(object obj) {
+//          DataPacket dp = obj as DataPacket;
+//          _rows = dp.newRows;
+//          // this is needed so the broker knows the state of the table
+//          _broker.create_table(_rows);
+//       }
+//       public void add_to_cloud(string name, string stream_exp, bool persist) {
+//          // check if it exists
+//          if (_rows == null) {
+//             BlueTable table = new BlueTable(_schema);
+//             IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
+//             table.addRow(nr);
+//             _broker.create_table(table);
+//             // create table  
+//          } else {
+//             if (exists(name))
+//                return;
+//             IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
+//             // TODO: use make_batch_changes instead
+//             Log.assert(false, "Fix addRow");
+//             //_broker.addRow(nr);
+//          }
+//       }
+
+//       bool exists(string name) {
+//          return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
+//       }
+//    }
+// }
@@ -1,17 +1,14 @@
 using System;
 using System.Collections;

-namespace BCG
-{
-   internal class AddColumnsRow : IBlueRow
-   {
+namespace BCG {
+   internal class AddColumnsRow : IBlueRow {
      private IBlueTableSchema mySchema;
      private IBlueRow myRow;
      //private object[] myCachedValues;

      #region Constructors
-      public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
-      {
+      public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) {
         mySchema = schema;
         myRow = row;
         //myCachedValues = new object[schema.columnNames.Length];
@@ -19,85 +16,78 @@ namespace BCG
      #endregion

      #region IBlueRow members
-      public IBlueTableSchema schema{get{return mySchema;}}
+      public IBlueTableSchema schema { get { return mySchema; } }
      #endregion

      #region ICollection members
-      public int Count {get{return mySchema.columnNames.Length;}}
+      public int Count { get { return mySchema.columnNames.Length; } }

-      public bool IsSynchronized {get{return false;}}
+      public bool IsSynchronized { get { return false; } }

-      public object SyncRoot {get{return this;}}
+      public object SyncRoot { get { return this; } }

-      public void CopyTo (Array array, int index)  
-      {
+      public void CopyTo(Array array, int index) {
         int i = index;
         foreach (object o in this as IEnumerable)
-            array.SetValue (o, i++);
+            array.SetValue(o, i++);
      }
      #endregion

      #region IDictionary members
-      public bool IsFixedSize {get{return true;}}
+      public bool IsFixedSize { get { return true; } }

-      public bool IsReadOnly {get{return true;}}
+      public bool IsReadOnly { get { return true; } }

      /// <summary>
      /// The key can either be the string corrsponding to the column name or
      /// the order of the column
      /// </summary>
-      public object this [object key]
-      {
-         get
-         {
+      public object this[object key] {
+         get {
            if (key is string)
               return this[(string)key];
            return this[(int)key];
         }
-         set {throw new NotSupportedException ("The IBlueRow is readonly");}
+         set { throw new NotSupportedException("The IBlueRow is readonly"); }
      }

-      public object this[int index]
-      {
-         get
-         {
+      public object this[int index] {
+         get {
            //if (myCachedValues[index] == null)
            //   myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
            //return myCachedValues[index];
-            return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
+            return mySchema[index].isConstant ? DBNull.Value : myRow[index];
         }
      }

-      public object this[string colName]  {get{return this[mySchema[colName].colNo];}}
+      public object this[string colName] { get { return this[mySchema[colName].colNo]; } }

-      public ICollection Keys{get{return mySchema.columnNames;}}
+      public ICollection Keys { get { return mySchema.columnNames; } }

-      public ICollection Values
-      {
-         get
-         {
+      public ICollection Values {
+         get {
            object[] retVal = new object[mySchema.columnNames.Length];
-            CopyTo (retVal, 0);
+            CopyTo(retVal, 0);
            return retVal;
         }
      }

-      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }

-      public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }

-      public bool Contains (object key){return mySchema[(string)key] != null;}
+      public bool Contains(object key) { return mySchema[(string)key] != null; }

      /// <summary>
      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
      /// iterate through the values, typecast this into an IEnumerable
      /// </summary>
      /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
-      
-      IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
+      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }

-      public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
+      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
      #endregion

   }
@@ -1,6 +1,34 @@
-namespace BCG;
-
-public class AddColumnsStream : MapStream {
-   public AddColumnsStream(ViewStream vs, string columnDef)
-      : base(vs, new ColumnAdder(columnDef)) { }
-}
+using System;
+
+namespace BCG {
+   public class AddColumnsStream : ViewStream {
+      public static Tuple<string, Type, object> parse_exp(string exp) {
+         string[] toks = exp.Split(':');
+         string col_name = toks[0];
+         string col_value = toks[1];
+         Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
+         return Tuple.Create(col_name, tup.Item1, tup.Item2);
+      }
+
+      private Tuple<string, Type, object> _new_col;
+      private IBlueTableSchema _out_schema;
+
+      public AddColumnsStream(ViewStream vs, string col_def) {
+         _new_col = parse_exp(col_def);
+         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
+         mySetRealSource(rs);
+      }
+
+      private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
+         if (_out_schema == null)
+            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
+         return _out_schema;
+      }
+
+      private IBlueRow _map(IBlueRow r) {
+         MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
+         rv.setValue(_new_col.Item1, _new_col.Item3);
+         return rv;
+      }
+   }
+}
@@ -1,48 +1,24 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.IO;
+using BCG;

-using System.Runtime.Serialization.Formatters.Binary;
-
-namespace BCG
-{
+namespace bts {
   [DataStreamUserRespresentation("loader")]
-   public class LoaderStream : ViewStream, ISideEffectStream
-   {
-      public class Exception : System.Exception
-      {public Exception(string msg) : base(msg) { }}
+   public class LoaderStream : ViewStream {

-      public LoaderStream(string fileName, int secondsBetweenPolls)
-      {
-         mySetRealSource(
-            new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-               ViewStreamDatabase.instance));
+      private string _filename;
+      private IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
+
+
+      public LoaderStream(string filename) {
+         _filename = filename;
      }

-      public LoaderStream(string fileName)
-      {
-        if (fileName == null)
-           throw new LoaderStream.Exception("File name cannot be null");
-
-        for (int i = 0; i < 5; i++)
-           try
-           {
-              using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-              {
-                  BinaryFormatter bf = new BinaryFormatter();
-#pragma warning disable SYSLIB0011
-                  BlueTable bt = (BlueTable)bf.Deserialize(fs);
-#pragma warning restore SYSLIB0011
-                  mySetRealSource(new BlueRowCollectionStaticStream(bt));
-              }
-              break;
-           }
-           catch (IOException)
-           {
-              System.Threading.Thread.Sleep(50);
-           }
-
+      protected override void myOnFirstSubscriber() {
+         byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename);
+         StreamIOMessage msg = _deser.deserialize(ser_msg);
+         Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE);
+         IBlueRowCollection rows = msg.create_message.rows;
+         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+         myFireEvent(dp);
      }
   }
 }
@@ -5,6 +5,7 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
+using bts;

 namespace BCG {
   // Writes all the udpates to a stream in a file.
@@ -24,7 +25,7 @@ namespace BCG {
      private ViewStream _source_stream;
      private string _token;
      private FileStream _out_fs;
-      private IBlueDataSerializer _serializer;
+      private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer();
      private string _file_name = String.Empty;
      private long _rollover_size;
      private readonly string _directory_path;
@@ -94,13 +95,12 @@ namespace BCG {
         DataPacket dp = new_value as DataPacket;
         byte[] serialized_update = null;
         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
-         _serializer = _serializer ?? new UniversalSerializer(dp.schema);
+         _serializer = _serializer ?? new UniversalStreamIOSerializer();
         if (_write_create) {
            // new files always starts with create
-            _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
+            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
         } else
-            _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
-
+            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
         BinaryWriter bw = new BinaryWriter(_out_fs);
         long utc_epoch = epoch_now();

@@ -1,68 +1,68 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;

-namespace BCG
-{
-   public class RecorderStream : ViewStream
-   {
-      private ViewStream mySource;
-      private DateTime myStartTime;
-      private FileStream myOutFile;
-      private string myFileName;
-      private IBlueDataSerializer mySerializer;
-            
-      public RecorderStream(ViewStream src, string recordFile)
-      {
-         mySource = src;
-         myFileName = recordFile;
+// namespace BCG
+// {
+//    public class RecorderStream : ViewStream
+//    {
+//       private ViewStream mySource;
+//       private DateTime myStartTime;
+//       private FileStream myOutFile;
+//       private string myFileName;
+//       private IBlueDataSerializer mySerializer;

-         myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-      }
+//       public RecorderStream(ViewStream src, string recordFile)
+//       {
+//          mySource = src;
+//          myFileName = recordFile;

-      protected override void myOnFirstSubscriber()
-      {
-         myStartTime = DateTime.Now;         
-         mySource.onChange += new ChangeDelegate(mySource_onChange);
-         myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-      }
-      
-      protected override void myOnLastSubscriber()
-      {
-         mySource.onChange -= mySource_onChange;
-         myOutFile.Close();
-      }
+//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
+//       }

-      void mySource_onChange(object newValue)
-      {
-         DataPacket dp = (DataPacket)newValue;         
-         
-         BlueTable toWrite = new BlueTable(dp.schema);
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStartTime = DateTime.Now;         
+//          mySource.onChange += new ChangeDelegate(mySource_onChange);
+//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
+//       }

-         if (mySerializer == null)
-         {
-            mySerializer = new UniversalSerializer(dp.schema);
-         }
-         foreach (IBlueRow r in dp.newRows)
-            toWrite.addRow(r);
-         
-         byte[] result;
+//       protected override void myOnLastSubscriber()
+//       {
+//          mySource.onChange -= mySource_onChange;
+//          myOutFile.Close();
+//       }

-         if (myLastValue == null)
-            mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-         else
-            mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+//       void mySource_onChange(object newValue)
+//       {
+//          DataPacket dp = (DataPacket)newValue;         

-         BinaryWriter bw = new BinaryWriter(myOutFile);
-         //write down what time this happened
-         bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-         bw.Write(result.Length);
-         bw.Write(result);         
-         myOutFile.Flush();
+//          BlueTable toWrite = new BlueTable(dp.schema);

-         myFireEvent(dp);
-      }
-   }
-}
+//          if (mySerializer == null)
+//          {
+//             mySerializer = new UniversalSerializer(dp.schema);
+//          }
+//          foreach (IBlueRow r in dp.newRows)
+//             toWrite.addRow(r);
+
+//          byte[] result;
+
+//          if (myLastValue == null)
+//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
+//          else
+//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+
+//          BinaryWriter bw = new BinaryWriter(myOutFile);
+//          //write down what time this happened
+//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
+//          bw.Write(result.Length);
+//          bw.Write(result);         
+//          myOutFile.Flush();
+
+//          myFireEvent(dp);
+//       }
+//    }
+// }
@@ -1,64 +1,37 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
+using BCG;

-// using System.Runtime.Serialization.Formatters.Binary;
+namespace bts {
+   // this saves the total state in the filr provided in the cons
+   [DataStreamUserRespresentation("saver")]
+   public class SaverStream : ViewStream {
+      private ViewStream _stream;
+      private string _filename;
+      private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();

-// namespace BCG
-// {
-//    [DataStreamUserRespresentation("saver")]
-//    public class SaverStream : ViewStream, ISideEffectStream
-//    {
-//       private ViewStream myStream;
-//       private string myFileName;
+      public SaverStream(ViewStream vs, string filename) {
+         _stream = vs;
+         _filename = filename;
+      }

-//       public class Exception : System.Exception
-//       {
-//          public Exception(string msg) : base(msg) { }
-//       }
+      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }

-//       public SaverStream(ViewStream vs, string fileName)
-//       {
-//          if (fileName == null)
-//             throw new SaverStream.Exception("File name cannot be null");
-//          myStream = vs;
-//          myFileName = fileName;
+      void _handle_changes(object dp_obj) {
+         DataPacket dp = dp_obj as DataPacket;
+         _save(dp);
+         myFireEvent(dp_obj);
+      }

-//       }
+      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }

-//       protected override void myOnFirstSubscriber()
-//       { myStream.onChange += new ChangeDelegate(myHandleChanges); }
+      protected void _save(DataPacket dp) {
+         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
+         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
+         StreamSerializationUtils.write_to_file(str_bytes, _filename);

-//       void myHandleChanges(object newValue)
-//       {
-//          DataPacket dp = newValue as DataPacket;
-//          mySave(dp);
-//          myFireEvent(newValue);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
-
-//       protected void mySave(DataPacket dp)
-//       {
-//          for (int i = 0; i < 5 ; i++)
-//             try
-//             {
-//                using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
-//                {
-//                   BinaryFormatter bf = new BinaryFormatter();
-//                   BlueTable t = new BlueTable(dp.schema);
-//                   foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
-//                      t.addRow(new BlueRow(r));
-//                   bf.Serialize(fs, t);
-//                }
-//                break;
-//             }
-//             catch (IOException)
-//             {
-//                System.Threading.Thread.Sleep(5);
-//             }
-//       }
-//    }
-// }
+      }
+   }
+}
@@ -1,12 +1,12 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-using System.Runtime.Serialization.Formatters.Binary;
-using BCG.Utilities.General;
-using bts.collections;
-using System.Reflection.Metadata;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+// using System.Runtime.Serialization.Formatters.Binary;
+// using BCG.Utilities.General;
+// using bts.collections;
+// using System.Reflection.Metadata;


 namespace BCG {
@@ -15,115 +15,115 @@ namespace BCG {
         FirstOfDay,
         LastOfDay
      };
-      private BlueTable _current_table;
-      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-      public DateTime _snapshot_time;
-      public string _snapshot_file;
-      public IEnumerable<PlaybackPacket> _packets;
-      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
-         DateTime dt;
-         var parsed = DateTime.TryParse(snapshot_date, out dt);
-         dt = dt.Date;
-         bts.Log.assert(parsed);
-         SpecificSnapshot specific_snapshot_enum;
-         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
-         bts.Log.assert(parsed);
-         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
-         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
-         mySetRealSource(rs);
-      }
+//       private BlueTable _current_table;
+//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+//       public DateTime _snapshot_time;
+//       public string _snapshot_file;
+//       public IEnumerable<PlaybackPacket> _packets;
+//       public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
+//          DateTime dt;
+//          var parsed = DateTime.TryParse(snapshot_date, out dt);
+//          dt = dt.Date;
+//          bts.Log.assert(parsed);
+//          SpecificSnapshot specific_snapshot_enum;
+//          parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
+//          bts.Log.assert(parsed);
+//          string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
+//          ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
+//          mySetRealSource(rs);
+//       }

-      public SnapshotStream(string snapshot_dt, string directory_name, string token) {
-         _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
-         string token_directory_name = $"{directory_name}\\{token}";
-         _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
-      }
+//       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
+//          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
+//          string token_directory_name = $"{directory_name}\\{token}";
+//          _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
+//       }

-      protected override void myOnFirstSubscriber() {
-         FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
-         _packets = StreamPersistorUtils.load_packets_from_file(fs);
-         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
-         if (snapshot_packet != null) {
-            foreach (PlaybackPacket p in _packets) {
-               byte[] ops;
-               Array row_array;
-               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-               for (int i = 0; i < ops.Length; i++) {
-                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-                  byte op = ops[i];
-                  if (op == StreamOpCode.create) {
-                     IBlueRowCollection full_table = rows_args[0];
-                     _current_table = new BlueTable(full_table);
-                  } else if (op == StreamOpCode.multipleRemoveAdd) {
-                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
-                  } else {
-                     bts.Log.assert(false, $"Oprator: {op} is not expected");
-                  }
-                  if (p.idx == snapshot_packet.idx) {
-                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-                     break;
-                  }
-               }
-            }
-         }
-      }
+//       protected override void myOnFirstSubscriber() {
+//          FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
+//          _packets = StreamPersistorUtils.load_packets_from_file(fs);
+//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
+//          if (snapshot_packet != null) {
+//             foreach (PlaybackPacket p in _packets) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+//                for (int i = 0; i < ops.Length; i++) {
+//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//                   byte op = ops[i];
+//                   if (op == StreamOpCode.create) {
+//                      IBlueRowCollection full_table = rows_args[0];
+//                      _current_table = new BlueTable(full_table);
+//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                   } else {
+//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
+//                   }
+//                   if (p.idx == snapshot_packet.idx) {
+//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+//                      break;
+//                   }
+//                }
+//             }
+//          }
+//       }

-      private void _absorb_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-            } else {
-               bts.Log.assert(false, $"Operator: {op} is not expected");
-            }
-         }
-      }
+//       private void _absorb_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//             } else {
+//                bts.Log.assert(false, $"Operator: {op} is not expected");
+//             }
+//          }
+//       }

-      protected override void myOnLastSubscriber() {
-      }
+//       protected override void myOnLastSubscriber() {
+//       }

-      // public event SnapshotStream.Progress onProgress;
-      // public event Action onFileExhausted;
-      private PlaybackPacket _locate_packet_to_replay() {
-         foreach (var p in _packets) {
-            // returns the first non replayed packet
-            if (!p.replayed)
-               return p;
-         }
-         return null;
-      }
+//       // public event SnapshotStream.Progress onProgress;
+//       // public event Action onFileExhausted;
+//       private PlaybackPacket _locate_packet_to_replay() {
+//          foreach (var p in _packets) {
+//             // returns the first non replayed packet
+//             if (!p.replayed)
+//                return p;
+//          }
+//          return null;
+//       }

-      private void _fire_packet(byte[] ops, Array row_array) {
-         for (int i = 0; i < ops.Length; i++) {
-            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-            byte op = ops[i];
-            if (op == StreamOpCode.create) {
-               IBlueRowCollection full_table = rows_args[0];
-               _current_table = new BlueTable(full_table);
-               //_current_table.addRows(full_table);
-               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-                                          (myLastValue as DataPacket).newRows));
-            } else if (op == StreamOpCode.multipleRemoveAdd) {
-               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-               BlueDataUtility.absorbChanges(_current_table, dels, adds);
-               myFireEvent(new DataPacket(_current_table, adds, dels));
-            } else {
-               bts.Log.assert(false, $"Oprator: {op} is not expected");
-            }
+//       private void _fire_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//                //_current_table.addRows(full_table);
+//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+//                                           (myLastValue as DataPacket).newRows));
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                myFireEvent(new DataPacket(_current_table, adds, dels));
+//             } else {
+//                bts.Log.assert(false, $"Oprator: {op} is not expected");
+//             }

-         }
-      }
+//          }
+//       }
   }
-}
+}
@@ -1,16 +1,10 @@
-using BCG;
-using System;
-using System.Data;
-using System.Collections;
-using BCG.Utilities;
-using System.Text.RegularExpressions;
-
 using Bts.DataStream;

-using mbts;
-using mbts.spark;
 using mbts.config;

+using System;
+using System.Text.RegularExpressions;
+
 namespace BCG.StreamWorld.Addition
 {
   [DataStreamUserRespresentation (false)]
@@ -151,7 +145,6 @@ namespace BCG
      private string myFilterString;
      private BCG.StreamWorld.Addition.FilterStream myFilterStream;
      private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
-      private HighPerformanceTimer _timer = new HighPerformanceTimer();

      #region constructors
      public delegate bool OnFilter(IBlueRow r);
@@ -226,7 +219,6 @@ namespace BCG

      private void myHandleCallbacks(object newValue)
      {
-         _timer.start();
         DataPacket retVal;
         DataPacket arrivedPacket = newValue as DataPacket;
         myLastReceivedRows = arrivedPacket.newRows;
@@ -255,7 +247,6 @@ namespace BCG
               if (myFilter.filter(r))
                  myLastFiredRows.addRow(r);
         }
-         _timer.stop();
         myFireEvent(retVal);
      }

@@ -1,37 +1,66 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+using System.Text.RegularExpressions;

 using BCG.Utilities.General;

-namespace BCG
-{
-   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
-   {
+namespace BCG {
+   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable {
+
+      public static TimeSpan parse_duration(string input) {
+         int days = 0, hours = 0, minutes = 0, seconds = 0;
+
+         // Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
+         Regex regex = new Regex(@"(\d+)([dhms])");
+         MatchCollection matches = regex.Matches(input);
+
+         foreach (Match match in matches) {
+            int value = int.Parse(match.Groups[1].Value);
+            string unit = match.Groups[2].Value;
+
+            switch (unit) {
+               case "d":
+                  days = value;
+                  break;
+               case "h":
+                  hours = value;
+                  break;
+               case "m":
+                  minutes = value;
+                  break;
+               case "s":
+                  seconds = value;
+                  break;
+            }
+         }
+
+         return new TimeSpan(days, hours, minutes, seconds);
+      }
+
      #region constructors
+
+      public TimeBasedFilter(string col_name, string row_time_to_live)
+      : this(col_name, parse_duration(row_time_to_live)) {
+
+      }
+
      public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
-         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
-      {
+      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
      }

      public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
-      {
+      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) {
      }

      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-         : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
-      {
+      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
      }

-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-      {
-         myVsd               = vsd;
-         myRowTimeColumnName = rowTimeColumnName;
-         myRowTimeToLive     = rowTimeToLive;
-         myFilterFrequency   = filterFrequency;
-         myNextFilter        = DateTime.MinValue;
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
+         myVsd = vsd;
+         _time_col = rowTimeColumnName;
+         _ttl = rowTimeToLive;
+         myFilterFrequency = filterFrequency;
+         myNextFilter = DateTime.MinValue;

         // add worker
         myVsd.addWorker(this);
@@ -41,32 +70,25 @@ namespace BCG
      public event FilterChangeDelegate onFilterChange;

      #region IStreamFilter
-      public bool filter(IBlueRow row)
-      {
-         DateTime now       = DateTime.Now;
-         DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
+      public bool filter(IBlueRow row) {
+         DateTime now = DateTime.Now;

-         if (now > timeStamp)
-         {
-            TimeSpan span = now - timeStamp;
-
-            if (span >= myRowTimeToLive)
-            {
+         DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
+         if (now > row_time) {
+            TimeSpan span = now - row_time;
+            if (span >= _ttl) {
               return (false);
            }
         }
-
         return (true);
      }
      #endregion

      #region IWorker
-      public void doWork()
-      {
+      public void doWork() {
         DateTime now = DateTime.Now;

-         if (now > myNextFilter)
-         {
+         if (now > myNextFilter) {
            // fire event
            onFilterChange();

@@ -76,8 +98,7 @@ namespace BCG
      #endregion

      #region IDisposable
-      public void Dispose()
-      {
+      public void Dispose() {
         // remove worker
         myVsd.removeWorker(this);
      }
@@ -85,10 +106,20 @@ namespace BCG

      #region state
      private ViewStreamDatabase myVsd;
-      private string             myRowTimeColumnName;
-      private TimeSpan           myRowTimeToLive;
-      private TimeSpan           myFilterFrequency;
-      private DateTime           myNextFilter;
+      private string _time_col;
+      private TimeSpan _ttl;
+      private TimeSpan myFilterFrequency;
+      private DateTime myNextFilter;
      #endregion
   }
+   [DataStreamUserRespresentation("timefilter")]
+   public class TimeBasedFilterStream : ViewStream {
+      public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
+      :
+      this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
+      public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
+         TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
+         mySetRealSource(new FilterStream(base_stream, filter));
+      }
+   }
 }
@@ -4,18 +4,14 @@ using BCG.Utilities;
 using System.Collections.Generic;
 using System.Linq;

-namespace BCG
-{
-   public class GroupByStream : MethodDrivenStream
-   {
-      private class MyFuncGroupByEvaluator : IGroupByEvaluator
-      {
+namespace BCG {
+   public class GroupByStream : MethodDrivenStream {
+      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
         private string[] _groupby_cols;
         private Func<DataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
-         {
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
            _groupby_cols = groupby_cols.ToArray();
-            _groupby_func = groupby_func;            
+            _groupby_func = groupby_func;
         }
         public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }

@@ -29,7 +25,7 @@ namespace BCG
      /// Serves as  a quick way to access all rows belonging to the same group
      /// </summary>
      private bool myIdGuard = true;
-      private RowDictionaryTable myBookKeepingTable ;
+      private RowDictionaryTable myBookKeepingTable;
      private IGroupByEvaluator myGroupByEvaluator;
      private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
      //can map a schema to another
@@ -37,112 +33,98 @@ namespace BCG
      private int myIdGenerator = 0;

      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
-         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
-      { }
+         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }

-      public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
-         : this (stream, groupByEvaluator, true)
-      {}
+      public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
+         : this(stream, groupByEvaluator, true) { }

-      public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
-         : base (stream)
-      {
+      public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
+         : base(stream) {
         myGroupByEvaluator = groupbyEvaluator;
         myIdGuard = idGuard;
      }

-      protected override void myHandleFirstCall ()
-      {
+      protected override void myHandleFirstCall() {
         mySchema = myLastArrivedPacket.schema;
-         myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
-         myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows, 
+         myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
+         myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
            myGroupByEvaluator.groupByColumns);
-         GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
-         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
-         {
-            IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
-               rowGroup, new BlueTable (mySchema)));
+         GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
+         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
+            IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
+               rowGroup, new BlueTable(mySchema)));
            if (newRow != null)
-               temp.addRow (newRow, myGetGroupString (rowGroup));
+               temp.addRow(newRow, myGetGroupString(rowGroup));
         }
-         myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
+         myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema));
      }

      /// <summary>
      /// Adding need not to anything as deletion already takes care
      /// of addition
      /// </summary>
-      protected override void myHandleIncrementalAdd()   {}
-      
+      protected override void myHandleIncrementalAdd() { }
+
      /// <summary>
      /// Both deletes and adds are handled here to make the intermediate results not
      /// appear in the spitted fired DataPacket (note this is done only in GroupByStream 
      /// because of the nature of it)
      /// </summary>
-      protected override void myHandleIncrementalDelete()   
-      {
+      protected override void myHandleIncrementalDelete() {
         myFireSwitch = false;
         RowDictionaryTable delGroups, addGroups;
         //form groups for deleted and added rows
-         myInitIncrementalChangeData (out delGroups, out addGroups);
-         ArrayList handledAddGroups = new ArrayList ();
+         myInitIncrementalChangeData(out delGroups, out addGroups);
+         ArrayList handledAddGroups = new ArrayList();
         string groupRepString;

-         foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
-         {
+         foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
            // BlueEventLog.logAssert (delGroup.Count > 0);
-            groupRepString = myGetGroupString (delGroup);
-            IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
-               new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
-            handledAddGroups.Add (addGroup);
-            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
-               new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
-            IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
-            myFireSwitch |= myMergeRow (newRow, groupRepString);           
+            groupRepString = myGetGroupString(delGroup);
+            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
+            handledAddGroups.Add(addGroup);
+            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
+               new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
+            IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
+            myFireSwitch |= myMergeRow(newRow, groupRepString);
         }

-         foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
-         {
+         foreach (IBlueRowCollection addGroup in addGroups.getGroups()) {
            // BlueEventLog.logAssert (addGroup.Count > 0);
-            if (!handledAddGroups.Contains (addGroup))
-            {
+            if (!handledAddGroups.Contains(addGroup)) {
               //this is the case when there are rows added to a group and none
               //have been removed, hence it 'escapes' from being dealt with in 
               //the loop above

-               groupRepString = myGetGroupString (addGroup);
-               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
+               groupRepString = myGetGroupString(addGroup);
+               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
               // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
-               IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, 
-                  new BlueTable (aggGroup.schema)));
-               myFireSwitch |= myMergeRow (newRow, groupRepString);
+               IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
+                  new BlueTable(aggGroup.schema)));
+               myFireSwitch |= myMergeRow(newRow, groupRepString);
            }
         }
      }

-      private bool myMergeRow (IBlueRow newRow, string groupRepString)
-      {
-         IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
+      private bool myMergeRow(IBlueRow newRow, string groupRepString) {
+         IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
         //BlueEventLog.logAssert (newRow != null || existingRow != null);
         if (existingRow == null && newRow == null)
            return false;
-         else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
+         else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
            return false;
-         else if (existingRow == null)
-         {
+         else if (existingRow == null) {
            //row added for the first time
-            ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
-            ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
+            ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
+            ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
            return true;
-         }
-         else
-         {
-            ((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
-            ((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
-            if (newRow != null)
-            {
-               ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
-               ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
+         } else {
+            ((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString);
+            ((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow);
+            if (newRow != null) {
+               ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
+               ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
            }
            return true;
         }
@@ -154,77 +136,68 @@ namespace BCG
      /// </summary>
      /// <param name="delGroups"></param>
      /// <param name="addGroups"></param>
-      private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
-      {
+      private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
         delGroups =
-            new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
         addGroups =
-            new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
         foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
-            myBookKeepingTable.removeRow (deletedRow);
+            myBookKeepingTable.removeRow(deletedRow);
         foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
-            myBookKeepingTable.addRow (addedRow);
-         myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
-         myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
+            myBookKeepingTable.addRow(addedRow);
+         myLastFiredPacket.addedRows = new BlueTable(myOutputSchema);
+         myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema);
      }

-      protected override void myOnTableClear ()
-      {
+      protected override void myOnTableClear() {
         if (myBookKeepingTable.Count == 0)
            return;  //ie there arent any rows, so no need to optimize the 'cleansing' process
-         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
-         (myLastFiredPacket.newRows as GroupByOutputTable).clear ();
-         myBookKeepingTable.clear ();
-         myGroupByEvaluator.clear ();
+         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
+         (myLastFiredPacket.newRows as GroupByOutputTable).clear();
+         myBookKeepingTable.clear();
+         myGroupByEvaluator.clear();
      }

-      private IBlueRow myEvaluate (DataPacket dp)
-      {
-         IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
-         return evalRow == null ? null : 
+      private IBlueRow myEvaluate(DataPacket dp) {
+         IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
+         return evalRow == null ? null :
            myIdGuard ?
-            BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
+            BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow;
      }

-      private string myGetGroupString (IBlueRowCollection group)
-      {
-         return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
+      private string myGetGroupString(IBlueRowCollection group) {
+         return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns);
      }

-      protected override  void myOnLastSubscriber () 
-      {
+      protected override void myOnLastSubscriber() {
         //the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream 
         //gets resubscribed, we start the process of building data again, and treat the callback as the first 
         //callback.
-         myInit ();
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+         myInit();
+         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
      }

-      protected override void myInit ()
-      {
-         base.myInit ();
-         if (myBookKeepingTable != null)
-         {
+      protected override void myInit() {
+         base.myInit();
+         if (myBookKeepingTable != null) {
            //to avoid calling init before the first callback
-            myBookKeepingTable.clear ();
-            myGroupByEvaluator.clear ();
+            myBookKeepingTable.clear();
+            myGroupByEvaluator.clear();
         }
      }

-      public static string getRepString (DataPacket dp, string[] colNames)
-      {
-         return BlueDataUtility.rowToString (getRepRow (dp), colNames);
+      public static string getRepString(DataPacket dp, string[] colNames) {
+         return BlueDataUtility.rowToString(getRepRow(dp), colNames);
      }

-      public static IBlueRow getRepRow (DataPacket dp)
-      {
+      public static IBlueRow getRepRow(DataPacket dp) {
         if (dp.newRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.newRows, 0);
+            return BlueDataUtility.extractRow(dp.newRows, 0);
         else if (dp.addedRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.addedRows, 0);
+            return BlueDataUtility.extractRow(dp.addedRows, 0);
         else if (dp.deletedRows.Count > 0)
-            return BlueDataUtility.extractRow (dp.deletedRows, 0);
+            return BlueDataUtility.extractRow(dp.deletedRows, 0);
         // BlueEventLog.logAssert (false, "Illegal Data Packet");
         return null;
      }
@@ -237,22 +210,21 @@ namespace BCG
      /// <param name="newValue"></param>
      /// <param name="isAdd"></param>
      /// <returns></returns>
-      public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
-      {
-         if (lastValue == null || lastValue.Equals (DBNull.Value))
+      public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
+         if (lastValue == null || lastValue.Equals(DBNull.Value))
            return newValue;
         // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
-            // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
-            // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
+         // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
+         // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
         Object retVal = null;
-         if (newValue.GetType().Equals (typeof(int)))
+         if (newValue.GetType().Equals(typeof(int)))
            retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
-         else if (newValue.GetType().Equals (typeof(Double)))
+         else if (newValue.GetType().Equals(typeof(Double)))
            retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
-         else if (lastValue.GetType().Equals (typeof(Decimal)))
+         else if (lastValue.GetType().Equals(typeof(Decimal)))
            retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
-         else{}
-            // BlueEventLog.logAssert (false, "Illegal type");
+         else { }
+         // BlueEventLog.logAssert (false, "Illegal type");
         return retVal;
      }

@@ -22,8 +22,8 @@ namespace BCG {
            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
            if (maxRow != null) {
                bts.Log.assert(currMaxRow != null);
-                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
-                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
+                bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0);
+                if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) {
                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
                    if (maxRow == null)
                        myOutputTable.removeRow(hashString);
@@ -34,7 +34,7 @@ namespace BCG {
            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
            // otherwise if maxRow > currMaxRow then its replaced
-            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
+            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0))
                myOutputTable.addRow(maxRow, hashString);
            return myProject(myOutputTable.getRow(hashString));
        }
@@ -18,7 +18,7 @@ namespace BCG {
            if (minRow != null) {
                // BlueEventLog.logAssert (currMinRow != null);
                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-                if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
+                if (BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) == 0) {
                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
                    if (minRow == null)
                        myOutputTable.removeRow(stringRep);
@@ -27,7 +27,7 @@ namespace BCG {
                }
            }
            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
-            if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
+            if (minRow != null && (currMinRow == null || BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) < 0))
                myOutputTable.addRow(minRow, stringRep);
            return myOutputTable.getRow(stringRep);
        }
@@ -1,10 +1,8 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using mbts;
-using mbts.spark;
+using System.Diagnostics;
+
+using bts;

 namespace BCG
 {
@@ -14,7 +12,7 @@ namespace BCG
      private INonIncGroupByEvaluator _groupby_func;
      private BlueRowHasher _row_hasher;
      private string[] _groupby_cols;
-      private HighPerformanceTimer _timer = new HighPerformanceTimer();
+      private Stopwatch _timer = new();
      private int _n_total_fires = 0, _n_expensive_fires = 0;

      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
@@ -33,21 +31,27 @@ namespace BCG
      {
         DataPacket dp = nv as DataPacket;
         ///grouping together
-         _timer.start();
+         _timer.Restart();
         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.stop();
-         // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
+         _timer.Stop();
+         var groupElapsedMs = _timer.ElapsedMilliseconds;
+         if (groupElapsedMs >= 50) {
+            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
+         }
+         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
         /// reducing
-         _timer.start();
+         _timer.Restart();
         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
-         _timer.stop();
+         _timer.Stop();
+         var applyElapsedMs = _timer.ElapsedMilliseconds;
         _n_total_fires++;         
-         if (_timer.lastIntervalTime > 50)
+         if (applyElapsedMs >= 50)
         {
            _n_expensive_fires++;
-            // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
+            Log.DataFlow.info(
+               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
         }         
         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
-      }      
+      }
   }
 }
@@ -27,7 +27,8 @@ namespace BCG
            string prependString = isLeft ? "left" : "right";
            return String.Concat (prependString, colName).GetHashCode ();
         }
-         public override bool Equals (object obj)  {return GetHashCode () == obj.GetHashCode ();}
+         public override bool Equals (object? obj)  =>
+            GetHashCode() == obj?.GetHashCode();
      }


@@ -10,9 +10,9 @@ namespace BCG
 {
   public class JoinStream : ViewStream
   {
-      protected InternalJoinStream myIjs;
-      protected ProjectStream myPs;
-      protected ColumnMap myLeftCm, myRightCm;
+      protected InternalJoinStream? myIjs;
+      protected ProjectStream? myPs;
+      protected ColumnMap? myLeftCm, myRightCm;
      protected bool myIsProject = true;
      private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
      
@@ -1,10 +1,10 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using bts;
+
 using mbts;
-using mbts.spark;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;

 namespace BCG
 {
@@ -40,8 +40,7 @@ namespace BCG

      public void _on_stream_cb(object dp_obj)
      {
-         HighPerformanceTimer timer = new HighPerformanceTimer();
-         timer.start();
+         var timer = Stopwatch.StartNew();
         DataPacket dp = (DataPacket)dp_obj;
         if (_schema == null)
         {
@@ -67,13 +66,12 @@ namespace BCG
                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
            }
         }
-         timer.stop();
+         timer.Stop();
         _n_total_fires++;
-         if (timer.lastIntervalTime > 50)
-         {
+         if (timer.ElapsedMilliseconds >= 50) {
            _n_expensive_fires++;
-            string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
-            // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
+            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
+               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
         }
         if (on_new_data != null)
            on_new_data();
@@ -1,10 +1,10 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using bts;
+
 using mbts;
-using mbts.spark;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;

 namespace BCG
 {
@@ -114,8 +114,7 @@ namespace BCG
         if (_is_ready())
         {
            _construct_schema();
-            HighPerformanceTimer timer = new HighPerformanceTimer();
-            timer.start();
+            var timer = Stopwatch.StartNew();
            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
            List<object[]> output_rows = new List<object[]>();

@@ -138,13 +137,12 @@ namespace BCG
               }
            }
            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
-            timer.stop();
+            timer.Stop();
            _n_total_fires++;
-            if (timer.lastIntervalTime >= 250)
-            {
+            if (timer.ElapsedMilliseconds >= 250) {
               _n_expensive_fires++;
-               string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
-               // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
+               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
            }
            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
         }
@@ -154,15 +152,12 @@ namespace BCG
      {
         if (_output_mapper != null)
         {
-            HighPerformanceTimer timer = new HighPerformanceTimer();
            IBlueRowCollection output_rows;
-            timer.start();
+            var timer = Stopwatch.StartNew();
            output_rows = _output_mapper.map_rows(unmapped_rows);
-            timer.stop();
-            if (timer.lastIntervalTime >= 50)
-            {
-               // string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
-               // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
+            timer.Stop();
+            if (timer.ElapsedMilliseconds >= 50) {
+               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
            }
            return output_rows;
         }
@@ -1,9 +1,7 @@
-using System;
+#nullable disable
+using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using BCG.Utilities;

 namespace BCG
 {
@@ -1,11 +1,7 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities.General;
-using BCG.Utilities;
 using mbts;
-using mbts.spark;
 using Spark.Core.Collections;

 namespace BCG
@@ -226,7 +222,8 @@ namespace BCG

      #region IDictionary Members

-      public void Add(object key, object value) { { throw new NotSupportedException("IBlueRow is readonly"); } }
+      public void Add(object key, object? value) =>
+         throw new NotSupportedException("IBlueRow is readonly");

      public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }

@@ -256,10 +253,9 @@ namespace BCG
         }
      }

-      public object this[object key]
-      {
-         get { return key is string ? this[key.ToString()] : this[(int)key]; }
-         set { throw new NotSupportedException("IBlueRow is readonly"); }
+      public object? this[object key] {
+         get => key is string str ? this[str] : this[(int)key];
+         set => throw new NotSupportedException("IBlueRow is readonly");
      }

      #endregion
@@ -294,7 +290,6 @@ namespace BCG
      protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
                                          myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
      private bool myManageUnmappedLeftRows = false;
-      protected HighPerformanceTimer myTimer = new HighPerformanceTimer();
      protected string myLeftProj = String.Empty, myRightProj = String.Empty;
      protected IBlueTableSchema myLeftSchema, myRightSchema;
      protected JoinProjectHelper myJoinProjectHelper;
@@ -361,7 +356,6 @@ namespace BCG

      private void myHandleStreamChanges(DataPacket dp, bool isLeft)
      {
-         myTimer.start();
         myInitSchema(dp.schema, isLeft);
         BlueTable dels = null, adds = null;
         if (myState != null)
@@ -371,7 +365,6 @@ namespace BCG
         }
         myDeleteRows(dp.deletedRows, isLeft, myState, dels);
         myAddRows(dp.addedRows, isLeft, myState, adds);
-         myTimer.stop();
         if (myState != null)
         {
            myFireEvent(new DataPacket(myState, adds, dels));
@@ -91,7 +91,8 @@ namespace BCG
         }
      }

-      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}
+      public void Add (object key, object? value) =>
+         throw new NotSupportedException ("The IBlueRow is readonly");

      public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}

@@ -1,51 +1,64 @@
 using System;
 using System.Collections;
+using System.Collections.Generic;
+
 using BCG;
 using BCG.Utilities;

-namespace BCG
-{
-   public class LiteralStream : ViewStream
-   {
-      public LiteralStream(string tableName, string schema, string value)
-      {
-         string [] columns = schema.Split(' ');
+namespace BCG {
+   public class LiteralStream : ViewStream {
+
+      public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
+         List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
+         string[] pairs = key_value_pairs.Split(' ');
+         foreach (string p in pairs) {
+            string[] kvp = p.Split(':');
+            string col_name = kvp[0];
+            string val = kvp[1];
+            var tup = BlueDataUtility.guess_type(val);
+            var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
+            rv.Add(Tuple.Create(ci, tup.Item2));
+         }
+         return rv;
+      }
+
+      public LiteralStream(string tableName, string schema, string value) {
+         string[] columns = schema.Split(' ');


-         ColumnInfo [] ci = new ColumnInfo[columns.Length + 1];
-         ci[0] = new ColumnInfo("id",0,typeof(int));
+         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
+         ci[0] = new ColumnInfo("id", 0, typeof(int));
         int n = 1;
-         foreach (string c in columns)
-         {
-            string [] columnAndType = c.Split('/');
+         foreach (string c in columns) {
+            string[] columnAndType = c.Split('/');
            if (columnAndType.Length == 1)
-               ci[n] = new ColumnInfo(c,n,typeof(double));
+               ci[n] = new ColumnInfo(c, n, typeof(double));
            else
-               ci[n] = new ColumnInfo(columnAndType[0],n,Type.GetType("System." + columnAndType[1],true));
+               ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
            n++;
         }


-         BlueTable t = new BlueTable(new BlueTableSchema(tableName,ci));
-         string [] rows = value.Split(';');
+         BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
+         string[] rows = value.Split(';');
         n = 0;
-         foreach (string r in rows) 
-            try 
-            {
-               if(r == string.Empty)
+         foreach (string r in rows)
+            try {
+               if (r == string.Empty)
                  continue;
               t.addRow(
-                  BlueDataUtility.createRowFromCSVFileRow (r, t.schema,++n));   
-            }
-            catch (Exception e)
-            {
+                  BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
+            } catch (Exception e) {
               throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
            }

         mySetRealSource(new BlueRowCollectionStaticStream(t));
      }

-      public LiteralStream(string schema, string value) : this ("literal",schema, value)
-      {}
+      public LiteralStream(string schema, string value) : this("literal", schema, value) { }
+
+      // public LiteralStream(string value) {
+
+      // }
   }
 }
@@ -1,24 +1,24 @@
-using System;
+// using System;

-namespace BCG
-{
-   public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
-   {
-      IBlueTableSchema mySchema;
+// namespace BCG
+// {
+//    public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
+//    {
+//       IBlueTableSchema mySchema;

-      public IBlueRow map (IBlueRow row)
-      {
-         if (mySchema == null)
-         {
-            ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
-            int i = 0;
-            foreach (ColumnInfo ci in row.schema.columnInfo)
-               cis[i++] = ci.colType == null ?
-                  new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
-                  new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
-            mySchema = new BlueTableSchema (row.schema.tableName, cis);
-         }
-         return new BlueRow (mySchema, row);
-      }
-   }
-}
+//       public IBlueRow map (IBlueRow row)
+//       {
+//          if (mySchema == null)
+//          {
+//             ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
+//             int i = 0;
+//             foreach (ColumnInfo ci in row.schema.columnInfo)
+//                cis[i++] = ci.colType == null ?
+//                   new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
+//                   new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
+//             mySchema = new BlueTableSchema (row.schema.tableName, cis);
+//          }
+//          return new BlueRow (mySchema, row);
+//       }
+//    }
+// }
@@ -1,15 +1,14 @@
-using System;
-using System.Data;
-using System.Collections;
+using bts;

 using Bts.DataStream;
+using Bts.DataStreamCore;

-using mbts;
-using mbts.spark;
 using mbts.config;
+using mbts.utils;

-namespace BCG
-{
+using System;
+
+namespace BCG {


   /// <summary>
@@ -21,24 +20,45 @@ namespace BCG
   [DataStreamUserRespresentation (false)]
   public class MapStream : MethodDrivenStream
   {
-      private HighPerformanceTimer t = new HighPerformanceTimer ();
-      protected IStreamMapper myMapper;
-      protected IChangeableStreamMapper myChangeableMapper = null;
+      protected IStreamMapper? myMapper;
+      protected IChangeableStreamMapper? myChangeableMapper;
      private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;

-      public MapStream(ViewStream vs, IStreamMapper mapper) : base (vs)
-      {
-         if (myUseNewMap)
-            mySetRealSource (new SuperMapStream (vs, mapper));
-         else
-            myMapper = mapper;
+      public class FuncWrapper : IStreamMapper {
+         private Func<IBlueTableSchema, IBlueTableSchema> _schema_map_func;
+         Func<IBlueRow, IBlueRow> _row_map_func;
+
+         public FuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
+         Func<IBlueRow, IBlueRow> row_map_func) {
+            _schema_map_func = schema_map_func;
+            _row_map_func = row_map_func;
+         }
+         public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
+            return _schema_map_func(input_schema);
+         }
+         public IBlueRow map(IBlueRow row) {
+            return _row_map_func(row);
+         }
+
      }
-      public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this (vs,(IStreamMapper)mapper)
-      {
+
+      public MapStream(ViewStream vs, IStreamMapper mapper) : base(vs) {
         if (myUseNewMap)
            mySetRealSource(new SuperMapStream(vs, mapper));
         else
-         {
+            myMapper = mapper;
+      }
+
+      public MapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
+                                       Func<IBlueRow, IBlueRow> row_map_func)
+                                       :
+                                       this(vs, new FuncWrapper(schema_map_func, row_map_func)) {
+      }
+
+      public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this(vs, (IStreamMapper)mapper) {
+         if (myUseNewMap)
+            mySetRealSource(new SuperMapStream(vs, mapper));
+         else {
            mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
            myChangeableMapper = mapper;
         }
@@ -47,77 +67,74 @@ namespace BCG
      /// <summary>
      /// Gets or sets the value of the mapper. Setting the value will result in re
      /// </summary>
-      private void myMapperChanged () 
-      {
+      private void myMapperChanged() {
         if (myLastArrivedPacket != null)
-            myChangeHandler (new DataPacket (myLastArrivedPacket.newRows, 
+            myChangeHandler(new DataPacket(myLastArrivedPacket.newRows,
                                             myLastArrivedPacket.newRows,
                                             myLastArrivedPacket.newRows));
      }


-      protected override void myHandleFirstCall ()
-      {
-         
-         IBlueTableSchema outputSchema = myMapper.getOutputSchema (myLastArrivedPacket.newRows.schema);
-         myLastFiredPacket = new DataPacket ();
-         myLastFiredPacket.newRows = new BlueTable (outputSchema);
-         myLastFiredPacket.deletedRows = new BlueTable (outputSchema);
+      protected override void myHandleFirstCall() {
+         ArgumentNullException.ThrowIfNull(myMapper);
+         IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
+         myLastFiredPacket = new DataPacket();
+         myLastFiredPacket.newRows = new BlueTable(outputSchema);
+         myLastFiredPacket.deletedRows = new BlueTable(outputSchema);
         foreach (IBlueRow row in myLastArrivedPacket.newRows)
-            ((BlueTable)(myLastFiredPacket.newRows)).addRow (myMapper.map(row));
+            ((BlueTable)(myLastFiredPacket.newRows)).addRow(myMapper.map(row));
         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows);
      }

-      protected override void myHandleIncrementalAdd()
-      {
-         //t.start ();
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.addedRows)
-         {
-            IBlueRow newRow = myMapper.map (row);
-            (myLastFiredPacket.newRows as BlueTable).addRow (newRow);
-            (myLastFiredPacket.addedRows as BlueTable).addRow (newRow);
+      protected override void myHandleIncrementalAdd() {
+         if (myLastArrivedPacket.addedRows.Count == 0) {
+            Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
+            return;
         }
-         //t.stop ();
-         //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
-         //System.Diagnostics.Trace.WriteLine (" ");
-      }
-
-      protected override void myHandleIncrementalDelete()
-      {  
-         //t.start ();
-         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
-         foreach (IBlueRow row in myLastArrivedPacket.deletedRows)
-         {
-            IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
-            (myLastFiredPacket.deletedRows as BlueTable).addRow (delRow);
-            (myLastFiredPacket.newRows as BlueTable).deleteRow (row);
+         ArgumentNullException.ThrowIfNull(myMapper);
+         var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
+         foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
+            var newRow = myMapper.map(row);
+            newRows.addRow(newRow);
+            addedRows.addRow(newRow);
         }
-         //t.stop ();
-         //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
-      }
-     
-
-      protected override void myOnFirstSubscriber() 
-      {
-         myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
+         myLastFiredPacket.addedRows = addedRows;
      }

-      protected override  void myOnLastSubscriber () 
-      {
-         myInit ();
+      protected override void myHandleIncrementalDelete() {
+         if (myLastArrivedPacket.deletedRows.Count == 0) {
+            Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
+            return;
+         }
+         var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
+         var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
+         foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
+            var delRow = newRows[row.Id()];
+            deletedRows.addRow(delRow);
+            newRows.deleteRow(row);
+         }
+         myLastFiredPacket.deletedRows = deletedRows;
+      }
+
+
+      protected override void myOnFirstSubscriber() {
+         myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler);
+      }
+
+      protected override void myOnLastSubscriber() {
+         myInit();
         if (myStream != null)
-         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
+            myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
      }

-      protected override void myOnTableClear ()
-      {
+      protected override void myOnTableClear() {
         myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-         myLastFiredPacket.newRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
-         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
+         myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
+         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
      }

-      
+
   }

   /*
@@ -1,12 +1,11 @@
+using Bts.DataStreamCore;
+
+using mbts.utils;
+
 using System;
 using System.Collections.Generic;
-using System.Text;
-using BCG.Utilities;
-using BCG.Utilities.General;
-using mbts;

-namespace BCG
-{
+namespace BCG {
   /// <summary>
   /// SuperMap gives the user the ability
   /// to map one row to one or more rows
@@ -17,13 +16,11 @@ namespace BCG
   /// can chage itself and raise an event causing
   /// a remapping to happen
   /// </summary>
-   public class SuperMapStream : ViewStream
-   {
+   public class SuperMapStream : ViewStream {
      #region super mapper that wraps around stream mapper
-      private class MyMappingWrapper : ISuperStreamMapper
-      {
+      private class MyMappingWrapper : ISuperStreamMapper {
         private IStreamMapper myMapper;
-         
+
         public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; }

         #region ISuperStreamMapper Members
@@ -32,8 +29,7 @@ namespace BCG

         public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; }

-         public event MapperChangeDelegate onMapperChange
-         {
+         public event MapperChangeDelegate onMapperChange {
            add { }
            remove { }
         }
@@ -42,53 +38,46 @@ namespace BCG
      }
      #endregion

-      private class MyFuncWrapper : ISuperStreamMapper
-      {
+      private class MyFuncWrapper : ISuperStreamMapper {
         private Func<IBlueTableSchema, IBlueTableSchema> _get_schema;
         private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows;
         public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema,
-                              Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         {
+                              Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) {
            _get_schema = get_schema;
            _map_rows = map_rows;
         }

         public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }

-         public IBlueRow[] map(IBlueRow r)
-         {
+         public IBlueRow[] map(IBlueRow r) {
            IEnumerable<IBlueRow> rows = _map_rows(r);
            return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
         }

-         public event MapperChangeDelegate onMapperChange;
-         private void _fake() { onMapperChange(); }
+         public event MapperChangeDelegate? onMapperChange;
      }

      //maps the id of the underlying stream to the
      //rows of the resulting stream
      private Dictionary<int, List<IBlueRow>> myMap = new Dictionary<int, List<IBlueRow>>();
      private int myID = 0;
-      private BlueTable myRows, myBaseRows;
+      private BlueTable? myRows;
+      private BlueTable? myBaseRows;
      private ViewStream myStream;
      private ISuperStreamMapper myMapper;
      public bool generate_ids = true;

      public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema,
                                            Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         : this(vs, new MyFuncWrapper(get_schema, map_rows))
-      { }
+         : this(vs, new MyFuncWrapper(get_schema, map_rows)) { }

      public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
-         : this(vs, s => out_schema, map_rows)
-      { }
+         : this(vs, s => out_schema, map_rows) { }

      public SuperMapStream(ViewStream vs, IStreamMapper mapper)
-         : this(vs, new MyMappingWrapper(mapper))
-      { }
+         : this(vs, new MyMappingWrapper(mapper)) { }

-      public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper)
-      {
+      public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) {
         myStream = vs;
         myMapper = mapper;
         myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
@@ -96,62 +85,45 @@ namespace BCG

      protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }

-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
         myStream.onChange -= new ChangeDelegate(myHandleData);
         myRows = null;
         myBaseRows = null;
         myMap.Clear();
-         //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
      }

-      private void myHandleData(object newValue)
-      {
-         DataPacket idp = newValue as DataPacket;
-         //HighPerformanceTimer timer = new HighPerformanceTimer();
-         //timer.start();
-         DataPacket dp = newValue as DataPacket;
-         myRows = myRows ?? new BlueTable(myMapper.getOutputSchema(dp.schema));
-         myBaseRows = myBaseRows ?? new BlueTable(dp.schema);
+      private void myHandleData(object newValue) {
+         var dp = ExceptionAssert.Cast(newValue).To<DataPacket>();
+         myRows ??= new BlueTable(myMapper.getOutputSchema(dp.schema));
+         myBaseRows ??= new BlueTable(dp.schema);
         BlueDataUtility.absorbChanges(myBaseRows, dp.deletedRows, dp.addedRows);
-         BlueTable dels = new BlueTable(myRows.schema),
-                     adds = new BlueTable(myRows.schema);
+         var dels = new BlueTable(myRows.schema);
+         var adds = new BlueTable(myRows.schema);

-         foreach (IBlueRow r in dp.deletedRows)
-         {
-            List<IBlueRow> mappedRows = myMap[(int)r["id"]];
-            foreach (IBlueRow mr in mappedRows)
-            {
+         foreach (IBlueRow r in dp.deletedRows) {
+            var mappedRows = myMap[r.Id()];
+            foreach (IBlueRow mr in mappedRows) {
               dels.addRow(mr);
               myRows.deleteRow(mr);
            }
-            myMap.Remove((int)r["id"]);
+            myMap.Remove(r.Id());
         }
-         foreach (IBlueRow r in dp.addedRows)
-         {
+         foreach (IBlueRow r in dp.addedRows) {
            IBlueRow[] mappedRows = myMapper.map(r);
-            foreach (IBlueRow mr in mappedRows)
-            {
+            foreach (IBlueRow mr in mappedRows) {
               IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
-               if (!myMap.ContainsKey((int)r["id"]))
-               {
-                  myMap.Add((int)r["id"], new List<IBlueRow>());
-               }
-               myMap[(int)r["id"]].Add(idChangedMapRow);
+               myMap.GetOrAdd(r.Id(), _ => new()).Add(idChangedMapRow);
               myRows.addRow(idChangedMapRow);
               adds.addRow(idChangedMapRow);
            }
         }
-         //timer.stop();
-         // if (timer.lastIntervalTime > 20)
-         //    {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
-         // myFireEvent(new DataPacket(myRows, adds, dels));
+         myFireEvent(new DataPacket(myRows, adds, dels));
      }

-      private void myHandleMapChange()
-      {
-         if (myBaseRows != null)
-            myHandleData(new DataPacket(myBaseRows, new BlueTable (myBaseRows), new BlueTable (myBaseRows)));
+      private void myHandleMapChange() {
+         if (myBaseRows is not null) {
+            myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
+         }
      }
   }
 }
				`@@ -0,0 +1 @@`
				<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>