Merge branch 'development.activ_md' of bitbucket.org:bluetradesys/spark into development.activ_md

active cme working small fixex and conf left
Add Activ md libs to sparkd
2024-04-09 12:07:41 -04:00 · 2024-03-19 14:17:52 -04:00 · 2024-03-19 14:17:50 -04:00 · 2024-03-19 14:07:11 -04:00 · 2024-03-19 13:59:41 -04:00 · 2024-03-19 13:58:48 -04:00
791 changed files with 24256 additions and 35315 deletions
@@ -1,123 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-# Note installations that take a while (more than 10s)
-
-RUN apt-get update
-RUN apt-get install -y ca-certificates
-RUN apt-get install -y ccache
-RUN apt-get install -y curl
-RUN apt-get install -y fish
-RUN apt-get install -y gdb
-# git-all: ~250s
-RUN apt-get install -y git-all
-RUN apt-get install -y gnupg
-# liblttng-ust-dev: ~30s
-RUN apt-get install -y liblttng-ust-dev
-RUN apt-get install -y libssh2-1-dev
-RUN apt-get install -y locales
-RUN apt-get install -y ninja-build
-# openssh-server: ~40s
-RUN apt-get install -y openssh-server
-RUN apt-get install -y pkg-config
-RUN apt-get install -y software-properties-common
-RUN apt-get install -y sudo
-RUN apt-get install -y tzdata
-RUN apt-get install -y vim
-RUN apt-get install -y zip
-
-# Python: ~30s
-RUN apt-get install -y \
-   python3-pip \
-   python3-venv \
-   && python3 -m pip install --upgrade pip
-
-# GCC
-ARG gcc_major=13
-RUN \
-   add-apt-repository ppa:ubuntu-toolchain-r/test \
-   && apt-get update \
-   && apt-get install -y g++-$gcc_major
-
-# Clang
-ARG clang_major=17
-RUN \
-   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
-   | tee -a "/etc/apt/sources.list.d/llvm.list" \
-   && apt-get update \
-   && apt-get install -y \
-   clang-$clang_major \
-   clang-format-$clang_major \
-   clangd-$clang_major
-
-# CMake
-RUN \
-   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
-   | gpg --dearmor - \
-   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
-   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
-   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
-   && apt-get update \
-   && apt-get install -y cmake
-
-# Dotnet
-RUN apt-get update && apt-get install -y dotnet7
-RUN apt-get update && apt-get install -y dotnet8
-
-# Node
-ARG node_major=18
-RUN mkdir -p /etc/apt/keyrings \
-   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
-   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
-   | tee /etc/apt/sources.list.d/nodesource.list \
-   && apt-get update \
-   && apt-get install -y nodejs
-
-# Emumba (deps)
-RUN apt-get update && apt-get -y install rapidjson-dev
-# Emumba
-COPY emumba /emumba
-RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
-    && rm -r /emumba
-
-# Pcap
-RUN apt-get update && apt-get install -y libpcap0.8-dev
-
-# Set up default executables (update-alternatives)
-RUN \
-   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
-   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
-   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
-   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
-   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
-   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
-
-# Add compiler symlinks for ccache 
-RUN \
-   ln -s /usr/bin/ccache /usr/local/bin/clang; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
-   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
-   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
-   ln -s /usr/bin/ccache /usr/local/bin/g++
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-   groupadd -g "$docker_uid" "$docker_user" \
-   && useradd \
-   -m \
-   -u "$docker_uid" \
-   -g "$docker_uid" \
-   -G sudo \
-   --password "$(openssl passwd -1 "$docker_pass")" \
-   "$docker_user"
@@ -1,58 +0,0 @@
-{
-  "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
-  "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
-  },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
-  "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
-  "updateRemoteUserUID": true,
-  "shutdownAction": "none",
-  // Note: If running multiple containers on the same host, each must use a
-  // unique set of ports. Usually we just add 1000 to each port as needed. For
-  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
-  "runArgs": [
-    "--shm-size",
-    "2gb",
-    "-p",
-    "5000:5000",
-    "-p",
-    "5001:5001",
-    "-p",
-    "5173:5173",
-    "-p",
-    "5174:5174",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003",
-    "-p",
-    "12345:12345",
-  ],
-  "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
-  ],
-  "customizations": {
-    "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
-      "extensions": [
-        "eamodio.gitlens",
-        "llvm-vs-code-extensions.vscode-clangd",
-        "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
-      ]
-    }
-  }
-}
@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
-sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
@@ -207,23 +207,17 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
 dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
 dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private

-# local symbols
-dotnet_naming_symbols.local_symbols.applicable_kinds = local
-
-# parameter symbols
-dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
-
 # Naming styles

-dotnet_naming_style.pascal_case.required_prefix =
-dotnet_naming_style.pascal_case.required_suffix =
-dotnet_naming_style.pascal_case.word_separator =
-dotnet_naming_style.pascal_case.capitalization = pascal_case
+dotnet_naming_style.snake_case.required_prefix =
+dotnet_naming_style.snake_case.required_suffix =
+dotnet_naming_style.snake_case.word_separator = _
+dotnet_naming_style.snake_case.capitalization = all_lower

-dotnet_naming_style.camel_case.required_prefix =
-dotnet_naming_style.camel_case.required_suffix =
-dotnet_naming_style.camel_case.word_separator =
-dotnet_naming_style.camel_case.capitalization = camel_case
+dotnet_naming_style.private_snake_case.required_prefix = _
+dotnet_naming_style.private_snake_case.required_suffix =
+dotnet_naming_style.private_snake_case.word_separator = _
+dotnet_naming_style.private_snake_case.capitalization = all_lower

 dotnet_naming_style.user_types.required_prefix =
 dotnet_naming_style.user_types.required_suffix =
@@ -235,10 +229,6 @@ dotnet_naming_style.interface.required_suffix =
 dotnet_naming_style.interface.word_separator =
 dotnet_naming_style.interface.capitalization = pascal_case

-dotnet_naming_style.local.capitalization = camel_case
-
-dotnet_naming_style.parameter.capitalization = camel_case
-
 # rules
 dotnet_naming_rule.user_types.symbols = user_types
 dotnet_naming_rule.user_types.style = user_types
@@ -248,18 +238,14 @@ dotnet_naming_rule.interfaces.symbols = interface
 dotnet_naming_rule.interfaces.style = interface_style
 dotnet_naming_rule.interfaces.severity = suggestion

-dotnet_naming_rule.public_members.symbols = public_symbols
-dotnet_naming_rule.public_members.style = pascal_case
-dotnet_naming_rule.public_members.severity = suggestion
+dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
+dotnet_naming_rule.public_members_snake_case.style = snake_case
+dotnet_naming_rule.public_members_snake_case.severity = suggestion

-dotnet_naming_rule.private_members.symbols = private_symbols
-dotnet_naming_rule.private_members.style = camel_case
-dotnet_naming_rule.private_members.severity = suggestion
+dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
+dotnet_naming_rule.private_members_snake_case.style = private_snake_case
+dotnet_naming_rule.private_members_snake_case.severity = suggestion

-dotnet_naming_rule.variables_are_camel_case.severity = suggestion
-dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
-dotnet_naming_rule.variables_are_camel_case.style = local
-
-dotnet_naming_rule.parameters.symbols = parameter_symbols
-dotnet_naming_rule.parameters.style = parameter
+dotnet_naming_rule.parameters.symbols = parameters
+dotnet_naming_rule.parameters.style = snake_case
 dotnet_naming_rule.parameters.severity = suggestion
@@ -1,3 +0,0 @@
-*.cmd text=auto eol=crlf
-*.cs text=auto eol=crlf
-*.sh text=auto eol=lf
@@ -8,9 +8,7 @@ cmake-build-debug/
 .idea/
 .dir-locals.el
 *.pyc
-**.vscode/*
-!.vscode/launch.json
-!.vscode/tasks.json
+**.vscode
 .vs
 obj/
 *.csproj.user
@@ -56,6 +54,6 @@ trace_data/
 /src/Spark.Admin.WebApp/src/bin/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
-.devcontainer/
+**/.devcontainer
 *.Generated.cs
 *.lst
@@ -1,6 +1,6 @@
 [submodule "externals"]
 	path = externals
-	url = https://bitbucket.org/bluetradesys/externals.git
+	url = git@bitbucket.org:bluetradesys/externals.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git
@@ -1,213 +0,0 @@
-{
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "name": ".NET Core Attach",
-      "type": "coreclr",
-      "request": "attach"
-    },
-    {
-      "name": "Bts.DataStreamWebApp.Server",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream WebApp (Debug)",
-      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
-      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-    {
-      "name": "Bts.DataStreamWebApp.Server (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
-      ],
-      "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-    {
-      "name": "DataStreamPublisher",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "DataStreamPublisher",
-      },
-      "windows": {
-        "program": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
-      ],
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DataStreamPublisher (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "DataStreamPublisher",
-      },
-      "windows": {
-        "program": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
-      ],
-      "cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DataStreamQueryAnalyzer",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
-      "program": "",
-      "windows": {
-        "program": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
-        "env": {
-          "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
-        },
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal",
-    },
-    {
-      "name": "PubSubServer",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "PubSubServer",
-      },
-      "windows": {
-        "program": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
-      "cwd": "build/Debug/bin/net7.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "PubSubServer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "PubSubServer",
-      },
-      "windows": {
-        "program": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
-      "cwd": "build/Debug/bin/net7.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "windows": {
-        "preLaunchTask": "Build Spark (Debug)",
-        "program": "Spark.exe",
-        "cwd": "build/Debug/bin/net7.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Spark (Parameterized)",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build Spark (Debug)",
-      "program": "",
-      "windows": {
-        "program": "Spark.exe",
-        "args": [
-          "--sparkd",
-          "${input:sparkConfigServer}"
-        ],
-        "cwd": "build/Debug/bin/net7.0-windows",
-      },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Sparkd",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build Sparkd (Debug | Clang)",
-      "program": "",
-      "linux": {
-        "program": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "cwd": "build/sparkd-debug-clang/server",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    }
-  ],
-  "inputs": [
-    {
-      "id": "dataStreamPublisherConfig",
-      "type": "pickString",
-      "description": "Publisher Configuration",
-      "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
-      ],
-      "default": "MarketDataPublisher",
-    },
-    {
-      "id": "pubSubServerConfig",
-      "type": "pickString",
-      "description": "PubSub Configuration",
-      "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
-      ],
-      "default": "PubSubServer",
-    },
-    {
-      "id": "sparkConfigServer",
-      "type": "promptString",
-      "description": "Config Server URL",
-      "default": "http://localhost:8000/config",
-    },
-  ]
-}
@@ -1,935 +0,0 @@
-{
-  "version": "2.0.0",
-  "tasks": [
-    {
-      "label": "Build Api Server",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_api_server.sh",
-      },
-      "windows": {
-        "command": "tools/build_api_server.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Api Server (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_api_server.sh",
-      },
-      "windows": {
-        "command": "tools/build_api_server.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Api Server (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_api_server.cmd",
-      },
-      "linux": {
-        "command": "tools/build_api_server.sh",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStreamQueryAnalyzer (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_data_stream_query_analyzer.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream WebApp",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream_web_app.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build DataStream WebApp (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/build_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/build_data_stream_web_app.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark.cmd",
-      },
-      "args": [
-        "--configuration",
-        "${input:buildConfiguration}",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark (Debug)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Debug",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Spark (Release)",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/build_spark.cmd",
-      },
-      "args": [
-        "--configuration",
-        "Release",
-      ],
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--preset",
-          "${input:sparkdCmakePreset}",
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Debug)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Debug",
-          "--compiler",
-          "${input:sparkdCompiler}"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Debug | Gcc)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Debug",
-          "--compiler",
-          "gcc"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Debug | Clang)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Debug",
-          "--compiler",
-          "clang"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Release)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Release",
-          "--compiler",
-          "${input:sparkdCompiler}"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Release | Gcc)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Release",
-          "--compiler",
-          "gcc"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Build Sparkd (Release | Clang)",
-      "group": "build",
-      "type": "shell",
-      "command": "",
-      "linux": {
-        "command": "tools/build_sparkd.sh",
-        "args": [
-          "--build-type",
-          "Release",
-          "--compiler",
-          "clang"
-        ],
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Publish DataStreamWebApp",
-      "group": "build",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/publish_data_stream_web_app.sh",
-      },
-      "windows": {
-        "command": "tools/publish_data_stream_web_app.cmd",
-      },
-      "icon": {
-        "id": "terminal",
-        "color": "terminal.ansiCyan"
-      },
-    },
-    {
-      "label": "Edit DataStream Config",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "editor",
-        "args": [
-          "${env:HOME}/.local/share/DataStream/conf/spark.conf"
-        ],
-      },
-      "windows": {
-        "command": "vim",
-        "args": [
-          "${env:APPDATA}/bts/DataStream/conf/spark.conf"
-        ],
-      },
-      "presentation": {
-        "reveal": "always",
-        "focus": true,
-        "panel": "shared",
-        "showReuseMessage": false,
-        "close": true,
-      },
-      "icon": {
-        "id": "wrench"
-      },
-    },
-    {
-      "label": "Edit Spark Config",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "vim",
-        "args": [
-          "${env:APPDATA}/bts/conf/spark.conf"
-        ],
-      },
-      "presentation": {
-        "reveal": "always",
-        "focus": true,
-        "panel": "dedicated",
-        "showReuseMessage": false,
-        "close": true,
-      },
-      "icon": {
-        "id": "wrench"
-      },
-    },
-    {
-      "label": "Run DataStreamQueryAnalyzer",
-      "dependsOn": "Build DataStreamQueryAnalyzer",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamQueryAnalyzer (Debug)",
-      "dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamQueryAnalyzer (Release)",
-      "dependsOn": "Build DataStreamQueryAnalyzer (Release)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "DataStreamQueryAnalyzer.exe",
-        "args": [
-          "--config",
-          "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
-        ],
-        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamPublisher",
-      "dependsOn": "Build DataStream",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "DataStreamPublisher",
-      },
-      "windows": {
-        "command": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamPublisher (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "DataStreamPublisher",
-      },
-      "windows": {
-        "command": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run DataStreamPublisher (Release)",
-      "dependsOn": "Build DataStream (Release)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "DataStreamPublisher",
-      },
-      "windows": {
-        "command": "DataStreamPublisher.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer",
-      "dependsOn": "Build DataStream",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer (Debug)",
-      "dependsOn": "Build DataStream (Debug)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer (Debug | Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer (Release)",
-      "dependsOn": "Build DataStream (Release)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run PubSubServer (Release | Skip Build)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "PubSubServer",
-      },
-      "windows": {
-        "command": "PubSubServer.exe",
-      },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Spark",
-      "dependsOn": "Build Spark",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
-    {
-      "label": "Run Spark (Debug)",
-      "dependsOn": "Build Spark (Debug)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
-    {
-      "label": "Run Spark (Release)",
-      "dependsOn": "Build Spark (Release)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "Spark.exe",
-        "options": {
-          "cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
-        },
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      }
-    },
-    {
-      "label": "Run Spark Config Query",
-      "dependsOn": "Build Spark",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "Spark.Console",
-      },
-      "windows": {
-        "command": "Spark.Console.exe",
-      },
-      "args": [
-        "config",
-        "--query",
-        "${input:sparkConfigQuery}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Sparkd",
-      "dependsOn": "Build Sparkd",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "run"
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "Run Sparkd Config Query",
-      "dependsOn": "Build Sparkd",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "sparkd",
-      },
-      "args": [
-        "config",
-        "--query",
-        "${input:sparkdConfigQuery}",
-      ],
-      "options": {
-        "cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
-      },
-      "icon": {
-        "id": "run",
-        "color": "terminal.ansiGreen",
-      },
-    },
-    {
-      "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
-      "type": "process",
-      "command": "",
-      "windows": {
-        "command": "tools/logging/show_last_run.cmd",
-        "args": [
-          "DataStream",
-          "DataStreamQueryAnalyzer.exe",
-          "'Starting DataStreamQueryAnalyzer'"
-        ]
-      },
-      "presentation": {
-        "reveal": "always",
-        "focus": true,
-        "panel": "shared",
-        "showReuseMessage": false,
-        "close": true,
-      },
-      "icon": {
-        "id": "wrench"
-      },
-    },
-    {
-      "label": "View PubSubServer Logs (Last Run)",
-      "type": "process",
-      "command": "",
-      "linux": {
-        "command": "tools/logging/show_last_run.sh",
-        "args": [
-          "${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
-          "Starting PubSubServer"
-        ]
-      },
-      "windows": {
-        "command": "tools/logging/show_last_run.cmd",
-        "args": [
-          "DataStream",
-          "PubSubServer.exe",
-          "Starting PubSubServer"
-        ]
-      },
-      "presentation": {
-        "reveal": "always",
-        "focus": true,
-        "panel": "shared",
-        "showReuseMessage": false,
-        "close": true,
-      },
-      "icon": {
-        "id": "wrench"
-      },
-    },
-  ],
-  "inputs": [
-    {
-      "id": "buildConfiguration",
-      "type": "pickString",
-      "description": "Configuration",
-      "options": [
-        "Debug",
-        "Release",
-      ],
-      "default": "Debug",
-    },
-    {
-      "id": "sparkdCmakePreset",
-      "type": "pickString",
-      "description": "Configuration",
-      "options": [
-        "sparkd-debug-clang",
-        "sparkd-debug-gcc",
-        "sparkd-release-clang",
-        "sparkd-release-gcc",
-      ],
-      "default": "sparkd-debug-clang",
-    },
-    {
-      "id": "sparkdCompiler",
-      "type": "pickString",
-      "description": "Configuration",
-      "options": [
-        "clang",
-        "gcc",
-      ],
-      "default": "sparkd-debug-clang",
-    },
-    {
-      "id": "dataStreamPublisherConfig",
-      "type": "pickString",
-      "description": "Publisher Configuration",
-      "options": [
-        "MarketDataPublisher",
-        "SimpleEchoClient",
-        "SimpleTestDataPublisher",
-        "TestDataPublisher",
-      ],
-      "default": "MarketDataPublisher",
-    },
-    {
-      "id": "pubSubServerConfig",
-      "type": "pickString",
-      "description": "PubSub Configuration",
-      "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
-      ],
-      "default": "PubSubServer",
-    },
-    {
-      "id": "sparkConfigQuery",
-      "type": "promptString",
-      "description": "Config Type(s)"
-    },
-    {
-      "id": "sparkdConfigQuery",
-      "type": "promptString",
-      "description": "Config Type(s)"
-    }
-  ]
-}
@@ -14,17 +14,12 @@ set(
  CACHE
  STRING
  "External dependencies root folder (should contain folders: lib, bin, include")
-
-# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
-# can run simultaneously without fighting over the same shm files (e.g., sparkd,
-# api server, etc.).
-set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
 set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
 set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
+set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
 set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
 set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
-set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
 if (WIN32)
  set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
  set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
@@ -36,13 +31,13 @@ endif()
 include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
 link_directories(${SPARK_LIB})

-add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
-add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
-if(DEFINED APP_DATA_DIR_OVERRIDE)
-  add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
+if(BUILD_API_SERVER)
+  add_compile_definitions(SPARK_ENABLE_API_SERVER)
 endif()
-add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
+
 if(WIN32)
+  add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
+  add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
  # subdirs
  add_subdirectory(src/lib)
 else()
@@ -52,7 +47,13 @@ else()
  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
  set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")

-  if(BUILD_SPARKD)
+  if(BUILD_API_SERVER)
+    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
+    add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
+  elseif(BUILD_SPARKD)
+    add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
+    add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
+
    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
    # read by their code). We override some below.
    add_compile_definitions(SPARK_ENABLE_CCAPI)
@@ -15,13 +15,11 @@
        "rhs": "Windows"
      },
      "generator": "Visual Studio 17 2022",
-      "binaryDir": "${sourceDir}/build/${presetName}",
+      "binaryDir": "${sourceDir}/build",
      "environment": {
        "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
      },
      "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
        "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
        "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
        "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
@@ -31,12 +29,8 @@
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "spark"
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
      }
    },

@@ -52,8 +46,6 @@
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
        "platform": "",
-        "APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
        "CMAKE_BUILD_WITH_INSTALL_RPATH": {
          "type": "BOOL",
          "value": "ON"
@@ -70,8 +62,7 @@
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "sparkd"
+        }
      }
    },

@@ -104,72 +95,98 @@
    {
      "name": "sparkd-debug-clang",
      "inherits": "sparkd-debug",
-      "displayName": "Sparkd (Debug | Clang)",
+      "displayName": "Debug (clang)",
      "description": "Debug build compiled w/ clang",
+      "binaryDir": "${sourceDir}/build/clang/debug",
      "cacheVariables": {
        "CMAKE_C_COMPILER": "clang",
        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG"
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
      }
    },

    {
      "name": "sparkd-debug-gcc",
      "inherits": "sparkd-debug",
-      "displayName": "Sparkd (Debug | Gcc)",
+      "displayName": "Debug (gcc)",
      "description": "Debug build compiled w/ gcc",
+      "binaryDir": "${sourceDir}/build/gcc/debug",
      "cacheVariables": {
-        "CMAKE_C_COMPILER": "gcc",
-        "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG"
+        "CMAKE_C_COMPILER": "gcc-12",
+        "CMAKE_CXX_COMPILER": "g++-12",
+        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
      }
    },

    {
      "name": "sparkd-release-clang",
      "inherits": "sparkd-release",
-      "displayName": "Sparkd (Release | Clang)",
+      "displayName": "Release (clang)",
      "description": "Release build compiled w/ clang",
+      "binaryDir": "${sourceDir}/build/clang/release",
      "cacheVariables": {
        "CMAKE_C_COMPILER": "clang",
        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
      }
    },

    {
      "name": "sparkd-release-gcc",
      "inherits": "sparkd-release",
-      "displayName": "Sparkd (Release | Gcc)",
+      "displayName": "Release (gcc)",
      "description": "Release build compiled w/ gcc",
+      "binaryDir": "${sourceDir}/build/gcc/release",
      "cacheVariables": {
        "CMAKE_C_COMPILER": "gcc",
        "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3"
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
+      }
+    },
+
+    {
+      "name": "spark-api-server-base",
+      "hidden": true,
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "cacheVariables": {
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        },
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

    {
      "name": "spark-api-server-windows",
      "inherits": "spark",
-      "displayName": "Spark Api Server (Windows)",
+      "displayName": "Spark Api Server",
      "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
-        "SHARED_MEMORY_PREFIX": "spark_api_server"
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        }
      }
    },

    {
      "name": "spark-api-server-linux",
      "inherits": "sparkd",
-      "displayName": "Spark Api Server (Linux)",
+      "displayName": "Spark Api Server",
      "generator": "Ninja Multi-Config",
      "cacheVariables": {
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
        "BUILD_ADMIN_WEB_APP": {
          "type": "BOOL",
          "value": "OFF"
        },
+        "BUILD_API_SERVER": {
+          "type": "bool",
+          "value": "ON"
+        },
        "BUILD_SPARKD": {
          "type": "BOOL",
          "value": "OFF"
@@ -180,11 +197,7 @@
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        }
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
      }
    },

@@ -207,63 +220,13 @@
    },

    {
-      "name": "data-stream-suite-windows",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Windows"
-      },
+      "name": "data-stream-suite",
      "inherits": "spark",
-      "displayName": "Data Stream Suite (Windows)",
+      "displayName": "Data Stream Suite",
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows;${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "SHARED_MEMORY_PREFIX": "data_stream"
-      }
-    },
-
-    {
-      "name": "data-stream-suite-linux",
-      "displayName": "Data Stream Suite (Linux)",
-      "description": "Data Stream utils ported over from Edge",
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Linux"
-      },
-      "inherits": "sparkd",
-      "generator": "Ninja Multi-Config",
-      "cacheVariables": {
-        "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
-        "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
-        "BUILD_ADMIN_WEB_APP": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SDK": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "BUILD_SPARKD": {
-          "type": "BOOL",
-          "value": "OFF"
-        },
-        "CMAKE_C_COMPILER": "clang",
-        "CMAKE_CXX_COMPILER": "clang++",
-        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
-        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
-        "EXPORT_MD_CONSUMER": {
-          "type": "BOOL",
-          "value": "ON"
-        },
-        "SHARED_MEMORY_PREFIX": "data_stream"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
      }
    }
  ],
@@ -281,7 +244,7 @@

    {
      "name": "spark-debug",
-      "displayName": "Spark (Debug)",
+      "displayName": "Debug",
      "inherits": "spark",
      "configurePreset": "spark",
      "configuration": "Debug"
@@ -289,7 +252,7 @@

    {
      "name": "spark-release",
-      "displayName": "Spark (Release)",
+      "displayName": "Release",
      "inherits": "spark",
      "configurePreset": "spark",
      "configuration": "Release"
@@ -317,28 +280,28 @@
    {
      "name": "sparkd-debug-clang",
      "configurePreset": "sparkd-debug-clang",
-      "displayName": "Sparkd (Debug | Clang)",
+      "displayName": "Debug (clang)",
      "inherits": "sparkd"
    },

    {
      "name": "sparkd-debug-gcc",
      "configurePreset": "sparkd-debug-gcc",
-      "displayName": "Sparkd (Debug | Gcc)",
+      "displayName": "Debug (gcc)",
      "inherits": "sparkd"
    },

    {
      "name": "sparkd-release-clang",
      "configurePreset": "sparkd-release-clang",
-      "displayName": "Sparkd (Release | Clang)",
+      "displayName": "Release (clang)",
      "inherits": "sparkd"
    },

    {
      "name": "sparkd-release-gcc",
      "configurePreset": "sparkd-release-gcc",
-      "displayName": "Sparkd (Release | Gcc)",
+      "displayName": "Release (gcc)",
      "inherits": "sparkd"
    },

@@ -355,14 +318,14 @@

    {
      "name": "spark-api-server-windows-debug",
-      "displayName": "Spark Api Server (Windows | Debug)",
+      "displayName": "Spark Api Server (Debug)",
      "inherits": "spark-api-server-windows",
      "configuration": "Debug"
    },

    {
      "name": "spark-api-server-windows-release",
-      "displayName": "Spark Api Server (Windows | Release)",
+      "displayName": "Spark Api Server (Debug)",
      "inherits": "spark-api-server-windows",
      "configuration": "Release"
    },
@@ -381,14 +344,14 @@
    {
      "name": "spark-api-server-linux-debug",
      "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Debug)",
+      "displayName": "Spark Api Server (Debug)",
      "configuration": "Debug"
    },

    {
      "name": "spark-api-server-linux-release",
      "inherits": "spark-api-server-linux",
-      "displayName": "Spark Api Server (Linux | Release)",
+      "displayName": "Spark Api Server (Release)",
      "configuration": "Release"
    },

@@ -408,30 +371,16 @@
    },

    {
-      "name": "data-stream-suite-windows-debug",
-      "displayName": "Data Stream Suite (Windows | Debug)",
-      "configurePreset": "data-stream-suite-windows",
+      "name": "data-stream-suite-debug",
+      "displayName": "DataStreamSuite (Debug)",
+      "configurePreset": "data-stream-suite",
      "configuration": "Debug"
    },

    {
-      "name": "data-stream-suite-windows-release",
-      "displayName": "Data Stream Suite (Windows | Release)",
-      "configurePreset": "data-stream-suite-windows",
-      "configuration": "Release"
-    },
-
-    {
-      "name": "data-stream-suite-linux-debug",
-      "displayName": "Data Stream Suite (Linux | Debug)",
-      "configurePreset": "data-stream-suite-linux",
-      "configuration": "Debug"
-    },
-
-    {
-      "name": "data-stream-suite-linux-release",
-      "displayName": "Data Stream Suite (Linux | Release)",
-      "configurePreset": "data-stream-suite-linux",
+      "name": "data-stream-suite-release",
+      "displayName": "DataStreamSuite (Release)",
+      "configurePreset": "data-stream-suite",
      "configuration": "Release"
    }
  ]
@@ -0,0 +1,45 @@
+/* global use, db */
+// MongoDB Playground
+// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
+// Make sure you are connected to enable completions and to be able to run a playground.
+// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
+// The result of the last command run in a playground is shown on the results panel.
+// By default the first 20 documents will be returned with a cursor.
+// Use 'console.log()' to print to the debug output.
+// For more documentation on playgrounds please refer to
+// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
+
+// Select the database to use.
+use('mongodbVSCodePlaygroundDB');
+
+// Insert a few documents into the sales collection.
+db.getCollection('sales').insertMany([
+    { 'item': 'abc', 'price': 10, 'quantity': 2, 'date': new Date('2014-03-01T08:00:00Z') },
+    { 'item': 'jkl', 'price': 20, 'quantity': 1, 'date': new Date('2014-03-01T09:00:00Z') },
+    { 'item': 'xyz', 'price': 5, 'quantity': 10, 'date': new Date('2014-03-15T09:00:00Z') },
+    { 'item': 'xyz', 'price': 5, 'quantity': 20, 'date': new Date('2014-04-04T11:21:39.736Z') },
+    { 'item': 'abc', 'price': 10, 'quantity': 10, 'date': new Date('2014-04-04T21:23:13.331Z') },
+    { 'item': 'def', 'price': 7.5, 'quantity': 5, 'date': new Date('2015-06-04T05:08:13Z') },
+    { 'item': 'def', 'price': 7.5, 'quantity': 10, 'date': new Date('2015-09-10T08:43:00Z') },
+    { 'item': 'abc', 'price': 10, 'quantity': 5, 'date': new Date('2016-02-06T20:20:13Z') },
+]);
+
+// Run a find command to view items sold on April 4th, 2014.
+const salesOnApril4th = db.getCollection('sales').find({
+    date: { $gte: new Date('2014-04-04'), $lt: new Date('2014-04-05') }
+}).count();
+
+// Print a message to the output window.
+console.log(`${salesOnApril4th} sales occurred in 2014.`);
+
+// Here we run an aggregation and open a cursor to the results.
+// Use '.toArray()' to exhaust the cursor to return the whole result set.
+// You can use '.hasNext()/.next()' to iterate through the cursor page by page.
+db.getCollection('sales').aggregate([
+    // Find all of the sales that occurred in 2014.
+    { $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } },
+    // Group the total sales for each product.
+    { $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } } } }
+]);
+
+
@@ -8,6 +8,7 @@
   public int Integer;
   public double Real;
   public string Identifier;
+   public DateTime DateTime; 
   public ImmutableList<IArithmeticExpression> Arguments;
   public ImmutableList<Assignment> Assignments;
   public IArithmeticExpression Argument;
@@ -1,22 +1,41 @@
-using System.Diagnostics;
+using mbts.spark;

-namespace BCG.Utilities.General;
+using System;
+using System.Runtime;

-/// <summary>
-/// Summary description for Crono.
-/// </summary>
-public class Crono {
-   readonly Stopwatch myTimer = new();
+namespace BCG.Utilities.General
+{
+	/// <summary>
+	/// Summary description for Crono.
+	/// </summary>
+	public class Crono
+	{
+      public Crono()
+      {
+         myTimer = new HighPerformanceTimer();
+      }

-   public void start() =>
-      myTimer.Start();
+      public void start() 
+      {myTimer.start();}

-   public void stop() =>
-      myTimer.Stop();
+      public void stop()
+      {myTimer.stop();}

-   public TimeSpan time =>
-      myTimer.Elapsed;
+      public TimeSpan time
+      {
+         get 
+         {
+            return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
+         }
+      }
+
+      public double miliseconds
+      {
+         get
+         {return myTimer.totalMilliseconds;}
+      }
+
+      private HighPerformanceTimer myTimer;
+	}

-   public double miliseconds =>
-      myTimer.Elapsed.TotalMilliseconds;
 }
@@ -1,192 +1,192 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Collections.Concurrent;
-using System.Reflection;
-using Microsoft.Win32;
-
-namespace BCG
-{
-   public enum RegistryRoot { HKCU, HKLM }
-
-   [Serializable]
-   public class BlueRegistry
-   {
-      public const string BlueSubKey = @"Software\Blue Capital Group\";
-      public static readonly string ApplicationPrefix;
-      protected string myFullPath;
-
-      public BlueRegistry(string subPath)
-      {
-         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
-      }
-
-      public BlueRegistry(BlueRegistry parent, string subPath)
-      {
-         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
-      }
-
-      static BlueRegistry()
-      {
-         string app = "Applications";
-
-         Assembly ass = Assembly.GetEntryAssembly();
-         if (ass != null)
-         {
-            app = ass.GetName().Name;
-
-            object[] attrs = ass.GetCustomAttributes(
-               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
-
-            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
-            {
-               app = ((AssemblyTitleAttribute)attrs[0]).Title;
-            }
-         }
-
-         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
-      }
-
-      public object getValue(string name)
-      {
-         return getValue(RegistryRoot.HKCU, name);
-      }
-
-      public object getValue(string name, object defaultValue)
-      {
-         return getValue(RegistryRoot.HKCU, name, defaultValue);
-      }
-
-      public void setValue(string name, object value)
-      {
-         setValue(RegistryRoot.HKCU, name, value);
-      }
-
-      public void deleteValue(string name)
-      {
-         deleteValue(RegistryRoot.HKCU, name);
-      }
-
-      private RegistryKey myGetRegistryKey(RegistryRoot root)
-      {
-         if (root == RegistryRoot.HKCU)
-            return Registry.CurrentUser.CreateSubKey(myFullPath);
-         else
-            return Registry.LocalMachine.CreateSubKey(myFullPath);
-      }
-
-      #region Generic Methods
-      public virtual void setValue(RegistryRoot root, string name, object value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.SetValue(name, value);
-         }
-      }
-
-      internal virtual object getValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name);
-         }
-      }
-
-      internal virtual void deleteValue(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            rk.DeleteValue(name);
-         }
-      }
-
-      public virtual object getValue(RegistryRoot root, string name, object defaultValue)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return rk.GetValue(name, defaultValue);
-         }
-      }
-      #endregion
-
-      #region Double
-      internal virtual void setDouble(RegistryRoot root, string name, double value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            DoubleRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual double getDouble(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return DoubleRegistryHandler.getValue(rk, name);
-         }
-      }
-      #endregion
-
-      #region Enum
-      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            EnumRegistryHandler.setValue(rk, name, value);
-         }
-      }
-
-      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return EnumRegistryHandler.getValue(rk, name, enumType);
-         }
-      }
-      #endregion
-
-      #region Array
-      internal virtual void setArray(RegistryRoot root, string name, Array array)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            ArrayRegistryHandler.setValue(rk, name, array);
-         }
-      }
-
-      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return ArrayRegistryHandler.getValue(rk, name, elementType);
-         }
-      }
-      #endregion
-
-      #region SortedList
-      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.setValue(rk, name, slist);
-         }
-      }
-
-      internal virtual SortedList getSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            return SortedListRegistryHandler.getValue(rk, name);
-         }
-      }
-
-      internal virtual void deleteSortedList(RegistryRoot root, string name)
-      {
-         using (RegistryKey rk = myGetRegistryKey(root))
-         {
-            SortedListRegistryHandler.deleteValue(rk, name);
-         }
-      }
-      #endregion
-   }
-}
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Reflection;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public enum RegistryRoot { HKCU, HKLM }
+
+   [Serializable]
+   public class BlueRegistry
+   {
+      public const string BlueSubKey = @"Software\Blue Capital Group\";
+      public static readonly string ApplicationPrefix;
+      protected string myFullPath;
+
+      public BlueRegistry(string subPath)
+      {
+         myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
+      }
+
+      public BlueRegistry(BlueRegistry parent, string subPath)
+      {
+         myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
+      }
+
+      static BlueRegistry()
+      {
+         string app = "Applications";
+
+         Assembly ass = Assembly.GetEntryAssembly();
+         if (ass != null)
+         {
+            app = ass.GetName().Name;
+
+            object[] attrs = ass.GetCustomAttributes(
+               Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
+
+            if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
+            {
+               app = ((AssemblyTitleAttribute)attrs[0]).Title;
+            }
+         }
+
+         ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
+      }
+
+      public object getValue(string name)
+      {
+         return getValue(RegistryRoot.HKCU, name);
+      }
+
+      public object getValue(string name, object defaultValue)
+      {
+         return getValue(RegistryRoot.HKCU, name, defaultValue);
+      }
+
+      public void setValue(string name, object value)
+      {
+         setValue(RegistryRoot.HKCU, name, value);
+      }
+
+      public void deleteValue(string name)
+      {
+         deleteValue(RegistryRoot.HKCU, name);
+      }
+
+      private RegistryKey myGetRegistryKey(RegistryRoot root)
+      {
+         if (root == RegistryRoot.HKCU)
+            return Registry.CurrentUser.CreateSubKey(myFullPath);
+         else
+            return Registry.LocalMachine.CreateSubKey(myFullPath);
+      }
+
+      #region Generic Methods
+      internal virtual void setValue(RegistryRoot root, string name, object value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.SetValue(name, value);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name);
+         }
+      }
+
+      internal virtual void deleteValue(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            rk.DeleteValue(name);
+         }
+      }
+
+      internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return rk.GetValue(name, defaultValue);
+         }
+      }
+      #endregion
+
+      #region Double
+      internal virtual void setDouble(RegistryRoot root, string name, double value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            DoubleRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual double getDouble(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return DoubleRegistryHandler.getValue(rk, name);
+         }
+      }
+      #endregion
+
+      #region Enum
+      internal virtual void setEnum(RegistryRoot root, string name, Enum value)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            EnumRegistryHandler.setValue(rk, name, value);
+         }
+      }
+
+      internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return EnumRegistryHandler.getValue(rk, name, enumType);
+         }
+      }
+      #endregion
+
+      #region Array
+      internal virtual void setArray(RegistryRoot root, string name, Array array)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            ArrayRegistryHandler.setValue(rk, name, array);
+         }
+      }
+
+      internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return ArrayRegistryHandler.getValue(rk, name, elementType);
+         }
+      }
+      #endregion
+
+      #region SortedList
+      internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.setValue(rk, name, slist);
+         }
+      }
+
+      internal virtual SortedList getSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            return SortedListRegistryHandler.getValue(rk, name);
+         }
+      }
+
+      internal virtual void deleteSortedList(RegistryRoot root, string name)
+      {
+         using (RegistryKey rk = myGetRegistryKey(root))
+         {
+            SortedListRegistryHandler.deleteValue(rk, name);
+         }
+      }
+      #endregion
+   }
+}
@@ -1,9 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net7.0-windows</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
+    <UseWindowsForms>true</UseWindowsForms>
  </PropertyGroup>

 </Project>
@@ -1,6 +1,4 @@
 using System;
-using System.Runtime.Versioning;
-
 using Microsoft.Win32;

 namespace BCG
@@ -12,7 +10,6 @@ namespace BCG
         rk.SetValue (name, value.ToString ());
      }

-      [SupportedOSPlatform("windows")]
      public static double getValue (RegistryKey rk, string name)
      {
         object v = rk.GetValue (name);
@@ -1,166 +1,166 @@
-using System;
-using System.Reflection;
-using System.Windows.Forms;
-using System.Drawing;
-using System.Diagnostics;
-using Microsoft.Win32;
-
-namespace BCG
-{
-   public class PersistWindowState
-   {
-      // event info that allows form to persist extra window state data
-      public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
-      public event WindowStateHandler LoadState;
-      public event WindowStateHandler SaveState;
-
-      private Form myParent;
-      private BlueRegistry myRegistry;
-
-      private int myNormalLeft;
-      private int myNormalTop;
-      private int myNormalWidth;
-      private int myNormalHeight;
-      private bool myResizable = true;
-      private FormWindowState myWindowState;
-      private bool myAllowSaveMinimized = false;
-      private bool mySavePosition = true;
-      private bool mySaveState = true;
-
-      public PersistWindowState (bool savePosition)
-      {
-         mySavePosition = savePosition;
-      }
-
-      public PersistWindowState ()
-      {
-      }
-
-      public bool saveState
-      {
-         [DebuggerStepThrough]
-         set { mySaveState = value; }
-      }
-
-      public Form parent
-      {
-         [DebuggerStepThrough]
-         get { return myParent; }
-         set
-         {
-            myParent = value;
-            myRegistry = new BlueRegistry (myParent.Text);
-
-            // subscribe to parent form's events
-            myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
-            myParent.Resize += new System.EventHandler(myOnResize);
-            myParent.Move += new System.EventHandler(myOnMove);
-            myParent.Load += new System.EventHandler(myOnLoad);
-
-            // get initial width and height in case form is never resized
-            myNormalWidth = myParent.Width;
-            myNormalHeight = myParent.Height;
-         }
-      }
-
-      public BlueRegistry blueRegistry
-      {
-         [DebuggerStepThrough]
-         get { return myRegistry; }
-         [DebuggerStepThrough]
-         set { myRegistry = value; }
-      }
-
-      public bool resizable
-      {
-         [DebuggerStepThrough]
-         set { myResizable = value; }
-      }
-
-      public bool allowSaveMinimized
-      {
-         [DebuggerStepThrough]
-         set { myAllowSaveMinimized = value; }
-      }
-
-      private void myOnResize(object sender, System.EventArgs e)
-      {
-         // save width and height
-         if (myParent.WindowState == FormWindowState.Normal && myResizable)
-         {
-            myNormalWidth = myParent.Width;
-            myNormalHeight = myParent.Height;
-         }
-      }
-
-      private void myOnMove(object sender, System.EventArgs e)
-      {
-         // save position
-         if(myParent.WindowState == FormWindowState.Normal)
-         {
-            myNormalLeft = myParent.Left;
-            myNormalTop = myParent.Top;
-         }
-         // save state
-         myWindowState = myParent.WindowState;
-      }
-
-      private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
-      {
-         // save position, size and state
-         if (mySavePosition)
-         {
-            myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
-            myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
-         }
-
-         if (myResizable)
-         {
-            myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
-            myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
-         }
-
-         // check if we are allowed to save the state as minimized (not normally)
-         if (!myAllowSaveMinimized)
-         {
-            if(myWindowState == FormWindowState.Minimized)
-               myWindowState = FormWindowState.Normal;
-         }
-
-         if (mySaveState)
-            myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
-        
-         // fire SaveState event
-         if (SaveState != null)
-            SaveState (this, myRegistry);
-      }
-
-      private void myOnLoad (object sender, System.EventArgs e)
-      {
-         if (mySavePosition)
-         {
-            int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
-            int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
-            myParent.Location = new Point (left, top);
-         }
-
-         if (myResizable)
-         {
-            int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
-            int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
-            myParent.Size = new Size (width, height);
-         }
-            
-         if (mySaveState)
-         {
-            FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState", 
-               (int) myParent.WindowState);
-            myParent.WindowState = windowState;
-         }
-
-         // fire LoadState event
-         if(LoadState != null)
-            LoadState (this, myRegistry);
-      }
-   }
-}
+using System;
+using System.Reflection;
+using System.Windows.Forms;
+using System.Drawing;
+using System.Diagnostics;
+using Microsoft.Win32;
+
+namespace BCG
+{
+   public class PersistWindowState
+   {
+      // event info that allows form to persist extra window state data
+      public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
+      public event WindowStateHandler LoadState;
+      public event WindowStateHandler SaveState;
+
+      private Form myParent;
+      private BlueRegistry myRegistry;
+
+      private int myNormalLeft;
+      private int myNormalTop;
+      private int myNormalWidth;
+      private int myNormalHeight;
+      private bool myResizable = true;
+      private FormWindowState myWindowState;
+      private bool myAllowSaveMinimized = false;
+      private bool mySavePosition = true;
+      private bool mySaveState = true;
+
+      public PersistWindowState (bool savePosition)
+      {
+         mySavePosition = savePosition;
+      }
+
+      public PersistWindowState ()
+      {
+      }
+
+      public bool saveState
+      {
+         [DebuggerStepThrough]
+         set { mySaveState = value; }
+      }
+
+      public Form parent
+      {
+         [DebuggerStepThrough]
+         get { return myParent; }
+         set
+         {
+            myParent = value;
+            myRegistry = new BlueRegistry (myParent.Text);
+
+            // subscribe to parent form's events
+            myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
+            myParent.Resize += new System.EventHandler(myOnResize);
+            myParent.Move += new System.EventHandler(myOnMove);
+            myParent.Load += new System.EventHandler(myOnLoad);
+
+            // get initial width and height in case form is never resized
+            myNormalWidth = myParent.Width;
+            myNormalHeight = myParent.Height;
+         }
+      }
+
+      public BlueRegistry blueRegistry
+      {
+         [DebuggerStepThrough]
+         get { return myRegistry; }
+         [DebuggerStepThrough]
+         set { myRegistry = value; }
+      }
+
+      public bool resizable
+      {
+         [DebuggerStepThrough]
+         set { myResizable = value; }
+      }
+
+      public bool allowSaveMinimized
+      {
+         [DebuggerStepThrough]
+         set { myAllowSaveMinimized = value; }
+      }
+
+      private void myOnResize(object sender, System.EventArgs e)
+      {
+         // save width and height
+         if (myParent.WindowState == FormWindowState.Normal && myResizable)
+         {
+            myNormalWidth = myParent.Width;
+            myNormalHeight = myParent.Height;
+         }
+      }
+
+      private void myOnMove(object sender, System.EventArgs e)
+      {
+         // save position
+         if(myParent.WindowState == FormWindowState.Normal)
+         {
+            myNormalLeft = myParent.Left;
+            myNormalTop = myParent.Top;
+         }
+         // save state
+         myWindowState = myParent.WindowState;
+      }
+
+      private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
+      {
+         // save position, size and state
+         if (mySavePosition)
+         {
+            myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
+            myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
+         }
+
+         if (myResizable)
+         {
+            myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
+            myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
+         }
+
+         // check if we are allowed to save the state as minimized (not normally)
+         if (!myAllowSaveMinimized)
+         {
+            if(myWindowState == FormWindowState.Minimized)
+               myWindowState = FormWindowState.Normal;
+         }
+
+         if (mySaveState)
+            myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
+        
+         // fire SaveState event
+         if (SaveState != null)
+            SaveState (this, myRegistry);
+      }
+
+      private void myOnLoad (object sender, System.EventArgs e)
+      {
+         if (mySavePosition)
+         {
+            int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
+            int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
+            myParent.Location = new Point (left, top);
+         }
+
+         if (myResizable)
+         {
+            int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
+            int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
+            myParent.Size = new Size (width, height);
+         }
+            
+         if (mySaveState)
+         {
+            FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState", 
+               (int) myParent.WindowState);
+            myParent.WindowState = windowState;
+         }
+
+         // fire LoadState event
+         if(LoadState != null)
+            LoadState (this, myRegistry);
+      }
+   }
+}
@@ -1,20 +0,0 @@
-module.exports = {
-  root: true,
-  env: { browser: true, es2020: true },
-  extends: [
-    'eslint:recommended',
-    'plugin:react/recommended',
-    'plugin:react/jsx-runtime',
-    'plugin:react-hooks/recommended',
-  ],
-  ignorePatterns: ['dist', '.eslintrc.cjs'],
-  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
-  settings: { react: { version: '18.3.1' } },
-  plugins: ['react-refresh'],
-  rules: {
-    'react-refresh/only-export-components': [
-      'warn',
-      { allowConstantExport: true },
-    ],
-  },
-}
@@ -1,24 +0,0 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-lerna-debug.log*
-
-node_modules
-dist
-dist-ssr
-*.local
-
-# Editor directories and files
-.vscode/*
-!.vscode/extensions.json
-.idea
-.DS_Store
-*.suo
-*.ntvs*
-*.njsproj
-*.sln
-*.sw?
@@ -1,11 +0,0 @@
-<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
-  <PropertyGroup>
-    <StartupCommand>npm run dev</StartupCommand>
-    <JavaScriptTestRoot>src\</JavaScriptTestRoot>
-    <JavaScriptTestFramework>Jest</JavaScriptTestFramework>
-    <!-- Allows the build (or compile) script located on package.json to run on Build -->
-    <ShouldRunBuildScript>true</ShouldRunBuildScript>
-    <!-- Folder where production build objects will be placed -->
-    <PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
-  </PropertyGroup>
-</Project>
@@ -1,8 +0,0 @@
-# React + Vite
-
-This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
-
-Currently, two official plugins are available:
-
- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
@@ -1,13 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <link rel="icon" type="image/ico" href="/logo.ico" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>BTS Cloud Streams</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.jsx"></script>
-  </body>
-</html>
@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-  <packageSources>
-    <clear />
-    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
-  </packageSources>
-  <disabledPackageSources>
-    <clear />
-  </disabledPackageSources>
-</configuration>
@@ -1,38 +0,0 @@
-{
-  "name": "bts.datastreamwebapp.client",
-  "private": true,
-  "version": "0.0.0",
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "vite build",
-    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
-    "preview": "vite preview"
-  },
-  "dependencies": {
-    "@angular/material": "^18.0.4",
-    "@emotion/react": "^11.11.4",
-    "@emotion/styled": "^11.11.5",
-    "@fontsource/noto-sans": "^5.0.22",
-    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
-    "@mui/icons-material": "^5.15.20",
-    "@mui/material": "^5.15.20",
-    "@mui/x-data-grid": "^7.7.1",
-    "bulma": "^1.0.1",
-    "react": "^18.3.1",
-    "react-data-table-component": "^7.6.2",
-    "react-dom": "^18.3.1",
-    "react-use-websocket": "^4.8.1",
-    "rpc-websockets": "^7.11.2"
-  },
-  "devDependencies": {
-    "@types/react": "^18.3.3",
-    "@types/react-dom": "^18.3.0",
-    "@vitejs/plugin-react": "^4.3.1",
-    "eslint": "^8.57.0",
-    "eslint-plugin-react": "^7.34.2",
-    "eslint-plugin-react-hooks": "^4.6.2",
-    "eslint-plugin-react-refresh": "^0.4.7",
-    "vite": "^5.2.13"
-  }
-}
@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
@@ -1,28 +0,0 @@
-@use "@material/checkbox";
-@use "@material/icon-button";
-@use "@material/data-table/data-table";
-
-h1#main-header {
-  padding: 4px 8px 4px 8px;
-  text-align: center;
-}
-
-td:hover {
-  background-color: cadetblue;
-}
-
-td#cell-decimal {
-  text-align: right;
-}
-
-td#cell-integer {
-  text-align: right;
-}
-
-td#cell-string {
-  text-align: center;
-}
-
-td#cell-date-time {
-  text-align: center;
-}
@@ -1,24 +0,0 @@
-import { createTheme, ThemeProvider, CssBaseline } from '@mui/material';
-import useMediaQuery from '@mui/material/useMediaQuery';
-import QueryControl from './QueryControl';
-import getTheme from './Theme';
-import '@fontsource/noto-sans';
-import './App.css';
-
-const App = () => {
-  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
-  const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
-
-  return (
-    <ThemeProvider theme={theme}>
-      <CssBaseline />
-      <h1 id='main-header' style={{
-        color: theme.palette.text.primary,
-        fontFamily: 'Noto Sans',
-      }}>BTS Cloud Streams</h1>
-      <QueryControl websocketUrl="ws://localhost:5000/ws" theme={theme} />
-    </ThemeProvider>
-  );
-};
-
-export default App;
@@ -1,32 +0,0 @@
-@use "@material/checkbox";
-@use "@material/icon-button";
-@use "@material/data-table/data-table";
-
-th {
-  font-size: large;  
-  padding: 15px;
-}
-
-td {
-  padding: 15px;
-}
-
-td:hover {
-  background-color: cadetblue;
-}
-
-td#cell-decimal {
-  text-align: right;
-}
-
-td#cell-integer {
-  text-align: right;
-}
-
-td#cell-string {
-  text-align: center;
-}
-
-td#cell-date-time {
-  text-align: center;
-}
@@ -1,52 +0,0 @@
-import PropTypes from 'prop-types';
-import { DataGrid } from '@mui/x-data-grid';
-
-function getGridColDef(col) {
-  return {
-    field: col.name,
-    headerName: col.name,
-  };
-}
-
-export default function DataTable({ columns, rows }) {
-  return (
-    <DataGrid
-      columns={columns.map(getGridColDef)}
-      rows={rows}
-      sx={{
-        borderColor: 'primary.main',
-        backgroundColor: 'primary.dark',
-        color: 'text.primary',
-        '& .MuiDataGrid-columnHeader': {
-          color: 'primary.main',
-          fontWeight: 700,
-          backgroundColor: 'primary.dark'
-        },
-        '& .MuiDataGrid-filler': {
-          color: 'primary.main',
-          fontWeight: 700,
-          backgroundColor: 'primary.dark'
-        },
-        '& .MuiDataGrid-cell': {
-          color: 'text.primary',
-        },
-        '& .MuiDataGrid-cell:hover': {
-          color: 'primary.main',
-        },
-        '& .MuiDataGrid-row': {
-          '--DataGrid-rowBorderColor': 'secondary.dark'
-        }
-      }}
-    />
-  );
-}
-
-DataTable.propTypes = {
-  columns: PropTypes.arrayOf(
-    PropTypes.shape({
-      name: PropTypes.string.isRequired,
-      type: PropTypes.string.isRequired,
-    })
-  ).isRequired,
-  rows: PropTypes.arrayOf(PropTypes.array).isRequired,
-};
@@ -1,100 +0,0 @@
-import PropTypes from 'prop-types';
-import DataTable from './DataTable';
-import QueryInputForm from './QueryInputForm';
-import Container from '@mui/material/Container';
-import React from 'react';
-
-var rpcReqIdCounter = 1;
-
-function subscribeReq(query) {
-  return {
-    jsonrpc: "2.0",
-    id: rpcReqIdCounter++,
-    method: "subscribe",
-    params: {
-      subscription_id: "1",
-      query: query
-    }
-  };
-}
-
-function unsubscribeReq() {
-  return {
-    jsonrpc: "2.0",
-    id: rpcReqIdCounter++,
-    method: "unsubscribe",
-    params: {
-      subscription_id: "1",
-    }
-  };
-}
-
-class QueryControl extends React.Component {
-  constructor({ websocketUrl, theme }) {
-    super(websocketUrl);
-    console.log('Connecting to WebSocket:', websocketUrl);
-    this.state = {
-      rows: null,
-      ws: new WebSocket(websocketUrl),
-      columns: null,
-      isRunning: false,
-      theme: theme,
-    };
-    this.state.ws.onopen = () => {
-      console.info('WebSocket open');
-    };
-    this.state.ws.onclose = () => {
-      console.info('WebSocket closed');
-    };
-    this.state.ws.onmessage = e => {
-      console.info(`Received message: ${e.data}`);
-      const msg = JSON.parse(e.data);
-      if (msg.id != null) {
-        let newState = { ...this.state };
-        newState.isRunning = msg.result.status == 'Subscribed';
-        this.setState(newState);
-      } else {
-        let newState = { ...this.state };
-        newState.rows = msg.params.all;
-        console.info(`Received rows: ${newState.rows}`);
-        newState.columns = msg.params.columns;
-        this.setState(newState);
-      }
-    };
-
-    this.start = (query) => {
-      this.state.ws.send(JSON.stringify(subscribeReq(query)));
-    };
-
-    this.stop = () => {
-      this.state.ws.send(JSON.stringify(unsubscribeReq()));
-    };
-  }
-
-  componentWillUnmount() {
-    if (this.state.ws) {
-      console.warn('Closing websocket because component is unmounting');
-      this.state.ws.close();
-    }
-  }
-
-  render() {
-    console.info('Rendering');
-    return (
-      <Container>
-        <QueryInputForm isRunning={this.state.isRunning} onStart={this.start} onStop={this.stop} />
-        {
-          this.state.rows == null ? <p style={{ color: this.state.theme.palette.warning.main }}>No data received</p> :
-            <DataTable columns={this.state.columns} rows={this.state.rows} />
-        }
-      </Container>
-    );
-  }
-}
-
-QueryControl.propTypes = {
-  websocketUrl: PropTypes.string.isRequired,
-  theme: PropTypes.object.isRequired,
-};
-
-export default QueryControl;
@@ -1,93 +0,0 @@
-import { useRef } from 'react';
-import Box from '@mui/material/Box';
-import Button from '@mui/material/Button';
-import Grid from '@mui/material/Grid';
-import Paper from '@mui/material/Paper';
-import PlayArrowIcon from '@mui/icons-material/PlayArrow';
-import PropTypes from 'prop-types';
-import StopIcon from '@mui/icons-material/Stop';
-import TextField from '@mui/material/TextField';
-import Tooltip from '@mui/material/Tooltip';
-
-const QueryInputForm = ({ isRunning, onStart, onStop }) => {
-  let textInput = useRef(null);
-
-  const onClearClick = () => {
-    textInput.current.value = '';
-  };
-
-  const onRunClick = () => {
-    onStart(textInput.current.value);
-  };
-
-  const onStopClick = () => {
-    onStop();
-  };
-
-  const onKeyDown = (e) => {
-    // Detect keyboard shortcuts.
-    //   Ctrl+Enter : Run
-    if (e.ctrlKey && e.key === 'Enter') {
-      onRunClick();
-    }
-  };
-
-  return (
-    <Box sx={{ p: 2 }}>
-      <Paper elevation={3} sx={{ color: 'primary.main' }}>
-        <Box sx={{ p: 2 }}>
-          <TextField
-            fullWidth
-            multiline
-            rows={10}
-            variant="outlined"
-            onKeyDown={onKeyDown}
-            inputRef={textInput}
-            placeholder="Enter Query..."
-          />
-        </Box>
-        <Box sx={{ p: 2, borderTop: 1 }}>
-          <Grid container spacing={2} justifyContent="flex-start">
-            <Grid item xs>
-              <Tooltip title="Clear Query Text">
-                <Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
-                  Clear
-                </Button>
-              </Tooltip>
-            </Grid>
-            <Grid item>
-              <Tooltip title="Run (Ctrl+Enter)">
-                <Button onClick={onRunClick} sx={{
-                  color: `query_start_button.${isRunning ? 'active' : 'inactive'}.color`,
-                  bgcolor: `query_start_button.${isRunning ? 'active' : 'inactive'}.background_color`,
-                  padding: '11px'
-                }}>
-                  <PlayArrowIcon />
-                </Button>
-              </Tooltip>
-            </Grid>
-            <Grid item>
-              <Tooltip title="Stop">
-                <Button onClick={onStopClick} sx={{
-                  color: `query_stop_button.${isRunning ? 'inactive' : 'active'}.color`,
-                  bgcolor: `query_stop_button.${isRunning ? 'inactive' : 'active'}.background_color`,
-                  padding: '11px'
-                }}>
-                  <StopIcon />
-                </Button>
-              </Tooltip>
-            </Grid>
-          </Grid>
-        </Box>
-      </Paper>
-    </Box>
-  );
-};
-
-QueryInputForm.propTypes = {
-  isRunning: PropTypes.bool.isRequired,
-  onStart: PropTypes.func.isRequired,
-  onStop: PropTypes.func.isRequired,
-};
-
-export default QueryInputForm;
@@ -1,88 +0,0 @@
-import { createTheme } from '@mui/material/styles';
-import PropTypes from 'prop-types';
-import '@fontsource/noto-sans';
-import './App.css';
-
-const getLightPalette = () => {
-  return {
-    mode: 'light',
-    primary: {
-      light: '#6573c3',
-      main: '#3f51b5',
-      dark: '#2c387e',
-      contrastText: '#ffffff',
-    },
-    secondary: {
-      light: '#f73378',
-      main: '#f50057',
-      dark: '#ab003c',
-      contrastText: '#ffffff',
-    },
-    text: {
-      primary: '#000000c4',
-      secondary: '#00000086',
-      disabled: '#0000005b',
-      hint: '#0000005b',
-    }
-  };
-};
-
-const getDarkPalette = () => {
-  return {
-    mode: 'dark',
-    primary: {
-      light: '#bcc5ff',
-      main: '#33baf9',
-      dark: '#202334',
-      contrastText: '#78a2ac',
-    },
-    secondary: {
-      light: '#9fd0b8',
-      main: '#5eb289',
-      dark: '#ffffff12',
-      contrastText: '#c8d4f7',
-    },
-    text: {
-      primary: '#eaf8f8',
-      dark: '#eaf8f8a0'
-    },
-    query_start_button: {
-      active: {
-        color: '#33baf9',
-        background_color: '#ffffff40',
-      },
-      inactive: {
-        color: '#ffffff40',
-        background_color: '#ffffff00',
-      }
-    },
-    warning: {
-      main: '#c4823f'
-    },
-    query_stop_button: {
-      active: {
-        color: '#ff4444a0',
-        background_color: '#ffffff40',
-      },
-      inactive: {
-        color: '#ff444450',
-        background_color: '#ffffff00',
-      }
-    },
-  };
-};
-
-const getTheme = ({ mode }) => {
-  return createTheme({
-    typography: {
-      fontFamily: '"Helvetica"'
-    },
-    palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
-  });
-};
-
-export default getTheme;
-
-getTheme.propTypes = {
-  mode: PropTypes.string.isRequired,
-};
@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>
@@ -1,51 +0,0 @@
-.dark-high-contrast {
-  --md-sys-color-primary: rgb(251 250 255);
-  --md-sys-color-surface-tint: rgb(165 200 255);
-  --md-sys-color-on-primary: rgb(0 0 0);
-  --md-sys-color-primary-container: rgb(173 204 255);
-  --md-sys-color-on-primary-container: rgb(0 0 0);
-  --md-sys-color-secondary: rgb(250 250 255);
-  --md-sys-color-on-secondary: rgb(0 0 0);
-  --md-sys-color-secondary-container: rgb(159 208 255);
-  --md-sys-color-on-secondary-container: rgb(0 0 0);
-  --md-sys-color-tertiary: rgb(255 249 251);
-  --md-sys-color-on-tertiary: rgb(0 0 0);
-  --md-sys-color-tertiary-container: rgb(222 193 231);
-  --md-sys-color-on-tertiary-container: rgb(0 0 0);
-  --md-sys-color-error: rgb(255 249 249);
-  --md-sys-color-on-error: rgb(0 0 0);
-  --md-sys-color-error-container: rgb(255 186 177);
-  --md-sys-color-on-error-container: rgb(0 0 0);
-  --md-sys-color-background: rgb(17 19 24);
-  --md-sys-color-on-background: rgb(225 226 233);
-  --md-sys-color-surface: rgb(17 19 24);
-  --md-sys-color-on-surface: rgb(255 255 255);
-  --md-sys-color-surface-variant: rgb(67 71 78);
-  --md-sys-color-on-surface-variant: rgb(251 250 255);
-  --md-sys-color-outline: rgb(200 202 212);
-  --md-sys-color-outline-variant: rgb(200 202 212);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(225 226 233);
-  --md-sys-color-inverse-on-surface: rgb(0 0 0);
-  --md-sys-color-inverse-primary: rgb(0 42 83);
-  --md-sys-color-primary-fixed: rgb(219 231 255);
-  --md-sys-color-on-primary-fixed: rgb(0 0 0);
-  --md-sys-color-primary-fixed-dim: rgb(173 204 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
-  --md-sys-color-secondary-fixed: rgb(214 233 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 0 0);
-  --md-sys-color-secondary-fixed-dim: rgb(159 208 255);
-  --md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
-  --md-sys-color-tertiary-fixed: rgb(249 222 255);
-  --md-sys-color-on-tertiary-fixed: rgb(0 0 0);
-  --md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
-  --md-sys-color-surface-dim: rgb(17 19 24);
-  --md-sys-color-surface-bright: rgb(55 57 62);
-  --md-sys-color-surface-container-lowest: rgb(12 14 19);
-  --md-sys-color-surface-container-low: rgb(25 28 32);
-  --md-sys-color-surface-container: rgb(29 32 36);
-  --md-sys-color-surface-container-high: rgb(40 42 47);
-  --md-sys-color-surface-container-highest: rgb(50 53 58);
-}
@@ -1,51 +0,0 @@
-.dark-medium-contrast {
-  --md-sys-color-primary: rgb(173 204 255);
-  --md-sys-color-surface-tint: rgb(165 200 255);
-  --md-sys-color-on-primary: rgb(0 22 49);
-  --md-sys-color-primary-container: rgb(112 146 198);
-  --md-sys-color-on-primary-container: rgb(0 0 0);
-  --md-sys-color-secondary: rgb(159 208 255);
-  --md-sys-color-on-secondary: rgb(0 24 42);
-  --md-sys-color-secondary-container: rgb(100 149 193);
-  --md-sys-color-on-secondary-container: rgb(0 0 0);
-  --md-sys-color-tertiary: rgb(222 193 231);
-  --md-sys-color-on-tertiary: rgb(33 14 42);
-  --md-sys-color-tertiary-container: rgb(162 136 171);
-  --md-sys-color-on-tertiary-container: rgb(0 0 0);
-  --md-sys-color-error: rgb(255 186 177);
-  --md-sys-color-on-error: rgb(55 0 1);
-  --md-sys-color-error-container: rgb(255 84 73);
-  --md-sys-color-on-error-container: rgb(0 0 0);
-  --md-sys-color-background: rgb(17 19 24);
-  --md-sys-color-on-background: rgb(225 226 233);
-  --md-sys-color-surface: rgb(17 19 24);
-  --md-sys-color-on-surface: rgb(251 250 255);
-  --md-sys-color-surface-variant: rgb(67 71 78);
-  --md-sys-color-on-surface-variant: rgb(200 202 212);
-  --md-sys-color-outline: rgb(160 163 171);
-  --md-sys-color-outline-variant: rgb(128 131 139);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(225 226 233);
-  --md-sys-color-inverse-on-surface: rgb(40 42 47);
-  --md-sys-color-inverse-primary: rgb(35 73 120);
-  --md-sys-color-primary-fixed: rgb(212 227 255);
-  --md-sys-color-on-primary-fixed: rgb(0 17 40);
-  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
-  --md-sys-color-secondary-fixed: rgb(206 229 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 18 34);
-  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
-  --md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
-  --md-sys-color-tertiary-fixed: rgb(247 216 255);
-  --md-sys-color-on-tertiary-fixed: rgb(28 9 37);
-  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
-  --md-sys-color-surface-dim: rgb(17 19 24);
-  --md-sys-color-surface-bright: rgb(55 57 62);
-  --md-sys-color-surface-container-lowest: rgb(12 14 19);
-  --md-sys-color-surface-container-low: rgb(25 28 32);
-  --md-sys-color-surface-container: rgb(29 32 36);
-  --md-sys-color-surface-container-high: rgb(40 42 47);
-  --md-sys-color-surface-container-highest: rgb(50 53 58);
-}
@@ -1,51 +0,0 @@
-.dark {
-  --md-sys-color-primary: rgb(165 200 255);
-  --md-sys-color-surface-tint: rgb(165 200 255);
-  --md-sys-color-on-primary: rgb(0 49 94);
-  --md-sys-color-primary-container: rgb(34 72 118);
-  --md-sys-color-on-primary-container: rgb(212 227 255);
-  --md-sys-color-secondary: rgb(154 203 250);
-  --md-sys-color-on-secondary: rgb(0 51 83);
-  --md-sys-color-secondary-container: rgb(12 74 114);
-  --md-sys-color-on-secondary-container: rgb(206 229 255);
-  --md-sys-color-tertiary: rgb(218 189 226);
-  --md-sys-color-on-tertiary: rgb(61 41 70);
-  --md-sys-color-tertiary-container: rgb(85 63 93);
-  --md-sys-color-on-tertiary-container: rgb(247 216 255);
-  --md-sys-color-error: rgb(255 180 171);
-  --md-sys-color-on-error: rgb(105 0 5);
-  --md-sys-color-error-container: rgb(147 0 10);
-  --md-sys-color-on-error-container: rgb(255 218 214);
-  --md-sys-color-background: rgb(17 19 24);
-  --md-sys-color-on-background: rgb(225 226 233);
-  --md-sys-color-surface: rgb(17 19 24);
-  --md-sys-color-on-surface: rgb(225 226 233);
-  --md-sys-color-surface-variant: rgb(67 71 78);
-  --md-sys-color-on-surface-variant: rgb(195 198 207);
-  --md-sys-color-outline: rgb(141 145 153);
-  --md-sys-color-outline-variant: rgb(67 71 78);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(225 226 233);
-  --md-sys-color-inverse-on-surface: rgb(46 48 53);
-  --md-sys-color-inverse-primary: rgb(60 96 144);
-  --md-sys-color-primary-fixed: rgb(212 227 255);
-  --md-sys-color-on-primary-fixed: rgb(0 28 58);
-  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
-  --md-sys-color-secondary-fixed: rgb(206 229 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
-  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
-  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
-  --md-sys-color-tertiary-fixed: rgb(247 216 255);
-  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
-  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
-  --md-sys-color-surface-dim: rgb(17 19 24);
-  --md-sys-color-surface-bright: rgb(55 57 62);
-  --md-sys-color-surface-container-lowest: rgb(12 14 19);
-  --md-sys-color-surface-container-low: rgb(25 28 32);
-  --md-sys-color-surface-container: rgb(29 32 36);
-  --md-sys-color-surface-container-high: rgb(40 42 47);
-  --md-sys-color-surface-container-highest: rgb(50 53 58);
-}
@@ -1,51 +0,0 @@
-.light-high-contrast {
-  --md-sys-color-primary: rgb(0 34 70);
-  --md-sys-color-surface-tint: rgb(60 96 144);
-  --md-sys-color-on-primary: rgb(255 255 255);
-  --md-sys-color-primary-container: rgb(29 68 114);
-  --md-sys-color-on-primary-container: rgb(255 255 255);
-  --md-sys-color-secondary: rgb(0 36 61);
-  --md-sys-color-on-secondary: rgb(255 255 255);
-  --md-sys-color-secondary-container: rgb(3 70 110);
-  --md-sys-color-on-secondary-container: rgb(255 255 255);
-  --md-sys-color-tertiary: rgb(46 26 55);
-  --md-sys-color-on-tertiary: rgb(255 255 255);
-  --md-sys-color-tertiary-container: rgb(81 59 89);
-  --md-sys-color-on-tertiary-container: rgb(255 255 255);
-  --md-sys-color-error: rgb(78 0 2);
-  --md-sys-color-on-error: rgb(255 255 255);
-  --md-sys-color-error-container: rgb(140 0 9);
-  --md-sys-color-on-error-container: rgb(255 255 255);
-  --md-sys-color-background: rgb(249 249 255);
-  --md-sys-color-on-background: rgb(25 28 32);
-  --md-sys-color-surface: rgb(249 249 255);
-  --md-sys-color-on-surface: rgb(0 0 0);
-  --md-sys-color-surface-variant: rgb(224 226 236);
-  --md-sys-color-on-surface-variant: rgb(32 36 43);
-  --md-sys-color-outline: rgb(63 67 74);
-  --md-sys-color-outline-variant: rgb(63 67 74);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(46 48 53);
-  --md-sys-color-inverse-on-surface: rgb(255 255 255);
-  --md-sys-color-inverse-primary: rgb(228 236 255);
-  --md-sys-color-primary-fixed: rgb(29 68 114);
-  --md-sys-color-on-primary-fixed: rgb(255 255 255);
-  --md-sys-color-primary-fixed-dim: rgb(0 45 88);
-  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-secondary-fixed: rgb(3 70 110);
-  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
-  --md-sys-color-secondary-fixed-dim: rgb(0 47 77);
-  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed: rgb(81 59 89);
-  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-surface-dim: rgb(217 218 224);
-  --md-sys-color-surface-bright: rgb(249 249 255);
-  --md-sys-color-surface-container-lowest: rgb(255 255 255);
-  --md-sys-color-surface-container-low: rgb(243 243 250);
-  --md-sys-color-surface-container: rgb(237 237 244);
-  --md-sys-color-surface-container-high: rgb(231 232 238);
-  --md-sys-color-surface-container-highest: rgb(225 226 233);
-}
@@ -1,51 +0,0 @@
-.light-medium-contrast {
-  --md-sys-color-primary: rgb(29 68 114);
-  --md-sys-color-surface-tint: rgb(60 96 144);
-  --md-sys-color-on-primary: rgb(255 255 255);
-  --md-sys-color-primary-container: rgb(83 118 167);
-  --md-sys-color-on-primary-container: rgb(255 255 255);
-  --md-sys-color-secondary: rgb(3 70 110);
-  --md-sys-color-on-secondary: rgb(255 255 255);
-  --md-sys-color-secondary-container: rgb(71 121 163);
-  --md-sys-color-on-secondary-container: rgb(255 255 255);
-  --md-sys-color-tertiary: rgb(81 59 89);
-  --md-sys-color-on-tertiary: rgb(255 255 255);
-  --md-sys-color-tertiary-container: rgb(133 108 141);
-  --md-sys-color-on-tertiary-container: rgb(255 255 255);
-  --md-sys-color-error: rgb(140 0 9);
-  --md-sys-color-on-error: rgb(255 255 255);
-  --md-sys-color-error-container: rgb(218 52 46);
-  --md-sys-color-on-error-container: rgb(255 255 255);
-  --md-sys-color-background: rgb(249 249 255);
-  --md-sys-color-on-background: rgb(25 28 32);
-  --md-sys-color-surface: rgb(249 249 255);
-  --md-sys-color-on-surface: rgb(25 28 32);
-  --md-sys-color-surface-variant: rgb(224 226 236);
-  --md-sys-color-on-surface-variant: rgb(63 67 74);
-  --md-sys-color-outline: rgb(91 95 103);
-  --md-sys-color-outline-variant: rgb(119 123 131);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(46 48 53);
-  --md-sys-color-inverse-on-surface: rgb(240 240 247);
-  --md-sys-color-inverse-primary: rgb(165 200 255);
-  --md-sys-color-primary-fixed: rgb(83 118 167);
-  --md-sys-color-on-primary-fixed: rgb(255 255 255);
-  --md-sys-color-primary-fixed-dim: rgb(58 93 141);
-  --md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-secondary-fixed: rgb(71 121 163);
-  --md-sys-color-on-secondary-fixed: rgb(255 255 255);
-  --md-sys-color-secondary-fixed-dim: rgb(43 96 137);
-  --md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed: rgb(133 108 141);
-  --md-sys-color-on-tertiary-fixed: rgb(255 255 255);
-  --md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
-  --md-sys-color-surface-dim: rgb(217 218 224);
-  --md-sys-color-surface-bright: rgb(249 249 255);
-  --md-sys-color-surface-container-lowest: rgb(255 255 255);
-  --md-sys-color-surface-container-low: rgb(243 243 250);
-  --md-sys-color-surface-container: rgb(237 237 244);
-  --md-sys-color-surface-container-high: rgb(231 232 238);
-  --md-sys-color-surface-container-highest: rgb(225 226 233);
-}
@@ -1,51 +0,0 @@
-.light {
-  --md-sys-color-primary: rgb(60 96 144);
-  --md-sys-color-surface-tint: rgb(60 96 144);
-  --md-sys-color-on-primary: rgb(255 255 255);
-  --md-sys-color-primary-container: rgb(212 227 255);
-  --md-sys-color-on-primary-container: rgb(0 28 58);
-  --md-sys-color-secondary: rgb(46 98 140);
-  --md-sys-color-on-secondary: rgb(255 255 255);
-  --md-sys-color-secondary-container: rgb(206 229 255);
-  --md-sys-color-on-secondary-container: rgb(0 29 50);
-  --md-sys-color-tertiary: rgb(110 86 118);
-  --md-sys-color-on-tertiary: rgb(255 255 255);
-  --md-sys-color-tertiary-container: rgb(247 216 255);
-  --md-sys-color-on-tertiary-container: rgb(39 20 48);
-  --md-sys-color-error: rgb(186 26 26);
-  --md-sys-color-on-error: rgb(255 255 255);
-  --md-sys-color-error-container: rgb(255 218 214);
-  --md-sys-color-on-error-container: rgb(65 0 2);
-  --md-sys-color-background: rgb(249 249 255);
-  --md-sys-color-on-background: rgb(25 28 32);
-  --md-sys-color-surface: rgb(249 249 255);
-  --md-sys-color-on-surface: rgb(25 28 32);
-  --md-sys-color-surface-variant: rgb(224 226 236);
-  --md-sys-color-on-surface-variant: rgb(67 71 78);
-  --md-sys-color-outline: rgb(116 119 127);
-  --md-sys-color-outline-variant: rgb(195 198 207);
-  --md-sys-color-shadow: rgb(0 0 0);
-  --md-sys-color-scrim: rgb(0 0 0);
-  --md-sys-color-inverse-surface: rgb(46 48 53);
-  --md-sys-color-inverse-on-surface: rgb(240 240 247);
-  --md-sys-color-inverse-primary: rgb(165 200 255);
-  --md-sys-color-primary-fixed: rgb(212 227 255);
-  --md-sys-color-on-primary-fixed: rgb(0 28 58);
-  --md-sys-color-primary-fixed-dim: rgb(165 200 255);
-  --md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
-  --md-sys-color-secondary-fixed: rgb(206 229 255);
-  --md-sys-color-on-secondary-fixed: rgb(0 29 50);
-  --md-sys-color-secondary-fixed-dim: rgb(154 203 250);
-  --md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
-  --md-sys-color-tertiary-fixed: rgb(247 216 255);
-  --md-sys-color-on-tertiary-fixed: rgb(39 20 48);
-  --md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
-  --md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
-  --md-sys-color-surface-dim: rgb(217 218 224);
-  --md-sys-color-surface-bright: rgb(249 249 255);
-  --md-sys-color-surface-container-lowest: rgb(255 255 255);
-  --md-sys-color-surface-container-low: rgb(243 243 250);
-  --md-sys-color-surface-container: rgb(237 237 244);
-  --md-sys-color-surface-container-high: rgb(231 232 238);
-  --md-sys-color-surface-container-highest: rgb(225 226 233);
-}
@@ -1,107 +0,0 @@
-:root {
-  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
-  line-height: 1.5;
-  font-weight: 400;
-
-  color-scheme: light dark;
-  color: rgba(255, 255, 255, 0.87);
-  background-color: #242424;
-
-  font-synthesis: none;
-  text-rendering: optimizeLegibility;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-}
-
-a {
-  font-weight: 500;
-  color: #646cff;
-  text-decoration: inherit;
-}
-a:hover {
-  color: #535bf2;
-}
-
-body {
-  margin: 0;
-  display: flex;
-  place-items: center;
-  min-width: 320px;
-  min-height: 100vh;
-}
-
-button {
-  border-radius: 8px;
-  border: 1px solid transparent;
-  padding: 0.6em 1.2em;
-  font-size: 1em;
-  font-weight: 500;
-  font-family: inherit;
-  background-color: #1a1a1a;
-  cursor: pointer;
-  transition: border-color 0.25s;
-}
-button:hover {
-  border-color: #646cff;
-}
-button:focus,
-button:focus-visible {
-  outline: 4px auto -webkit-focus-ring-color;
-}
-
-button#query-button {
-  height: 5em;
-  vertical-align: top;
-}
-
-button#play-button {
-  padding: '10px 20px';
-  font-size: '16px';
-  cursor: 'pointer';
-  background-color: '#28a745';
-  color: '#fff';
-  border: 'none';
-  border-radius: '5px';
-};
-
-button#pause-button {
-  background-color: #ffc107;
-}
-
-button#stop-button {
-  background-color: #dc3545;
-}
-
-h1 {
-  font-size: 3.2em;
-  line-height: 1.1;
-}
-
-p#footer {
-  float: none;
-  clear: both;
-  vertical-align: bottom;
-}
-
-table#query-output {
-  float: none;
-  clear: both;
-}
-
-textarea#query-input {
-  height: 5em;
-  width: 50em;
-}
-
-@media (prefers-color-scheme: light) {
-  :root {
-    color: #213547;
-    background-color: #ffffff;
-  }
-  a:hover {
-    color: #747bff;
-  }
-  button {
-    background-color: #f9f9f9;
-  }
-}
@@ -1,9 +0,0 @@
-import React from 'react';
-import ReactDOM from 'react-dom/client';
-import App from './App';
-
-ReactDOM.createRoot(document.getElementById('root')).render(
-  <React.StrictMode>
-    <App />
-  </React.StrictMode>
-)
@@ -1,69 +0,0 @@
-import { fileURLToPath, URL } from 'node:url';
-import { defineConfig } from 'vite';
-import plugin from '@vitejs/plugin-react';
-import fs from 'fs';
-import path from 'path';
-import child_process from 'child_process';
-import process, { env } from 'process';
-
-const baseFolder =
-  process.env.APPDATA !== undefined && process.env.APPDATA !== ''
-    ? `${process.env.APPDATA}/ASP.NET/https`
-    : `${process.env.HOME}/.aspnet/https`;
-
-const certificateName = "bts.datastreamwebapp.client";
-const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
-const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
-
-if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
-  if (0 !== child_process.spawnSync('dotnet', [
-    'dev-certs',
-    'https',
-    '--export-path',
-    certFilePath,
-    '--format',
-    'Pem',
-    '--no-password',
-  ], { stdio: 'inherit', }).status) {
-    throw new Error("Could not create certificate.");
-  }
-}
-
-const target = env.ASPNETCORE_HTTPS_PORT
-  ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
-  : env.ASPNETCORE_URLS
-    ? env.ASPNETCORE_URLS.split(';')[0]
-    : 'https://localhost:5001';
-
-export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
-  return {
-    plugins: [plugin()],
-    resolve: {
-      alias: {
-        '@': fileURLToPath(new URL('./src', import.meta.url))
-      }
-    },
-    build: {
-      minify: false,
-      sourcemap: true,
-      outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
-    },
-    server: {
-      proxy: {
-        '^/api': {
-          target,
-          secure: false
-        },
-        '^/ws': {
-          target,
-          secure: false
-        }
-      },
-      port: 5173,
-      https: {
-        key: fs.readFileSync(keyFilePath),
-        cert: fs.readFileSync(certFilePath),
-      }
-    }
-  };
-});
@@ -1,35 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
-
-  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <InvariantGlobalization>true</InvariantGlobalization>
-    <SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
-    <SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
-    <SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
-    <PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-    <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
-    <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
-    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
-    <ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
-  </ItemGroup>
-
-  <!-- <ItemGroup>
-    <Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
-  </ItemGroup> -->
-</Project>
@@ -1,106 +0,0 @@
-{
-  "MasterWorker": {
-    "Interval": "00:00:00.1"
-  },
-  "ViewStreamDatabase": {
-    "Interval": "00:00:00.1"
-  },
-  "Sparkd": {
-    "Config": {
-      "ConfigServer": "http://localhost:8000/config"
-    },
-    "Credentials": {
-      "User": "bts",
-      "Password": ""
-    },
-    "EnableAdmin": true,
-    "EnableOrders": true,
-    "EnablePosition": true
-  },
-  "PubSub": {
-    "MillisecondsBetweenSubInfoUpdates": 1000,
-    "MillisecondsBetweenTimeSliceUpdates": 5000,
-    "MillisecondsBetweenNoBufferSpaceRetries": 100,
-    "RetriesOnNoBufferSpace": 1,
-    "DoItAllSecondsBetweenLogs": 0,
-    "HeartbeatInterval": 15000,
-    "MissingHeartbeatsBeforeGc": 50,
-    "UseManagedSockets": true,
-    "StateUpdatePeriod": 5000,
-    "StatisticsUpdatePeriod": 5000,
-    "DisposedConnectionCheckPeriod": 1000,
-    "MessageProcessingWarningThreshold": 1000,
-    "ServerPort": 12345,
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:12345",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:12345",
-          "TagRegExp": ".*"
-        }
-      ]
-    },
-    "MaxPacketSize": 500000000,
-    "BlueCommMustDie": true,
-    "CrushableTags": [],
-    "TagTypeToAssemblyMap": {
-      "Entries": [
-        {
-          "HandlerClass": "BCG.PubSubServerTagHandler",
-          "Assembly": "DataStreamCore.dll"
-        }
-      ]
-    },
-    "ServerQueue": "PubSubQueue",
-    "PortRange": {
-      "First": 30400,
-      "Last": 31700
-    },
-    "BindToRandomPort": false,
-    "DebugTags": "^NOTAGS$",
-    "LatencyReportThresholdMs": 1500,
-    "UseSubstations": false,
-    "UseDynamicClients": false,
-    "IsSubstation": false,
-    "InternallyPublishSubInfo": true,
-    "UseManagedClientsInNativeCode": false,
-    "CompressedTags": "^NOTAGS$",
-    "TagsToReportCompressionRatio": "^NOTAGS$",
-    "DisableMsmq": false,
-    "UsePrepublishedServerState": false,
-    "IpTranslation": null,
-    "GcActivationThreshold": 0,
-    "WaitPeriodAfterGc": 1000,
-    "TagHandlerFactory": {
-      "Handlers": [
-        {
-          "TagType": "DataStream",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": "System_Sub",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemSubscriptionMapPublisher"
-        },
-        {
-          "TagType": "System_Stats",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemStatsPublisher"
-        },
-        {
-          "TagType": ".*",
-          "Assembly": "DataStreamCore",
-         "Class": "bts.PubsubServerGenericDataHandler"
-        }
-      ]
-    }
-  }
-}
@@ -1,4 +0,0 @@
-{
-  "content_root_path": "build/Bts.DataStreamWebApp.Client/production",
-  "web_root_path": "dist"
-}
@@ -1,30 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using Microsoft.AspNetCore.Mvc;
-
-[Route("/ws")]
-public class DataStreamSubscriptionController : ControllerBase {
-   readonly ILogger logger;
-
-   public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
-      this.logger = logger;
-      logger.LogInformation("Creating {ControllerType}", GetType().Name);
-   }
-
-   [Route("/ws")]
-   public async Task Get(CancellationToken cancellationToken) {
-      if (!HttpContext.WebSockets.IsWebSocketRequest) {
-         logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
-         return;
-      }
-      logger.LogInformation("Establishing websocket connection...");
-      try {
-         using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
-         logger.LogInformation("Websocket connection established.");
-         using var server = new DataStreamWebSocketServer(webSocket, logger);
-         await server.Run(cancellationToken);
-      } catch (Exception ex) {
-         logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
-      }
-   }
-}
@@ -1,125 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using BCG;
-
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamCore;
-using Bts.DataStreamWebApp.Server.Messages;
-
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class DataStreamSubscription : IDisposable, IHostedService {
-   public readonly ViewStream ViewStream;
-   public DataPacket? LastFired;
-
-   readonly ILogger logger;
-   readonly WebSocket webSocket;
-
-   public DataStreamSubscription(string id, JsonRpcId requestId, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
-      ViewStream = viewStream;
-      RequestId = requestId;
-      Id = id;
-      this.webSocket = webSocket;
-      this.logger = logger;
-   }
-
-   public string Id { get; }
-   public JsonRpcId RequestId { get; }
-   public SubscriptionStatus Status { get; private set; }
-
-   public void Start() {
-      if (Status == SubscriptionStatus.Subscribed) {
-         logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
-         return;
-      }
-      logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
-      Status = SubscriptionStatus.Subscribed;
-      ViewStream.onChange += onChange;
-   }
-
-   public Task StartAsync(CancellationToken cancellationToken) {
-      cancellationToken.ThrowIfCancellationRequested();
-      Start();
-      return Task.CompletedTask;
-   }
-
-   public Task Clear() {
-      if (LastFired is null) {
-         return Task.CompletedTask;
-      }
-      logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
-      return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
-   }
-
-   public void Stop() {
-      if (Status == SubscriptionStatus.Unsubscribed) {
-         logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
-         return;
-      }
-      logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
-      Status = SubscriptionStatus.Unsubscribed;
-      ViewStream.onChange -= onChange;
-   }
-
-   public Task StopAsync(CancellationToken cancellationToken) {
-      cancellationToken.ThrowIfCancellationRequested();
-      Stop();
-      return Task.CompletedTask;
-   }
-
-   public void Dispose() =>
-      Stop();
-
-   async void onChange(object newValue) {
-      if (newValue is not DataPacket dp) {
-         logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
-         return;
-      }
-      if (webSocket.State != WebSocketState.Open) {
-         logger.LogError(
-            "Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
-         Dispose();
-         return;
-      }
-      await sendDataPacket("subscribe", dp);
-   }
-
-   async Task sendDataPacket(string method, DataPacket dp) {
-      var serializerOpts = new JsonSerializerOptions {
-         NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
-      };
-      serializerOpts.Converters.Add(new DataPacketJsonConverter() {
-         IncludeCurrent = true,
-         UseObjectRows = true,
-      });
-      var res = JsonRpcRequest.CreateNotification(method, dp);
-      var str = serialize(res, serializerOpts);
-      if (str is null) {
-         logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
-         Dispose();
-         return;
-      }
-      logger.LogInformation(
-         "Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
-         dp.newRows.Count,
-         dp.schema.columnNames.Length,
-         str.Length);
-      var bytes = Encoding.UTF8.GetBytes(str);
-      await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
-      LastFired = dp;
-   }
-
-   string? serialize<T>(T message, JsonSerializerOptions? opts) {
-      opts ??= JsonSerializerOptions.Default;
-      try {
-         return JsonSerializer.Serialize(message, opts);
-      } catch (Exception ex) {
-         logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
-      }
-      return null;
-   }
-}
@@ -1,89 +0,0 @@
-namespace Bts.DataStreamWebApp.Server;
-
-using Bts.PubSub;
-
-using CommandLine;
-
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-class DataStreamWebAppServerConfig {
-   /// <summary>
-   /// The login user for sparkd
-   /// </summary>
-   [JsonPropertyName("user")]
-   [Option("user", HelpText = "The sparkd login user")]
-   public string? User { get; set; }
-
-   /// <summary>
-   /// The login password for sparkd
-   /// </summary>
-   [JsonPropertyName("password")]
-   [Option("pass", HelpText = "The sparkd login password")]
-   public string? Password { get; set; }
-
-   /// <summary>
-   /// The config server url for sparkd
-   /// </summary>
-   [JsonPropertyName("sparkd")]
-   [Option("sparkd", HelpText = "The sparkd config server")]
-   public string? Sparkd { get; set; }
-
-   [JsonPropertyName("pubsub")]
-   public PubSubConfig? PubSub { get; set; }
-
-   [Option("content-root-path")]
-   [JsonPropertyName("content_root_path")]
-   public string? ContentRootPath { get; set; }
-
-   [Option("web-root-path")]
-   [JsonPropertyName("web_root_path")]
-   public string? WebRootPath { get; set; }
-
-   [Option("environment-name")]
-   [JsonPropertyName("environment_name")]
-   public string? EnvironmentName { get; set; }
-
-   public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
-
-   [Option(
-      "config",
-      MetaValue = "PATH",
-      HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
-   [JsonIgnore]
-   public string? ConfigFile { get; set; }
-
-   public static DataStreamWebAppServerConfig Default() =>
-      new() {
-         User = "bts",
-         PubSub = PubSubConfig.Default(),
-      };
-
-   public void MergeWith(DataStreamWebAppServerConfig other) {
-      User ??= other.User;
-      Password ??= other.Password;
-      PubSub ??= other.PubSub;
-      Sparkd ??= other.Sparkd;
-      ContentRootPath ??= other.ContentRootPath;
-      WebRootPath ??= other.WebRootPath;
-      ConfigFile ??= other.ConfigFile;
-   }
-
-   /// <summary>
-   /// Merges options with those defined in `ConfigFile` (does not overwrite
-   /// non-null existing values).
-   /// </summary>
-   /// <exception cref="InvalidOperationException">
-   /// JsonSerializer.Deserialize returns null
-   /// </exception>
-   public void MergeWithConfigFile() {
-      if (ConfigFile is null) {
-         return;
-      }
-      var cfgString = File.ReadAllText(ConfigFile);
-      var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
-         throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
-      MergeWith(parsedCfg);
-      ConfigFile = null;
-   }
-}
@@ -1,121 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Controllers;
-
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamWebApp.Server.Messages;
-
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-
-public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
-   readonly ILogger logger;
-   readonly WebSocket webSocket;
-   readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
-
-   public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
-      this.webSocket = webSocket;
-      this.logger = logger;
-   }
-
-   public async Task<int> Run(CancellationToken cancellationToken) {
-      logger.LogInformation("DataStreamWebSocketServer running");
-      await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger, cancellationToken)) {
-         await (req.Method switch {
-            "subscribe" => onSubscribe(req, cancellationToken),
-            "unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>(), cancellationToken),
-            _ => Task.CompletedTask,
-         });
-      }
-      logger.LogInformation("DataStreamWebSocketServer stopped");
-      return 0;
-   }
-
-   /// <summary>
-   /// Disposes internal subscriptions. To additionally notify the client of the subscription status
-   /// change(s), use <see cref="DisposeAsync"/>.
-   /// </summary>
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      disposeSubscriptions();
-   }
-
-   /// <summary>
-   /// Disposes internal subscriptions (like <see cref="Dispose"/>), then sends 'subscription status change'
-   /// notifications to the client (unlike <see cref="Dispose"/>).
-   /// </summary>
-   public async ValueTask DisposeAsync() {
-      GC.SuppressFinalize(this);
-      var finalSubs = subscriptions.Values.Select(s => (s.RequestId, s.Id)).ToList();
-      disposeSubscriptions();
-      // Send a 'subscription status change' msg to the client for each one.
-      foreach (var (requestId, id) in finalSubs) {
-         await sendSubscriptionStatus(requestId, id, "unsubscribe", SubscriptionStatus.Unsubscribed, CancellationToken.None);
-      }
-   }
-
-   async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
-      ArgumentNullException.ThrowIfNull(req.Id);
-      ArgumentException.ThrowIfNullOrEmpty(req.Method);
-      if (!SubscribeRequest.ParseQuery(req, out var vs, out var subId, out var errMsg, out var errCode)) {
-         logger.LogError("Invalid 'subscribe' request: {Reason}", errMsg);
-         var rpcError = new JsonRpcError<JsonNode>() {
-            Code = (long)errCode,
-            Message = $"{errCode}: {errMsg}",
-         };
-         await send(rpcError, cancellationToken);
-         return;
-      }
-      if (subscriptions.TryGetValue(subId, out var oldSub)) {
-         oldSub.Dispose();
-      }
-      var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
-      subscriptions[subId] = sub;
-      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
-      await sub.StartAsync(cancellationToken);
-   }
-
-   async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
-      ArgumentNullException.ThrowIfNull(req.Id);
-      ArgumentException.ThrowIfNullOrEmpty(req.Method);
-      var subId = req.Params?.SubscriptionId ?? string.Empty;
-      if (!subscriptions.TryGetValue(subId, out var sub)) {
-         logger.LogInformation("Ignoring '{Method}' request because no subscription was found with id {Id}.", req.Method, subId);
-         return;
-      }
-      logger.LogInformation("Disposing subscription");
-      sub.Dispose();
-      subscriptions.Remove(subId);
-      await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Unsubscribed, cancellationToken);
-   }
-
-   async Task unsubscribe(JsonRpcId requestId, string subscriptionId, CancellationToken cancellationToken) {
-      await sendSubscriptionStatus(requestId, subscriptionId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
-   }
-
-   Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
-      logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
-      var response = JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
-         SubscriptionId = subId,
-         Status = status,
-      });
-      return send(response, cancellationToken);
-   }
-
-   Task send<T>(T message, CancellationToken cancellationToken) {
-      if (message is null) {
-         return Task.CompletedTask;
-      }
-      var str = JsonSerializer.Serialize(message);
-      var bytes = Encoding.UTF8.GetBytes(str);
-      logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
-      return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken);
-   }
-
-   void disposeSubscriptions() {
-      logger.LogInformation("Disposing DataStreamWebSocketServer subscriptions. Count: {Count}", subscriptions.Count);
-      subscriptions.Values.DisposeAll();
-      subscriptions.Clear();
-   }
-}
@@ -1,6 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-enum ErrorCode {
-   None = 0,
-   InvalidSubscriptionRequest,
-}
@@ -1,67 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-using BCG;
-
-using bts.websocket_core.json_rpc;
-
-using Bts.DataStreamExpressions;
-
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using System.Text.Json.Nodes;
-using System.Text.Json.Serialization;
-
-public class SubscribeRequest {
-   [JsonPropertyName("query")]
-   public string? Query { get; set; }
-
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-
-   internal static bool ParseQuery(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(true)] out string? subscriptionId, [NotNullWhen(false)] out string? errorMessage, out ErrorCode errorCode) {
-      stream = null;
-      subscriptionId = null;
-      errorCode = ErrorCode.None;
-      var paramsJsonObj = req.Params?.AsObject();
-      if (paramsJsonObj is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "'params' object not found";
-         return false;
-      }
-      var subReq = paramsJsonObj.Deserialize<SubscribeRequest>();
-      if (subReq is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "'params' object invalid";
-         return false;
-      }
-      subscriptionId = subReq.SubscriptionId ?? string.Empty;
-      var query = subReq.Query;
-      if (query is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "'params.query' field not found (expected a string)";
-         return false;
-      }
-      errorMessage = null;
-      var exp = DataStreamExpressionsInterpreter.Evaluate(query);
-      if (exp is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "'params.query' expression has invalid syntax";
-         return false;
-      }
-      string? exceptionMsg = null;
-      try {
-         stream = exp.getView(new Dictionary<string, StatementExpression>(0));
-      } catch (Exception ex) {
-         exceptionMsg = ex.ToString();
-      }
-      if (stream is null) {
-         errorCode = ErrorCode.InvalidSubscriptionRequest;
-         errorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
-         if (exceptionMsg is not null) {
-            errorMessage += ". Error" + exceptionMsg;
-         }
-         return false;
-      }
-      return true;
-   }
-}
@@ -1,18 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-using System.Text.Json.Serialization;
-
-enum SubscriptionStatus {
-   None,
-   Subscribed,
-   Unsubscribed,
-}
-
-class SubscriptionStatusMessage {
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-
-   [JsonPropertyName("status")]
-   [JsonConverter(typeof(JsonStringEnumConverter))]
-   public SubscriptionStatus Status { get; set; } = SubscriptionStatus.None;
-}
@@ -1,8 +0,0 @@
-namespace Bts.DataStreamWebApp.Server.Messages;
-
-using System.Text.Json.Serialization;
-
-public class UnsubscribeRequest {
-   [JsonPropertyName("subscription_id")]
-   public string? SubscriptionId { get; set; }
-}
@@ -1,59 +0,0 @@
-using Bts.DataStreamCore.Services;
-using Bts.PubSub.Services;
-
-using Spark.Api.Services;
-
-using System.Reactive.Linq;
-
-var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
-   Args = args,
-   ApplicationName = "Bts.DataStreamWebApp.Server",
-});
-ConfigureApp(builder.Configuration);
-ConfigureServices(builder.Services, builder.Configuration);
-
-var app = builder.Build();
-if (app.Environment.IsDevelopment()) {
-   app.UseDirectoryBrowser();
-}
-var wsOpts = new WebSocketOptions {
-   KeepAliveInterval = TimeSpan.FromMinutes(2)
-};
-app.UseWebSockets(wsOpts);
-app.UseDefaultFiles();
-app.UseStaticFiles();
-if (app.Environment.IsDevelopment()) {
-   app.UseSwagger();
-   app.UseSwaggerUI();
-}
-app.UseHttpsRedirection();
-app.UseAuthorization();
-app.MapControllers();
-try {
-   Console.Write($"Starting Application");
-   app.Run();
-   Console.Write($"Application exited gracefully.");
-} catch (Exception ex) {
-   Console.Error.Write($"Application crashed: {ex}");
-   return 1;
-}
-return 0;
-
-static void ConfigureApp(ConfigurationManager configBuilder) {
-   configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
-   var cfgRoot = configBuilder.GetSection("config");
-   if (cfgRoot?.Value is not null) {
-      configBuilder.AddJsonFile(cfgRoot.Value);
-   }
-}
-
-static void ConfigureServices(IServiceCollection services, IConfiguration config) {
-   services.AddDirectoryBrowser();
-   services.AddControllers();
-   services.AddEndpointsApiExplorer();
-   services.AddSwaggerGen();
-   services.AddMasterWorker(config.GetSection("MasterWorker"));
-   services.AddViewStreamDatabase(config.GetSection("ViewStreamDatabase"));
-   services.AddPubSubClient(config.GetSection("PubSub"));
-   services.AddSparkd(config.GetSection("Sparkd"));
-}
@@ -1,26 +0,0 @@
-{
-  "$schema": "http://json.schemastore.org/launchsettings.json",
-  "profiles": {
-    "http": {
-      "commandName": "Project",
-      "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "launchUrl": "swagger",
-      "applicationUrl": "http://localhost:5000",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-    "https": {
-      "commandName": "Project",
-      "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "applicationUrl": "http://localhost:5000;https://localhost:5001",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
-        "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
-      }
-    },
-  },
-}
@@ -1,9 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  },
-  "AllowedHosts": "*"
-}
@@ -1,168 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <title></title>
-    <style>
-        table {
-            border: 0
-        }
-
-        .commslog-data {
-            font-family: Consolas, Courier New, Courier, monospace;
-        }
-
-        .commslog-server {
-            background-color: red;
-            color: white
-        }
-
-        .commslog-client {
-            background-color: green;
-            color: white
-        }
-    </style>
-</head>
-<body>
-    <h1>ALT WebSocket Sample Application</h1>
-    <p id="stateLabel">Ready to connect...</p>
-    <div>
-        <label for="connectionUrl">WebSocket Server URL:</label>
-        <input id="connectionUrl" />
-        <button id="connectButton" type="submit">Connect</button>
-    </div>
-    <p></p>
-    <div>
-        <label for="sendMessage">Message to send:</label>
-        <input id="sendMessage" disabled />
-        <button id="sendButton" type="submit" disabled>Send</button>
-        <button id="closeButton" disabled>Close Socket</button>
-    </div>
-
-    <h2>Communication Log</h2>
-    <table style="width: 800px">
-        <thead>
-            <tr>
-                <td style="width: 100px">From</td>
-                <td style="width: 100px">To</td>
-                <td>Data</td>
-            </tr>
-        </thead>
-        <tbody id="commsLog">
-        </tbody>
-    </table>
-
-    <script>
-        var connectionUrl = document.getElementById("connectionUrl");
-        var connectButton = document.getElementById("connectButton");
-        var stateLabel = document.getElementById("stateLabel");
-        var sendMessage = document.getElementById("sendMessage");
-        var sendButton = document.getElementById("sendButton");
-        var commsLog = document.getElementById("commsLog");
-        var closeButton = document.getElementById("closeButton");
-        var socket;
-
-        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
-        var port = document.location.port ? (":" + document.location.port) : "";
-
-        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
-
-        function updateState() {
-            function disable() {
-                sendMessage.disabled = true;
-                sendButton.disabled = true;
-                closeButton.disabled = true;
-            }
-            function enable() {
-                sendMessage.disabled = false;
-                sendButton.disabled = false;
-                closeButton.disabled = false;
-            }
-
-            connectionUrl.disabled = true;
-            connectButton.disabled = true;
-
-            if (!socket) {
-                disable();
-            } else {
-                switch (socket.readyState) {
-                    case WebSocket.CLOSED:
-                        stateLabel.innerHTML = "Closed";
-                        disable();
-                        connectionUrl.disabled = false;
-                        connectButton.disabled = false;
-                        break;
-                    case WebSocket.CLOSING:
-                        stateLabel.innerHTML = "Closing...";
-                        disable();
-                        break;
-                    case WebSocket.CONNECTING:
-                        stateLabel.innerHTML = "Connecting...";
-                        disable();
-                        break;
-                    case WebSocket.OPEN:
-                        stateLabel.innerHTML = "Open";
-                        enable();
-                        break;
-                    default:
-                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
-                        disable();
-                        break;
-                }
-            }
-        }
-
-        closeButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            socket.close(1000, "Closing from client");
-        };
-
-        sendButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            var data = sendMessage.value;
-            socket.send(data);
-            commsLog.innerHTML += '<tr>' +
-                '<td class="commslog-client">Client</td>' +
-                '<td class="commslog-server">Server</td>' +
-                '<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
-        };
-
-        connectButton.onclick = function() {
-            stateLabel.innerHTML = "Connecting";
-            socket = new WebSocket(connectionUrl.value);
-            socket.onopen = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
-                '</tr>';
-            };
-            socket.onclose = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
-                '</tr>';
-            };
-            socket.onerror = updateState;
-            socket.onmessage = function (event) {
-                commsLog.innerHTML += '<tr>' +
-                    '<td class="commslog-server">Server</td>' +
-                    '<td class="commslog-client">Client</td>' +
-                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
-            };
-        };
-
-        function htmlEscape(str) {
-            return str.toString()
-                .replace(/&/g, '&amp;')
-                .replace(/"/g, '&quot;')
-                .replace(/'/g, '&#39;')
-                .replace(/</g, '&lt;')
-                .replace(/>/g, '&gt;');
-        }
-    </script>
-</body>
-</html>
@@ -1,176 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <title></title>
-    <style>
-        table {
-            border: 0
-        }
-
-        .commslog-data {
-            font-family: Consolas, Courier New, Courier, monospace;
-        }
-
-        .commslog-server {
-            background-color: red;
-            color: white
-        }
-
-        .commslog-client {
-            background-color: green;
-            color: white
-        }
-    </style>
-</head>
-<body>
-    <h1>DataStreams via WebSocket</h1>
-    <p id="stateLabel">Ready to connect...</p>
-    <div>
-        <label for="connectionUrl">WebSocket Server URL:</label>
-        <input id="connectionUrl" />
-        <button id="connectButton" type="submit">Connect</button>
-    </div>
-    <p></p>
-    <div>
-        <label for="sendMessage">Query:</label>
-        <input id="sendMessage" disabled />
-        <button id="sendButton" type="submit" disabled>Send</button>
-        <button id="closeButton" disabled>Close Socket</button>
-    </div>
-
-    <h2>Communication Log</h2>
-    <table style="width: 800px">
-        <thead>
-            <tr>
-                <td style="width: 100px">From</td>
-                <td style="width: 100px">To</td>
-                <td>Data</td>
-            </tr>
-        </thead>
-        <tbody id="commsLog">
-        </tbody>
-    </table>
-
-    <script>
-        var connectionUrl = document.getElementById("connectionUrl");
-        var connectButton = document.getElementById("connectButton");
-        var stateLabel = document.getElementById("stateLabel");
-        var sendMessage = document.getElementById("sendMessage");
-        var sendButton = document.getElementById("sendButton");
-        var commsLog = document.getElementById("commsLog");
-        var closeButton = document.getElementById("closeButton");
-        var socket;
-
-        var scheme = document.location.protocol === "https:" ? "wss" : "ws";
-        var port = document.location.port ? (":" + document.location.port) : "";
-
-        connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
-
-        function updateState() {
-            function disable() {
-                sendMessage.disabled = true;
-                sendButton.disabled = true;
-                closeButton.disabled = true;
-            }
-            function enable() {
-                sendMessage.disabled = false;
-                sendButton.disabled = false;
-                closeButton.disabled = false;
-            }
-
-            connectionUrl.disabled = true;
-            connectButton.disabled = true;
-
-            if (!socket) {
-                disable();
-            } else {
-                switch (socket.readyState) {
-                    case WebSocket.CLOSED:
-                        stateLabel.innerHTML = "Closed";
-                        disable();
-                        connectionUrl.disabled = false;
-                        connectButton.disabled = false;
-                        break;
-                    case WebSocket.CLOSING:
-                        stateLabel.innerHTML = "Closing...";
-                        disable();
-                        break;
-                    case WebSocket.CONNECTING:
-                        stateLabel.innerHTML = "Connecting...";
-                        disable();
-                        break;
-                    case WebSocket.OPEN:
-                        stateLabel.innerHTML = "Open";
-                        enable();
-                        break;
-                    default:
-                        stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
-                        disable();
-                        break;
-                }
-            }
-        }
-
-        closeButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            socket.close(1000, "Closing from client");
-        };
-
-        sendButton.onclick = function () {
-            if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("socket not connected");
-            }
-            const rpcReq = {
-              jsonrpc: "2.0",
-              method: "subscribe",
-              params: {
-                query: sendMessage.value,
-              },
-              id: 1,
-            };
-            const rpcReqStr = JSON.stringify(rpcReq);
-            socket.send(rpcReqStr);
-            commsLog.innerHTML += '<tr>' +
-                '<td class="commslog-client">Client</td>' +
-                '<td class="commslog-server">Server</td>' +
-                '<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
-        };
-
-        connectButton.onclick = function() {
-            stateLabel.innerHTML = "Connecting";
-            socket = new WebSocket(connectionUrl.value);
-            socket.onopen = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection opened</td>' +
-                '</tr>';
-            };
-            socket.onclose = function (event) {
-                updateState();
-                commsLog.innerHTML += '<tr>' +
-                    '<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
-                '</tr>';
-            };
-            socket.onerror = updateState;
-            socket.onmessage = function (event) {
-                commsLog.innerHTML += '<tr>' +
-                    '<td class="commslog-server">Server</td>' +
-                    '<td class="commslog-client">Client</td>' +
-                    '<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
-            };
-        };
-
-        function htmlEscape(str) {
-            return str.toString()
-                .replace(/&/g, '&amp;')
-                .replace(/"/g, '&quot;')
-                .replace(/'/g, '&#39;')
-                .replace(/</g, '&lt;')
-                .replace(/>/g, '&gt;');
-        }
-    </script>
-</body>
-</html>
@@ -1,26 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Reactive.Linq;
-
-public static class DataStreamExtensions {
-   public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
-      source as ViewStream ?? new ObservableViewStream(source);
-
-   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
-      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
-
-      IDisposable Subscribe(IObserver<DataPacket> observer) {
-         source.onChange += OnChange;
-         return Disposable.Create(() => source.onChange -= OnChange);
-
-         void OnChange(object? newValue) =>
-            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
-               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
-      }
-   }
-}
@@ -1,29 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts;
-
-using System;
-using System.IO;
-
-public class PandasScriptStream : ViewStream {
-   public PandasScriptStream(ViewStream vs, string file)
-      : this(vs, file, null) { }
-
-   public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
-      : this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
-
-   public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
-      mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
-
-      string readFile() {
-         try {
-            return File.ReadAllText(file);
-         } catch (Exception ex) {
-            Log.Gen.error($"PandasScriptStream error reading file: {ex}");
-            throw;
-         }
-      }
-   }
-}
@@ -1,130 +1,62 @@
-namespace Bts.DataStream;
-
-using BCG;
-using BCG.Extensions;
-using BCG.Utilities.General;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using System.Linq;
-using System.Runtime.CompilerServices;
-using System.Text;
-
-public class PandasStream : ViewStream, IWorker {
-   readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
-   readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
-   readonly string pythonExe;
-   readonly ViewStreamDatabase vsd;
-   readonly CSVSaverStream inputCsvStream;
-   readonly Lazy<string> pythonCode;
-   IDictionary<int, IBlueRow>? lastRowMap = null;
-   int inputVersion = 0;
-   int outputVersion = 0;
-   DateTime lastFireTime = DateTime.MinValue;
-   TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
-   IDisposable? dispose;
-
-   public PandasStream(ViewStream vs, string dataFrameExpression)
-      : this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }
-
-   public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
-      pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
-      this.pythonExe = pythonExe;
-      this.vsd = vsd;
-      // This should fire an update when it writes to the output file
-      inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
-      pythonCode = new(getPythonCode);
-
-      string getPythonCode() {
-         var imports =
-@"import numpy as np
-import pandas as pd
-import sys
-";
-         var bd = new StringBuilder(imports);
-         bd.AppendLine($"df=pd.read_csv('{inputFile}')");
-         bd.AppendLine(dataFrameExpression);
-         bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
-         return bd.ToString();
-      }
-   }
-
-   protected override void myOnFirstSubscriber() {
-      vsd.addWorker(this);
-      inputCsvStream.onChange += onInputChanged;
-      this.dispose = Disposable.Create(dispose);
-
-      void onInputChanged(object newValue) =>
-         inputVersion++;
-
-      void dispose() {
-         inputCsvStream.onChange -= onInputChanged;
-         vsd.removeWorker(this);
-         deleteFile(inputFile);
-         deleteFile(outputFile);
-      }
-
-      void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
-         if (!File.Exists(file)) {
-            return;
-         }
-         try {
-            File.Delete(file);
-         } catch (Exception ex) {
-            Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
-         }
-      }
-   }
-
-   protected override void myOnLastSubscriber() =>
-      Disposable.dispose(ref dispose);
-
-   static void executePythonCode(string pythonCode, string? pythonExe = null) {
-      pythonExe ??= "python";
-      var start = new ProcessStartInfo {
-         FileName = pythonExe,
-         Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
-         UseShellExecute = false,
-         RedirectStandardOutput = true
-      };
-      using var process = Process.Start(start)
-         ?? throw new InvalidOperationException("Failed to launch python");
-      process.WaitForExit();
-   }
-
-   void IWorker.doWork() {
-      if (outputVersion == inputVersion) {
-         return;
-      }
-      var now = DateTime.Now;
-      var elapsedSinceLastFire = now - lastFireTime;
-      if (elapsedSinceLastFire < fireThrottle) {
-         return;
-      }
-      outputVersion = inputVersion;
-      executePythonCode(pythonCode.Value, pythonExe);
-      var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
-      var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
-      var comp = BlueRowEqualityComparer.Instance;
-      var deletedRows = new BlueTable(schema);
-      var addedRows = new BlueTable(schema);
-      if (lastRowMap is not null) {
-         deletedRows.SetItems(lastRowMap.Where(oldRow =>
-            !(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
-         addedRows.SetItems(rowMap.Where(newRow =>
-            !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
-      }
-      lastRowMap = rowMap;
-      lastFireTime = DateTime.Now;
-      myFireEvent(new DataPacket(rows, addedRows, deletedRows));
-   }
-}
+// using System;
+// using System.Diagnostics;
+
+// using BCG;
+// using BCG.Utilities.General;
+
+
+// namespace bts {
+
+//     public class PandasStream : ViewStream, IWorker {
+//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
+
+//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
+
+//         }
+
+//         public PandasStream(ViewStream vs, string pandas_code) :
+//         this(vs, ViewStreamDatabase.instance, pandas_code) {
+//         }
+
+
+//         protected override void myOnFirstSubscriber() {
+
+//             // string python_code = @"
+//             //    import winsound
+//             //    frequency = 2500  # Set frequency (in Hz)
+//             //    duration = 1000   # Set duration (in milliseconds)
+//             //    winsound.Beep(frequency, duration)
+//             // ";
+//             string python_code = @"
+//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
+//             with open(file_path, \"w\") as f:
+//                 f.write(\"Hello, world!\n\")
+//             ";
+
+//             execute_python_code(python_code);
+//         }
+
+//         static void execute_python_code(string python_code) {
+//             ProcessStartInfo start = new ProcessStartInfo();
+//             start.FileName = "python"; // Python executable
+//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
+//             start.UseShellExecute = false;
+//             start.RedirectStandardOutput = true;
+
+//             using (Process process = Process.Start(start)) {
+//                 using (System.IO.StreamReader reader = process.StandardOutput) {
+//                     string result = reader.ReadToEnd();
+//                     Console.Write(result);
+//                 }
+//             }
+//         }
+
+//         protected override void myOnLastSubscriber() {
+//         }
+
+//         public void doWork() {
+
+//         }
+
+//     }
+// }
@@ -39,8 +39,8 @@ namespace BCG {

        protected override void myOnFirstSubscriber() {
            MasterWorker.getInstance(false).addWorker(this, _send_period);
-            var server_rci = new RemoteConnectionInfo(_ip, _port);
-            bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
+            var server_address = new BlutoSubAddress(_ip, _port);
+            bts.AsyncTCPConnector_APM.connect_to(server_address, this);
        }

        protected override void myOnLastSubscriber() {
@@ -1,37 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts;
-
-using System;
-
-public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
-   readonly IObservable<DataPacket> source;
-   IDisposable? subscription;
-
-   public ObservableViewStream(IObservable<DataPacket> source) =>
-      this.source = source;
-
-   public void Dispose() {
-      GC.SuppressFinalize(this);
-      if (subscription is not null) {
-         subscription.Dispose();
-         subscription = null;
-      }
-   }
-
-   protected override void myOnFirstSubscriber() {
-      Log.isnull(subscription);
-      subscription = source.Subscribe(myFireEvent);
-   }
-
-   protected override void myOnLastSubscriber() {
-      Log.notnull(subscription);
-      subscription.Dispose();
-      subscription = null;
-   }
-
-   IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
-      source.Subscribe(observer);
-}
@@ -0,0 +1,32 @@
+using BCG;
+
+namespace BCG
+{
+   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
+   [DataStreamUserRespresentation("add_to_cloud")]
+   public class AddToCloudStream : ViewStream
+   {
+      private readonly string _name, _stream_exp;
+      private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
+      public AddToCloudStream(string name, params string[] lines)
+      : this(name, 0, lines)
+      {
+      }
+
+      public AddToCloudStream(string name, int persist, params string[] lines)
+      {
+         _name = name;
+         _stream_exp = getExpression(lines);
+         _requestor.add_to_cloud(_name, _stream_exp, persist == 1);
+         mySetRealSource(new TableStream("cloud_streams"));
+      }
+
+      static string getExpression(params string[] lines) {
+         for (int i = 0; i < lines.Length; i++) {
+            // unescape quotes
+            lines[i] = lines[i].Replace("\\\"", "\"");
+         }
+         return string.Join("\n", lines);
+      }
+   }
+}
@@ -1,24 +1,31 @@
+using BCG;
+
 using System;
 using System.Collections.Generic;

+using mbts.config;
+using bts.utils;
+using System.Threading.Tasks;
 using Bts.DataStreamExpressions;
+using System.Collections.Generic;
 using BCG.Utilities.General;
-using bts;
-using Bts.PubSub;
-using Bts.DataStreamCore;

-namespace BCG {
+namespace BCG
+{
   // listens to the requests posted on "cloud_streams" request table, and 
   // if unfulfilled starts a ViewPubsliher
-   public class RemoteQueryManager : IWorker {
-      public class StreamPubUnit {
+   public class CloudStreamManager : IWorker
+   {
+      public class StreamPubUnit
+      {
         public string name;
         public string stream_expression;
         public ViewStream stream;
         public bool persist;
         public ViewPublisher view_publisher;

-         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
+         public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
+         {
            name = name_;
            stream_expression = stream_expression_;
            stream = stream_;
@@ -26,7 +33,7 @@ namespace BCG {
            view_publisher = view_publisher_;
         }
      }
-      static ViewStream? build_stream(string stream_expression)
+      static ViewStream build_stream(string stream_expression)
      {
         try
         {
@@ -34,7 +41,9 @@ namespace BCG {
            var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
            return rv;

-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
            bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
            return null;
         }
@@ -42,40 +51,35 @@ namespace BCG {
      private IDictionary<string, StreamPubUnit>
         _stream_pub_map = new Dictionary<string, StreamPubUnit>();
      private ViewStream _stream = new TableStream("cloud_streams");
-      readonly IPubTableBroker _broker;
-      readonly IManagedSubscriber _sub;
+      private PubTableBroker _broker = new PubTableBroker();

-
-      public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
-         _sub = managedSubscriber;
-         _broker = pubTableBroker;
+      public CloudStreamManager()
+      {
         _stream.onChange += _on_str;
-         _sub.Subscribe("RemoteRequest", (string tag, byte[] data) => {
-            RemoteRequest req = RemoteRequest.construct_from_bytes(data);
-            _spawn_remote_stream(req.published_table_name, req.stream_expression, false);
-         });
+
      }
      private void _on_str(object obj)
      {
-         if (obj is not DataPacket dp) {
-            return;
-         }
+         DataPacket dp = obj as DataPacket;
         foreach (IBlueRow r in dp.newRows)
         {
            string name = r.get<string>("name");
            bool persist = r.get<bool>("persist");
            string stream_exp = r.get<string>("stream_exp");
-            if (!_stream_pub_map.ContainsKey(name)) {
-               _spawn_remote_stream(name, stream_exp, persist);
+            if (!_stream_pub_map.ContainsKey(name))
+            {
+               _assign(name, stream_exp, persist);
            }
         }
      }

-      private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
+      private void _assign(string name, string stream_expression, bool persist)
+      {
         bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
         ViewStream stream = build_stream(stream_expression);
         _expose_to_ws(name, stream);
-         if (persist) {
+         if (persist)
+         {
            stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000);   // 10MBs
         }
         ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
@@ -83,12 +87,14 @@ namespace BCG {
         _stream_pub_map.Add(name, u);
      }

-      private void _expose_to_ws(string name, ViewStream stream) {
+      private void _expose_to_ws(string name, ViewStream stream)
+      {
         // TODO: john
      }

-      public void doWork() {
+      public void doWork()
+      {

      }
   }
-}
+}
@@ -1,48 +0,0 @@
-using BCG;
-
-using bts;
-
-using System;
-
-namespace BCG {
-   // this is a stream so we can run it from the DSQA (since dsqa can only load streams)
-   [DataStreamUserRespresentation("remoteQuery")]
-   public class RemoteQueryStream : ViewStream {
-      private readonly string _name, _stream_exp;
-      private PublisherCore _publisher_core = new PublisherCore();
-      private ViewStream _source_str;
-
-      public RemoteQueryStream(params string[] lines) {
-         _stream_exp = get_expression(lines);
-         _name = Guid.NewGuid().ToString();
-         _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
-         //_name.Replace("-", "_");
-
-      }
-
-      protected override void myOnFirstSubscriber() {
-         RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
-         byte[] rem_req_bytes = rem_req.serialize();
-         _publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
-         _source_str = new TableStream(_name);
-         _source_str.onChange += new ChangeDelegate(_handle_data);
-      }
-
-      protected override void myOnLastSubscriber() {
-         //maybe there should be a way to turn remote requests off ?
-         _source_str.onChange -= new ChangeDelegate(_handle_data);
-      }
-
-      static string get_expression(params string[] lines) {
-         for (int i = 0; i < lines.Length; i++) {
-            // unescape quotes
-            lines[i] = lines[i].Replace("\\\"", "\"");
-         }
-         return string.Join("\n", lines);
-      }
-
-      private void _handle_data(object o) {
-         myFireEvent(o);
-      }
-   }
-}
@@ -1,31 +0,0 @@
-namespace bts {
-   using System.IO;
-   using BCG;
-
-   using System.IO;
-   using System.Threading.Tasks.Dataflow;
-
-   public class RemoteRequest {
-      public readonly string stream_expression;
-      public readonly string published_table_name;
-      public RemoteRequest(string stream_expression_, string published_table_name_) {
-         stream_expression = stream_expression_;
-         published_table_name = published_table_name_;
-      }
-      public byte[] serialize() {
-         var str = new MemoryStream();
-         var bw = new BinaryWriter(str);
-         bw.write_string_cpp(stream_expression);
-         bw.write_string_cpp(published_table_name);
-         return str.ToArray();
-      }
-
-      public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
-         BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
-         string stream_expression_ = br.read_string_cpp();
-         string published_table_name_ = br.read_string_cpp();
-         return new RemoteRequest(stream_expression_, published_table_name_);
-
-      }
-   }
-}
@@ -1,54 +1,59 @@
-// using BCG;
+using BCG;

-// using bts;
+using System.Linq;

-// using System.Linq;
+namespace BCG
+{
+   public class StreamOffloadRequestor
+   {
+      public class OffloadOp
+      {
+         string name;
+         string stream_exp;
+      }
+      private IBlueTableSchema _schema;
+      private IBlueRowCollection _rows;
+      private PubTableBroker _broker = new PubTableBroker();
+      ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
+      public StreamOffloadRequestor()
+      {
+         _schema = new BlueTableSchema("cloud_streams",
+                                   new ColumnInfo("id", 0, typeof(int)),
+                                   new ColumnInfo("name", 1, typeof(string)),
+                                   new ColumnInfo("persist", 2, typeof(bool)),
+                                   new ColumnInfo("stream_exp", 3, typeof(string)));
+         _existing_offloads_vs.onChange += _on_str;
+      }
+      private void _on_str(object obj)
+      {
+         DataPacket dp = obj as DataPacket;
+         _rows = dp.newRows;
+         // this is needed so the broker knows the state of the table
+         _broker.createTable(_rows);
+      }
+      public void add_to_cloud(string name, string stream_exp, bool persist)
+      {
+         // check if it exists
+         if (_rows == null)
+         {
+            BlueTable table = new BlueTable(_schema);
+            IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
+            table.addRow(nr);
+            _broker.createTable(table);
+            // create table  
+         }
+         else
+         {
+            if (exists(name))
+               return;
+            IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
+            _broker.addRow(nr);
+         }
+      }

-// namespace BCG {
-//    // used on the client
-//    public class StreamOffloadRequestor {
-//       public class OffloadOp {
-//          string name;
-//          string stream_exp;
-//       }
-//       private IBlueTableSchema _schema;
-//       private IBlueRowCollection _rows;
-//       private PubTableBroker _broker = new PubTableBroker();
-//       ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
-//       public StreamOffloadRequestor() {
-//          _schema = new BlueTableSchema("cloud_streams",
-//                                    new ColumnInfo("id", 0, typeof(int)),
-//                                    new ColumnInfo("name", 1, typeof(string)),
-//                                    new ColumnInfo("persist", 2, typeof(bool)),
-//                                    new ColumnInfo("stream_exp", 3, typeof(string)));
-//          _existing_offloads_vs.onChange += _on_str;
-//       }
-//       private void _on_str(object obj) {
-//          DataPacket dp = obj as DataPacket;
-//          _rows = dp.newRows;
-//          // this is needed so the broker knows the state of the table
-//          _broker.create_table(_rows);
-//       }
-//       public void add_to_cloud(string name, string stream_exp, bool persist) {
-//          // check if it exists
-//          if (_rows == null) {
-//             BlueTable table = new BlueTable(_schema);
-//             IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
-//             table.addRow(nr);
-//             _broker.create_table(table);
-//             // create table  
-//          } else {
-//             if (exists(name))
-//                return;
-//             IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
-//             // TODO: use make_batch_changes instead
-//             Log.assert(false, "Fix addRow");
-//             //_broker.addRow(nr);
-//          }
-//       }
-
-//       bool exists(string name) {
-//          return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
-//       }
-//    }
-// }
+      bool exists(string name)
+      {
+         return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
+      }
+   }
+}
@@ -1,14 +1,17 @@
 using System;
 using System.Collections;

-namespace BCG {
-   internal class AddColumnsRow : IBlueRow {
+namespace BCG
+{
+   internal class AddColumnsRow : IBlueRow
+   {
      private IBlueTableSchema mySchema;
      private IBlueRow myRow;
      //private object[] myCachedValues;

      #region Constructors
-      public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) {
+      public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
+      {
         mySchema = schema;
         myRow = row;
         //myCachedValues = new object[schema.columnNames.Length];
@@ -16,78 +19,85 @@ namespace BCG {
      #endregion

      #region IBlueRow members
-      public IBlueTableSchema schema { get { return mySchema; } }
+      public IBlueTableSchema schema{get{return mySchema;}}
      #endregion

      #region ICollection members
-      public int Count { get { return mySchema.columnNames.Length; } }
+      public int Count {get{return mySchema.columnNames.Length;}}

-      public bool IsSynchronized { get { return false; } }
+      public bool IsSynchronized {get{return false;}}

-      public object SyncRoot { get { return this; } }
+      public object SyncRoot {get{return this;}}

-      public void CopyTo(Array array, int index) {
+      public void CopyTo (Array array, int index)  
+      {
         int i = index;
         foreach (object o in this as IEnumerable)
-            array.SetValue(o, i++);
+            array.SetValue (o, i++);
      }
      #endregion

      #region IDictionary members
-      public bool IsFixedSize { get { return true; } }
+      public bool IsFixedSize {get{return true;}}

-      public bool IsReadOnly { get { return true; } }
+      public bool IsReadOnly {get{return true;}}

      /// <summary>
      /// The key can either be the string corrsponding to the column name or
      /// the order of the column
      /// </summary>
-      public object this[object key] {
-         get {
+      public object this [object key]
+      {
+         get
+         {
            if (key is string)
               return this[(string)key];
            return this[(int)key];
         }
-         set { throw new NotSupportedException("The IBlueRow is readonly"); }
+         set {throw new NotSupportedException ("The IBlueRow is readonly");}
      }

-      public object this[int index] {
-         get {
+      public object this[int index]
+      {
+         get
+         {
            //if (myCachedValues[index] == null)
            //   myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
            //return myCachedValues[index];
-            return mySchema[index].isConstant ? DBNull.Value : myRow[index];
+            return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
         }
      }

-      public object this[string colName] { get { return this[mySchema[colName].colNo]; } }
+      public object this[string colName]  {get{return this[mySchema[colName].colNo];}}

-      public ICollection Keys { get { return mySchema.columnNames; } }
+      public ICollection Keys{get{return mySchema.columnNames;}}

-      public ICollection Values {
-         get {
+      public ICollection Values
+      {
+         get
+         {
            object[] retVal = new object[mySchema.columnNames.Length];
-            CopyTo(retVal, 0);
+            CopyTo (retVal, 0);
            return retVal;
         }
      }

-      public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      public void Add (object key, object value)   {throw new NotSupportedException ("The IBlueRow is readonly");}

-      public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+      public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}

-      public bool Contains(object key) { return mySchema[(string)key] != null; }
+      public bool Contains (object key){return mySchema[(string)key] != null;}

      /// <summary>
      /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
      /// iterate through the values, typecast this into an IEnumerable
      /// </summary>
      /// <returns></returns>
-      public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+      public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
+      
+      IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}

-      IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-      public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+      public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
      #endregion

   }
@@ -1,34 +1,6 @@
-using System;
-
-namespace BCG {
-   public class AddColumnsStream : ViewStream {
-      public static Tuple<string, Type, object> parse_exp(string exp) {
-         string[] toks = exp.Split(':');
-         string col_name = toks[0];
-         string col_value = toks[1];
-         Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
-         return Tuple.Create(col_name, tup.Item1, tup.Item2);
-      }
-
-      private Tuple<string, Type, object> _new_col;
-      private IBlueTableSchema _out_schema;
-
-      public AddColumnsStream(ViewStream vs, string col_def) {
-         _new_col = parse_exp(col_def);
-         ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
-         mySetRealSource(rs);
-      }
-
-      private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
-         if (_out_schema == null)
-            _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
-         return _out_schema;
-      }
-
-      private IBlueRow _map(IBlueRow r) {
-         MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
-         rv.setValue(_new_col.Item1, _new_col.Item3);
-         return rv;
-      }
-   }
-}
+namespace BCG;
+
+public class AddColumnsStream : MapStream {
+   public AddColumnsStream(ViewStream vs, string columnDef)
+      : base(vs, new ColumnAdder(columnDef)) { }
+}
@@ -1,24 +1,48 @@
-using BCG;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;

-namespace bts {
+using System.Runtime.Serialization.Formatters.Binary;
+
+namespace BCG
+{
   [DataStreamUserRespresentation("loader")]
-   public class LoaderStream : ViewStream {
+   public class LoaderStream : ViewStream, ISideEffectStream
+   {
+      public class Exception : System.Exception
+      {public Exception(string msg) : base(msg) { }}

-      private string _filename;
-      private IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
-
-
-      public LoaderStream(string filename) {
-         _filename = filename;
+      public LoaderStream(string fileName, int secondsBetweenPolls)
+      {
+         mySetRealSource(
+            new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
+               ViewStreamDatabase.instance));
      }

-      protected override void myOnFirstSubscriber() {
-         byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename);
-         StreamIOMessage msg = _deser.deserialize(ser_msg);
-         Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE);
-         IBlueRowCollection rows = msg.create_message.rows;
-         DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
-         myFireEvent(dp);
+      public LoaderStream(string fileName)
+      {
+        if (fileName == null)
+           throw new LoaderStream.Exception("File name cannot be null");
+
+        for (int i = 0; i < 5; i++)
+           try
+           {
+              using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
+              {
+                  BinaryFormatter bf = new BinaryFormatter();
+#pragma warning disable SYSLIB0011
+                  BlueTable bt = (BlueTable)bf.Deserialize(fs);
+#pragma warning restore SYSLIB0011
+                  mySetRealSource(new BlueRowCollectionStaticStream(bt));
+              }
+              break;
+           }
+           catch (IOException)
+           {
+              System.Threading.Thread.Sleep(50);
+           }
+
      }
   }
 }
@@ -5,7 +5,6 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using bts;

 namespace BCG {
   // Writes all the udpates to a stream in a file.
@@ -25,7 +24,7 @@ namespace BCG {
      private ViewStream _source_stream;
      private string _token;
      private FileStream _out_fs;
-      private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer();
+      private IBlueDataSerializer _serializer;
      private string _file_name = String.Empty;
      private long _rollover_size;
      private readonly string _directory_path;
@@ -95,12 +94,13 @@ namespace BCG {
         DataPacket dp = new_value as DataPacket;
         byte[] serialized_update = null;
         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
-         _serializer = _serializer ?? new UniversalStreamIOSerializer();
+         _serializer = _serializer ?? new UniversalSerializer(dp.schema);
         if (_write_create) {
            // new files always starts with create
-            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
+            _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
         } else
-            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
+            _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
+
         BinaryWriter bw = new BinaryWriter(_out_fs);
         long utc_epoch = epoch_now();

@@ -1,68 +1,68 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;

-// namespace BCG
-// {
-//    public class RecorderStream : ViewStream
-//    {
-//       private ViewStream mySource;
-//       private DateTime myStartTime;
-//       private FileStream myOutFile;
-//       private string myFileName;
-//       private IBlueDataSerializer mySerializer;
+namespace BCG
+{
+   public class RecorderStream : ViewStream
+   {
+      private ViewStream mySource;
+      private DateTime myStartTime;
+      private FileStream myOutFile;
+      private string myFileName;
+      private IBlueDataSerializer mySerializer;
+            
+      public RecorderStream(ViewStream src, string recordFile)
+      {
+         mySource = src;
+         myFileName = recordFile;

-//       public RecorderStream(ViewStream src, string recordFile)
-//       {
-//          mySource = src;
-//          myFileName = recordFile;
+         myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
+      }

-//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-//       }
+      protected override void myOnFirstSubscriber()
+      {
+         myStartTime = DateTime.Now;         
+         mySource.onChange += new ChangeDelegate(mySource_onChange);
+         myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
+      }
+      
+      protected override void myOnLastSubscriber()
+      {
+         mySource.onChange -= mySource_onChange;
+         myOutFile.Close();
+      }

-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStartTime = DateTime.Now;         
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-//       }
+      void mySource_onChange(object newValue)
+      {
+         DataPacket dp = (DataPacket)newValue;         
+         
+         BlueTable toWrite = new BlueTable(dp.schema);

-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//          myOutFile.Close();
-//       }
+         if (mySerializer == null)
+         {
+            mySerializer = new UniversalSerializer(dp.schema);
+         }
+         foreach (IBlueRow r in dp.newRows)
+            toWrite.addRow(r);
+         
+         byte[] result;

-//       void mySource_onChange(object newValue)
-//       {
-//          DataPacket dp = (DataPacket)newValue;         
+         if (myLastValue == null)
+            mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
+         else
+            mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);

-//          BlueTable toWrite = new BlueTable(dp.schema);
+         BinaryWriter bw = new BinaryWriter(myOutFile);
+         //write down what time this happened
+         bw.Write((DateTime.Now - myStartTime).TotalSeconds);
+         bw.Write(result.Length);
+         bw.Write(result);         
+         myOutFile.Flush();

-//          if (mySerializer == null)
-//          {
-//             mySerializer = new UniversalSerializer(dp.schema);
-//          }
-//          foreach (IBlueRow r in dp.newRows)
-//             toWrite.addRow(r);
-
-//          byte[] result;
-
-//          if (myLastValue == null)
-//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-//          else
-//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
-
-//          BinaryWriter bw = new BinaryWriter(myOutFile);
-//          //write down what time this happened
-//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-//          bw.Write(result.Length);
-//          bw.Write(result);         
-//          myOutFile.Flush();
-
-//          myFireEvent(dp);
-//       }
-//    }
-// }
+         myFireEvent(dp);
+      }
+   }
+}
@@ -1,37 +1,64 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.IO;
-using BCG;
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.IO;

-namespace bts {
-   // this saves the total state in the filr provided in the cons
-   [DataStreamUserRespresentation("saver")]
-   public class SaverStream : ViewStream {
-      private ViewStream _stream;
-      private string _filename;
-      private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
+// using System.Runtime.Serialization.Formatters.Binary;

-      public SaverStream(ViewStream vs, string filename) {
-         _stream = vs;
-         _filename = filename;
-      }
+// namespace BCG
+// {
+//    [DataStreamUserRespresentation("saver")]
+//    public class SaverStream : ViewStream, ISideEffectStream
+//    {
+//       private ViewStream myStream;
+//       private string myFileName;

-      protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); }
+//       public class Exception : System.Exception
+//       {
+//          public Exception(string msg) : base(msg) { }
+//       }

-      void _handle_changes(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
-         _save(dp);
-         myFireEvent(dp_obj);
-      }
+//       public SaverStream(ViewStream vs, string fileName)
+//       {
+//          if (fileName == null)
+//             throw new SaverStream.Exception("File name cannot be null");
+//          myStream = vs;
+//          myFileName = fileName;

-      protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); }
+//       }

-      protected void _save(DataPacket dp) {
-         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
-         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
-         StreamSerializationUtils.write_to_file(str_bytes, _filename);
+//       protected override void myOnFirstSubscriber()
+//       { myStream.onChange += new ChangeDelegate(myHandleChanges); }

-      }
-   }
-}
+//       void myHandleChanges(object newValue)
+//       {
+//          DataPacket dp = newValue as DataPacket;
+//          mySave(dp);
+//          myFireEvent(newValue);
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
+
+//       protected void mySave(DataPacket dp)
+//       {
+//          for (int i = 0; i < 5 ; i++)
+//             try
+//             {
+//                using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
+//                {
+//                   BinaryFormatter bf = new BinaryFormatter();
+//                   BlueTable t = new BlueTable(dp.schema);
+//                   foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
+//                      t.addRow(new BlueRow(r));
+//                   bf.Serialize(fs, t);
+//                }
+//                break;
+//             }
+//             catch (IOException)
+//             {
+//                System.Threading.Thread.Sleep(5);
+//             }
+//       }
+//    }
+// }
@@ -1,12 +1,12 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using System.Runtime.Serialization.Formatters.Binary;
+using BCG.Utilities.General;
+using bts.collections;
+using System.Reflection.Metadata;


 namespace BCG {
@@ -15,115 +15,115 @@ namespace BCG {
         FirstOfDay,
         LastOfDay
      };
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       public DateTime _snapshot_time;
-//       public string _snapshot_file;
-//       public IEnumerable<PlaybackPacket> _packets;
-//       public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
-//          DateTime dt;
-//          var parsed = DateTime.TryParse(snapshot_date, out dt);
-//          dt = dt.Date;
-//          bts.Log.assert(parsed);
-//          SpecificSnapshot specific_snapshot_enum;
-//          parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
-//          bts.Log.assert(parsed);
-//          string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
-//          ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
-//          mySetRealSource(rs);
-//       }
+      private BlueTable _current_table;
+      private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+      public DateTime _snapshot_time;
+      public string _snapshot_file;
+      public IEnumerable<PlaybackPacket> _packets;
+      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
+         DateTime dt;
+         var parsed = DateTime.TryParse(snapshot_date, out dt);
+         dt = dt.Date;
+         bts.Log.assert(parsed);
+         SpecificSnapshot specific_snapshot_enum;
+         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
+         bts.Log.assert(parsed);
+         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
+         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
+         mySetRealSource(rs);
+      }

-//       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
-//          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
-//       }
+      public SnapshotStream(string snapshot_dt, string directory_name, string token) {
+         _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
+         string token_directory_name = $"{directory_name}\\{token}";
+         _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
+      }

-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
-//          _packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
+      protected override void myOnFirstSubscriber() {
+         FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
+         _packets = StreamPersistorUtils.load_packets_from_file(fs);
+         PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
+         if (snapshot_packet != null) {
+            foreach (PlaybackPacket p in _packets) {
+               byte[] ops;
+               Array row_array;
+               _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+               for (int i = 0; i < ops.Length; i++) {
+                  IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+                  byte op = ops[i];
+                  if (op == StreamOpCode.create) {
+                     IBlueRowCollection full_table = rows_args[0];
+                     _current_table = new BlueTable(full_table);
+                  } else if (op == StreamOpCode.multipleRemoveAdd) {
+                     //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+                     IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+                     IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+                     BlueDataUtility.absorbChanges(_current_table, dels, adds);
+                  } else {
+                     bts.Log.assert(false, $"Oprator: {op} is not expected");
+                  }
+                  if (p.idx == snapshot_packet.idx) {
+                     myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+                     break;
+                  }
+               }
+            }
+         }
+      }

-//       private void _absorb_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//             } else {
-//                bts.Log.assert(false, $"Operator: {op} is not expected");
-//             }
-//          }
-//       }
+      private void _absorb_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+            } else {
+               bts.Log.assert(false, $"Operator: {op} is not expected");
+            }
+         }
+      }

-//       protected override void myOnLastSubscriber() {
-//       }
+      protected override void myOnLastSubscriber() {
+      }

-//       // public event SnapshotStream.Progress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          return null;
-//       }
+      // public event SnapshotStream.Progress onProgress;
+      // public event Action onFileExhausted;
+      private PlaybackPacket _locate_packet_to_replay() {
+         foreach (var p in _packets) {
+            // returns the first non replayed packet
+            if (!p.replayed)
+               return p;
+         }
+         return null;
+      }

-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
+      private void _fire_packet(byte[] ops, Array row_array) {
+         for (int i = 0; i < ops.Length; i++) {
+            IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+            byte op = ops[i];
+            if (op == StreamOpCode.create) {
+               IBlueRowCollection full_table = rows_args[0];
+               _current_table = new BlueTable(full_table);
+               //_current_table.addRows(full_table);
+               myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+                                          (myLastValue as DataPacket).newRows));
+            } else if (op == StreamOpCode.multipleRemoveAdd) {
+               //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+               IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+               IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+               BlueDataUtility.absorbChanges(_current_table, dels, adds);
+               myFireEvent(new DataPacket(_current_table, adds, dels));
+            } else {
+               bts.Log.assert(false, $"Oprator: {op} is not expected");
+            }

-//          }
-//       }
+         }
+      }
   }
-}
+}
@@ -1,10 +1,16 @@
+using BCG;
+using System;
+using System.Data;
+using System.Collections;
+using BCG.Utilities;
+using System.Text.RegularExpressions;
+
 using Bts.DataStream;

+using mbts;
+using mbts.spark;
 using mbts.config;

-using System;
-using System.Text.RegularExpressions;
-
 namespace BCG.StreamWorld.Addition
 {
   [DataStreamUserRespresentation (false)]
@@ -145,6 +151,7 @@ namespace BCG
      private string myFilterString;
      private BCG.StreamWorld.Addition.FilterStream myFilterStream;
      private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
+      private HighPerformanceTimer _timer = new HighPerformanceTimer();

      #region constructors
      public delegate bool OnFilter(IBlueRow r);
@@ -219,6 +226,7 @@ namespace BCG

      private void myHandleCallbacks(object newValue)
      {
+         _timer.start();
         DataPacket retVal;
         DataPacket arrivedPacket = newValue as DataPacket;
         myLastReceivedRows = arrivedPacket.newRows;
@@ -247,6 +255,7 @@ namespace BCG
               if (myFilter.filter(r))
                  myLastFiredRows.addRow(r);
         }
+         _timer.stop();
         myFireEvent(retVal);
      }

@@ -1,66 +1,37 @@
 using System;
-using System.Text.RegularExpressions;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;

 using BCG.Utilities.General;

-namespace BCG {
-   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable {
-
-      public static TimeSpan parse_duration(string input) {
-         int days = 0, hours = 0, minutes = 0, seconds = 0;
-
-         // Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
-         Regex regex = new Regex(@"(\d+)([dhms])");
-         MatchCollection matches = regex.Matches(input);
-
-         foreach (Match match in matches) {
-            int value = int.Parse(match.Groups[1].Value);
-            string unit = match.Groups[2].Value;
-
-            switch (unit) {
-               case "d":
-                  days = value;
-                  break;
-               case "h":
-                  hours = value;
-                  break;
-               case "m":
-                  minutes = value;
-                  break;
-               case "s":
-                  seconds = value;
-                  break;
-            }
-         }
-
-         return new TimeSpan(days, hours, minutes, seconds);
-      }
-
+namespace BCG
+{
+   public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
+   {
      #region constructors
-
-      public TimeBasedFilter(string col_name, string row_time_to_live)
-      : this(col_name, parse_duration(row_time_to_live)) {
-
-      }
-
      public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
+         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
+      {
      }

      public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
-      : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) {
+         : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
+      {
      }

      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
-      : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
+         : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
+      {
      }

-      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
-         myVsd = vsd;
-         _time_col = rowTimeColumnName;
-         _ttl = rowTimeToLive;
-         myFilterFrequency = filterFrequency;
-         myNextFilter = DateTime.MinValue;
+      public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
+      {
+         myVsd               = vsd;
+         myRowTimeColumnName = rowTimeColumnName;
+         myRowTimeToLive     = rowTimeToLive;
+         myFilterFrequency   = filterFrequency;
+         myNextFilter        = DateTime.MinValue;

         // add worker
         myVsd.addWorker(this);
@@ -70,25 +41,32 @@ namespace BCG {
      public event FilterChangeDelegate onFilterChange;

      #region IStreamFilter
-      public bool filter(IBlueRow row) {
-         DateTime now = DateTime.Now;
+      public bool filter(IBlueRow row)
+      {
+         DateTime now       = DateTime.Now;
+         DateTime timeStamp = (DateTime)row[myRowTimeColumnName];

-         DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
-         if (now > row_time) {
-            TimeSpan span = now - row_time;
-            if (span >= _ttl) {
+         if (now > timeStamp)
+         {
+            TimeSpan span = now - timeStamp;
+
+            if (span >= myRowTimeToLive)
+            {
               return (false);
            }
         }
+
         return (true);
      }
      #endregion

      #region IWorker
-      public void doWork() {
+      public void doWork()
+      {
         DateTime now = DateTime.Now;

-         if (now > myNextFilter) {
+         if (now > myNextFilter)
+         {
            // fire event
            onFilterChange();

@@ -98,7 +76,8 @@ namespace BCG {
      #endregion

      #region IDisposable
-      public void Dispose() {
+      public void Dispose()
+      {
         // remove worker
         myVsd.removeWorker(this);
      }
@@ -106,20 +85,10 @@ namespace BCG {

      #region state
      private ViewStreamDatabase myVsd;
-      private string _time_col;
-      private TimeSpan _ttl;
-      private TimeSpan myFilterFrequency;
-      private DateTime myNextFilter;
+      private string             myRowTimeColumnName;
+      private TimeSpan           myRowTimeToLive;
+      private TimeSpan           myFilterFrequency;
+      private DateTime           myNextFilter;
      #endregion
   }
-   [DataStreamUserRespresentation("timefilter")]
-   public class TimeBasedFilterStream : ViewStream {
-      public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
-      :
-      this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
-      public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
-         TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
-         mySetRealSource(new FilterStream(base_stream, filter));
-      }
-   }
 }
@@ -4,14 +4,18 @@ using BCG.Utilities;
 using System.Collections.Generic;
 using System.Linq;

-namespace BCG {
-   public class GroupByStream : MethodDrivenStream {
-      private class MyFuncGroupByEvaluator : IGroupByEvaluator {
+namespace BCG
+{
+   public class GroupByStream : MethodDrivenStream
+   {
+      private class MyFuncGroupByEvaluator : IGroupByEvaluator
+      {
         private string[] _groupby_cols;
         private Func<DataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
+         {
            _groupby_cols = groupby_cols.ToArray();
-            _groupby_func = groupby_func;
+            _groupby_func = groupby_func;            
         }
         public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }

@@ -25,7 +29,7 @@ namespace BCG {
      /// Serves as  a quick way to access all rows belonging to the same group
      /// </summary>
      private bool myIdGuard = true;
-      private RowDictionaryTable myBookKeepingTable;
+      private RowDictionaryTable myBookKeepingTable ;
      private IGroupByEvaluator myGroupByEvaluator;
      private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
      //can map a schema to another
@@ -33,98 +37,112 @@ namespace BCG {
      private int myIdGenerator = 0;

      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
-         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
+         : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
+      { }

-      public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
-         : this(stream, groupByEvaluator, true) { }
+      public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
+         : this (stream, groupByEvaluator, true)
+      {}

-      public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
-         : base(stream) {
+      public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
+         : base (stream)
+      {
         myGroupByEvaluator = groupbyEvaluator;
         myIdGuard = idGuard;
      }

-      protected override void myHandleFirstCall() {
+      protected override void myHandleFirstCall ()
+      {
         mySchema = myLastArrivedPacket.schema;
-         myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
-         myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
+         myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
+         myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows, 
            myGroupByEvaluator.groupByColumns);
-         GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
-         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
-            IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
-               rowGroup, new BlueTable(mySchema)));
+         GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
+         foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
+         {
+            IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
+               rowGroup, new BlueTable (mySchema)));
            if (newRow != null)
-               temp.addRow(newRow, myGetGroupString(rowGroup));
+               temp.addRow (newRow, myGetGroupString (rowGroup));
         }
-         myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema));
+         myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
      }

      /// <summary>
      /// Adding need not to anything as deletion already takes care
      /// of addition
      /// </summary>
-      protected override void myHandleIncrementalAdd() { }
-
+      protected override void myHandleIncrementalAdd()   {}
+      
      /// <summary>
      /// Both deletes and adds are handled here to make the intermediate results not
      /// appear in the spitted fired DataPacket (note this is done only in GroupByStream 
      /// because of the nature of it)
      /// </summary>
-      protected override void myHandleIncrementalDelete() {
+      protected override void myHandleIncrementalDelete()   
+      {
         myFireSwitch = false;
         RowDictionaryTable delGroups, addGroups;
         //form groups for deleted and added rows
-         myInitIncrementalChangeData(out delGroups, out addGroups);
-         ArrayList handledAddGroups = new ArrayList();
+         myInitIncrementalChangeData (out delGroups, out addGroups);
+         ArrayList handledAddGroups = new ArrayList ();
         string groupRepString;

-         foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
+         foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
+         {
            // BlueEventLog.logAssert (delGroup.Count > 0);
-            groupRepString = myGetGroupString(delGroup);
-            IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
-               new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
-            handledAddGroups.Add(addGroup);
-            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
-               new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
-            IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
-            myFireSwitch |= myMergeRow(newRow, groupRepString);
+            groupRepString = myGetGroupString (delGroup);
+            IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
+               new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
+            handledAddGroups.Add (addGroup);
+            IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
+               new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
+            IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
+            myFireSwitch |= myMergeRow (newRow, groupRepString);           
         }

-         foreach (IBlueRowCollection addGroup in addGroups.getGroups()) {
+         foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
+         {
            // BlueEventLog.logAssert (addGroup.Count > 0);
-            if (!handledAddGroups.Contains(addGroup)) {
+            if (!handledAddGroups.Contains (addGroup))
+            {
               //this is the case when there are rows added to a group and none
               //have been removed, hence it 'escapes' from being dealt with in 
               //the loop above

-               groupRepString = myGetGroupString(addGroup);
-               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
+               groupRepString = myGetGroupString (addGroup);
+               IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
               // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
-               IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
-                  new BlueTable(aggGroup.schema)));
-               myFireSwitch |= myMergeRow(newRow, groupRepString);
+               IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, 
+                  new BlueTable (aggGroup.schema)));
+               myFireSwitch |= myMergeRow (newRow, groupRepString);
            }
         }
      }

-      private bool myMergeRow(IBlueRow newRow, string groupRepString) {
-         IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
+      private bool myMergeRow (IBlueRow newRow, string groupRepString)
+      {
+         IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
         //BlueEventLog.logAssert (newRow != null || existingRow != null);
         if (existingRow == null && newRow == null)
            return false;
-         else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
+         else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
            return false;
-         else if (existingRow == null) {
+         else if (existingRow == null)
+         {
            //row added for the first time
-            ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
-            ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
+            ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
+            ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
            return true;
-         } else {
-            ((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString);
-            ((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow);
-            if (newRow != null) {
-               ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
-               ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
+         }
+         else
+         {
+            ((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
+            ((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
+            if (newRow != null)
+            {
+               ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
+               ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
            }
            return true;
         }
@@ -136,68 +154,77 @@ namespace BCG {
      /// </summary>
      /// <param name="delGroups"></param>
      /// <param name="addGroups"></param>
-      private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
+      private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
+      {
         delGroups =
-            new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
         addGroups =
-            new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
+            new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
         foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
-            myBookKeepingTable.removeRow(deletedRow);
+            myBookKeepingTable.removeRow (deletedRow);
         foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
-            myBookKeepingTable.addRow(addedRow);
-         myLastFiredPacket.addedRows = new BlueTable(myOutputSchema);
-         myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema);
+            myBookKeepingTable.addRow (addedRow);
+         myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
+         myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
      }

-      protected override void myOnTableClear() {
+      protected override void myOnTableClear ()
+      {
         if (myBookKeepingTable.Count == 0)
            return;  //ie there arent any rows, so no need to optimize the 'cleansing' process
-         myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
-         myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
-         (myLastFiredPacket.newRows as GroupByOutputTable).clear();
-         myBookKeepingTable.clear();
-         myGroupByEvaluator.clear();
+         myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
+         myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
+         (myLastFiredPacket.newRows as GroupByOutputTable).clear ();
+         myBookKeepingTable.clear ();
+         myGroupByEvaluator.clear ();
      }

-      private IBlueRow myEvaluate(DataPacket dp) {
-         IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
-         return evalRow == null ? null :
+      private IBlueRow myEvaluate (DataPacket dp)
+      {
+         IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
+         return evalRow == null ? null : 
            myIdGuard ?
-            BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow;
+            BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
      }

-      private string myGetGroupString(IBlueRowCollection group) {
-         return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns);
+      private string myGetGroupString (IBlueRowCollection group)
+      {
+         return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
      }

-      protected override void myOnLastSubscriber() {
+      protected override  void myOnLastSubscriber () 
+      {
         //the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream 
         //gets resubscribed, we start the process of building data again, and treat the callback as the first 
         //callback.
-         myInit();
-         myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
+         myInit ();
+         myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
      }

-      protected override void myInit() {
-         base.myInit();
-         if (myBookKeepingTable != null) {
+      protected override void myInit ()
+      {
+         base.myInit ();
+         if (myBookKeepingTable != null)
+         {
            //to avoid calling init before the first callback
-            myBookKeepingTable.clear();
-            myGroupByEvaluator.clear();
+            myBookKeepingTable.clear ();
+            myGroupByEvaluator.clear ();
         }
      }

-      public static string getRepString(DataPacket dp, string[] colNames) {
-         return BlueDataUtility.rowToString(getRepRow(dp), colNames);
+      public static string getRepString (DataPacket dp, string[] colNames)
+      {
+         return BlueDataUtility.rowToString (getRepRow (dp), colNames);
      }

-      public static IBlueRow getRepRow(DataPacket dp) {
+      public static IBlueRow getRepRow (DataPacket dp)
+      {
         if (dp.newRows.Count > 0)
-            return BlueDataUtility.extractRow(dp.newRows, 0);
+            return BlueDataUtility.extractRow (dp.newRows, 0);
         else if (dp.addedRows.Count > 0)
-            return BlueDataUtility.extractRow(dp.addedRows, 0);
+            return BlueDataUtility.extractRow (dp.addedRows, 0);
         else if (dp.deletedRows.Count > 0)
-            return BlueDataUtility.extractRow(dp.deletedRows, 0);
+            return BlueDataUtility.extractRow (dp.deletedRows, 0);
         // BlueEventLog.logAssert (false, "Illegal Data Packet");
         return null;
      }
@@ -210,21 +237,22 @@ namespace BCG {
      /// <param name="newValue"></param>
      /// <param name="isAdd"></param>
      /// <returns></returns>
-      public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
-         if (lastValue == null || lastValue.Equals(DBNull.Value))
+      public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
+      {
+         if (lastValue == null || lastValue.Equals (DBNull.Value))
            return newValue;
         // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
-         // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
-         // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
+            // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
+            // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
         Object retVal = null;
-         if (newValue.GetType().Equals(typeof(int)))
+         if (newValue.GetType().Equals (typeof(int)))
            retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
-         else if (newValue.GetType().Equals(typeof(Double)))
+         else if (newValue.GetType().Equals (typeof(Double)))
            retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
-         else if (lastValue.GetType().Equals(typeof(Decimal)))
+         else if (lastValue.GetType().Equals (typeof(Decimal)))
            retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
-         else { }
-         // BlueEventLog.logAssert (false, "Illegal type");
+         else{}
+            // BlueEventLog.logAssert (false, "Illegal type");
         return retVal;
      }

@@ -22,8 +22,8 @@ namespace BCG {
            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
            if (maxRow != null) {
                bts.Log.assert(currMaxRow != null);
-                bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0);
-                if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) {
+                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
+                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
                    if (maxRow == null)
                        myOutputTable.removeRow(hashString);
@@ -34,7 +34,7 @@ namespace BCG {
            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
            // otherwise if maxRow > currMaxRow then its replaced
-            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0))
+            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
                myOutputTable.addRow(maxRow, hashString);
            return myProject(myOutputTable.getRow(hashString));
        }
@@ -18,7 +18,7 @@ namespace BCG {
            if (minRow != null) {
                // BlueEventLog.logAssert (currMinRow != null);
                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-                if (BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) == 0) {
+                if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
                    if (minRow == null)
                        myOutputTable.removeRow(stringRep);
@@ -27,7 +27,7 @@ namespace BCG {
                }
            }
            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
-            if (minRow != null && (currMinRow == null || BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) < 0))
+            if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
                myOutputTable.addRow(minRow, stringRep);
            return myOutputTable.getRow(stringRep);
        }
@@ -1,8 +1,10 @@
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
-
-using bts;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using mbts;
+using mbts.spark;

 namespace BCG
 {
@@ -12,7 +14,7 @@ namespace BCG
      private INonIncGroupByEvaluator _groupby_func;
      private BlueRowHasher _row_hasher;
      private string[] _groupby_cols;
-      private Stopwatch _timer = new();
+      private HighPerformanceTimer _timer = new HighPerformanceTimer();
      private int _n_total_fires = 0, _n_expensive_fires = 0;

      public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
@@ -31,27 +33,21 @@ namespace BCG
      {
         DataPacket dp = nv as DataPacket;
         ///grouping together
-         _timer.Restart();
+         _timer.start();
         IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
-         _timer.Stop();
-         var groupElapsedMs = _timer.ElapsedMilliseconds;
-         if (groupElapsedMs >= 50) {
-            Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
-         }
-         // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
+         _timer.stop();
+         // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
         /// reducing
-         _timer.Restart();
+         _timer.start();
         IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
-         _timer.Stop();
-         var applyElapsedMs = _timer.ElapsedMilliseconds;
+         _timer.stop();
         _n_total_fires++;         
-         if (applyElapsedMs >= 50)
+         if (_timer.lastIntervalTime > 50)
         {
            _n_expensive_fires++;
-            Log.DataFlow.info(
-               $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
+            // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
         }         
         myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));             
-      }
+      }      
   }
 }
@@ -27,8 +27,7 @@ namespace BCG
            string prependString = isLeft ? "left" : "right";
            return String.Concat (prependString, colName).GetHashCode ();
         }
-         public override bool Equals (object? obj)  =>
-            GetHashCode() == obj?.GetHashCode();
+         public override bool Equals (object obj)  {return GetHashCode () == obj.GetHashCode ();}
      }


@@ -10,9 +10,9 @@ namespace BCG
 {
   public class JoinStream : ViewStream
   {
-      protected InternalJoinStream? myIjs;
-      protected ProjectStream? myPs;
-      protected ColumnMap? myLeftCm, myRightCm;
+      protected InternalJoinStream myIjs;
+      protected ProjectStream myPs;
+      protected ColumnMap myLeftCm, myRightCm;
      protected bool myIsProject = true;
      private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
      
@@ -1,10 +1,10 @@
-using bts;
-
-using mbts;
-
-using System;
+using System;
 using System.Collections.Generic;
-using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using mbts;
+using mbts.spark;

 namespace BCG
 {
@@ -40,7 +40,8 @@ namespace BCG

      public void _on_stream_cb(object dp_obj)
      {
-         var timer = Stopwatch.StartNew();
+         HighPerformanceTimer timer = new HighPerformanceTimer();
+         timer.start();
         DataPacket dp = (DataPacket)dp_obj;
         if (_schema == null)
         {
@@ -66,12 +67,13 @@ namespace BCG
                  _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
            }
         }
-         timer.Stop();
+         timer.stop();
         _n_total_fires++;
-         if (timer.ElapsedMilliseconds >= 50) {
+         if (timer.lastIntervalTime > 50)
+         {
            _n_expensive_fires++;
-            Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
-               + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
+            string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
+            // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
         }
         if (on_new_data != null)
            on_new_data();
@@ -1,10 +1,10 @@
-using bts;
-
-using mbts;
-
-using System;
+using System;
 using System.Collections.Generic;
-using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using mbts;
+using mbts.spark;

 namespace BCG
 {
@@ -114,7 +114,8 @@ namespace BCG
         if (_is_ready())
         {
            _construct_schema();
-            var timer = Stopwatch.StartNew();
+            HighPerformanceTimer timer = new HighPerformanceTimer();
+            timer.start();
            IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
            List<object[]> output_rows = new List<object[]>();

@@ -137,12 +138,13 @@ namespace BCG
               }
            }
            IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
-            timer.Stop();
+            timer.stop();
            _n_total_fires++;
-            if (timer.ElapsedMilliseconds >= 250) {
+            if (timer.lastIntervalTime >= 250)
+            {
               _n_expensive_fires++;
-               string log_msg = $"Join and map time for {_book_keeping_data.Count} "
-                  + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
+               string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
+               // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
            }
            myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
         }
@@ -152,12 +154,15 @@ namespace BCG
      {
         if (_output_mapper != null)
         {
+            HighPerformanceTimer timer = new HighPerformanceTimer();
            IBlueRowCollection output_rows;
-            var timer = Stopwatch.StartNew();
+            timer.start();
            output_rows = _output_mapper.map_rows(unmapped_rows);
-            timer.Stop();
-            if (timer.ElapsedMilliseconds >= 50) {
-               Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
+            timer.stop();
+            if (timer.lastIntervalTime >= 50)
+            {
+               // string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
+               // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
            }
            return output_rows;
         }
Author	SHA1	Message	Date
Mohammad Arham	518bd5771e	Merge branch 'development.activ_md' of bitbucket.org:bluetradesys/spark into development.activ_md	2024-04-09 12:07:41 -04:00
Mohammad Arham	16522056ba	active cme working small fixex and conf left	2024-03-19 14:17:52 -04:00
John Colvin	51178d85d3	Add Activ md libs to sparkd WARNING: This breaks the sparkd clang build (only works with gcc)	2024-03-19 14:17:50 -04:00
John Colvin	86b50e58f4	Add Activ md libs to sparkd WARNING: This breaks the sparkd clang build (only works with gcc)	2024-03-19 14:07:11 -04:00
Mohammad Arham	8431c05dfe	active cme working small fixex and conf left	2024-03-19 13:59:41 -04:00
Mohammad Arham	18c05e2c8c	fixed hardcoded values, should be config	2024-03-19 13:58:48 -04:00
Mohammad Arham	06535c0938	spelling fix	2024-03-19 13:58:48 -04:00
Mohammad Arham	0d8f01c8bb	fixed get trades	2024-03-19 13:58:48 -04:00
Mohammad Arham	36f85b1e62	fixed sec spell	2024-03-19 13:58:48 -04:00
John Colvin	502a22a2d9	Add Activ md libs to sparkd WARNING: This breaks the sparkd clang build (only works with gcc)	2024-03-19 13:58:48 -04:00
John Colvin	daa49a5568	build: Upgrade gcc to v12 (explicitly)	2024-03-19 13:58:48 -04:00
Mohammad Arham	0ef371cc40	merged dev into activ	2024-03-14 08:32:53 -04:00
Mohammad Arham	b23f0fc21b	Merge branch 'development.activ_md' of https://bitbucket.org/bluetradesys/spark into development.activ_md	2024-02-15 11:37:26 -05:00
Mohammad Arham	d0dc1b8e51	fixed hardcoded values, should be config	2024-02-15 11:35:02 -05:00
Mohammad Arham	0bbc4f737b	spelling fix	2024-02-15 11:34:26 -05:00
Mohammad Arham	29fb1b427b	fixed get trades	2024-02-15 11:34:08 -05:00
Mohammad Arham	9de6d3ffe6	fixed sec spell	2024-02-15 11:16:49 -05:00
Mohammad Arham	a168d4da99	active cme working small fixex and conf left	2024-02-12 04:38:15 -05:00
John Colvin	d65d4b3258	Add Activ md libs to sparkd WARNING: This breaks the sparkd clang build (only works with gcc)	2024-01-23 09:56:33 -08:00
John Colvin	b4771abad0	build: Upgrade gcc to v12 (explicitly)	2024-01-23 09:56:33 -08:00
				`@@ -1 +0,0 @@`
				<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>